From f783bb939d9accde56d853c4e8c2c9cfb994311a Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 13 Oct 2026 06:41:49 -0700 Subject: [PATCH 01/25] Add fluent WithStatusResult/WithErrorStatus to HttpRequestBuilder, MCP sample and docs updates Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .github/copilot-instructions.md | 2 + .../coreex-host-setup.instructions.md | 2 + .../coreex-repositories.instructions.md | 43 +- .github/skills/coreex-adapter/SKILL.md | 9 +- .../coreex-adapter/references/workflow.md | 44 +- CoreEx.Samples.Build.slnf | 1 + CoreEx.slnx | 3 + Directory.Packages.props | 2 + .../.github/copilot-instructions.md | 2 +- samples/README.md | 3 +- samples/aspire/Contoso.Aspire/AppHost.cs | 9 +- .../Contoso.Aspire/Contoso.Aspire.csproj | 1 + .../Contoso.Test.Aspire.csproj | 2 + .../aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs | 103 +++ samples/aspire/Contoso.Test.Aspire/E2ETest.cs | 6 +- .../aspire/Contoso.Test.Aspire/GlobalUsing.cs | 3 + samples/docs/aspire.md | 25 +- samples/docs/infrastructure-layer.md | 22 +- samples/src/Contoso.Mcp/Contoso.Mcp.csproj | 13 + samples/src/Contoso.Mcp/GlobalUsing.cs | 17 + samples/src/Contoso.Mcp/Program.cs | 24 + .../Properties/launchSettings.json | 14 + .../src/Contoso.Mcp/Tools/CommerceTools.cs | 162 +++++ .../Contoso.Mcp/appsettings.Development.json | 9 + .../Clients/Customers/CustomersHttpClient.cs | 14 +- .../Clients/Products/ProductsHttpClient.cs | 7 +- .../Clients/SendGrid/SendGridHttpClient.cs | 7 +- .../GlobalUsing.cs | 1 + .../Data/mcp-demo.seed.yaml | 3 + src/CoreEx/AGENTS.md | 65 +- .../CoreExExtensions.ApplicationBuilder.cs | 97 ++- src/CoreEx/Extensions.HttpClient.cs | 165 +++++ src/CoreEx/Extensions.HttpRequestMessage.cs | 6 + src/CoreEx/Extensions.HttpResponseMessage.cs | 34 +- src/CoreEx/Http/HttpRequestBuilder.cs | 455 +++++++++++++ src/CoreEx/Http/README.md | 107 ++- .../Http/ValidatedHttpRequestBuilderT.cs | 185 +++++ src/CoreEx/Http/ValidatedHttpResponseT.cs | 93 +++ src/CoreEx/README.md | 30 +- .../ValidatorExtensions.Requires.cs | 12 + .../Http/HttpClientExtensionsTests.cs | 55 ++ .../CoreEx.Test.Unit/Http/HttpClientTests.cs | 9 +- .../Http/HttpRequestBuilderTests.cs | 640 ++++++++++++++++++ .../HttpResponseMessageExtensionsTests.cs | 85 +++ .../Validation/ValidationTests.cs | 25 +- 45 files changed, 2486 insertions(+), 130 deletions(-) create mode 100644 samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs create mode 100644 samples/src/Contoso.Mcp/Contoso.Mcp.csproj create mode 100644 samples/src/Contoso.Mcp/GlobalUsing.cs create mode 100644 samples/src/Contoso.Mcp/Program.cs create mode 100644 samples/src/Contoso.Mcp/Properties/launchSettings.json create mode 100644 samples/src/Contoso.Mcp/Tools/CommerceTools.cs create mode 100644 samples/src/Contoso.Mcp/appsettings.Development.json create mode 100644 samples/tests/Contoso.Customers.Test.Common/Data/mcp-demo.seed.yaml create mode 100644 src/CoreEx/Extensions.HttpClient.cs create mode 100644 src/CoreEx/Http/HttpRequestBuilder.cs create mode 100644 src/CoreEx/Http/ValidatedHttpRequestBuilderT.cs create mode 100644 tests/CoreEx.Test.Unit/Http/HttpClientExtensionsTests.cs create mode 100644 tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b08fa8480..c3360c55d 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -112,6 +112,8 @@ Connection strings for each service in development are in each host's `appsettin ### Adapters (Anti-Corruption Layer) - When a domain needs to interact with another domain or external service, define an **adapter interface** in `Application\Adapters\`. The Application layer depends on this domain-idiomatic abstraction — never on the remote API's schema or transport directly. - Infrastructure implements the adapter using a **typed HTTP client** (`Infrastructure\Clients\`) for the transport concern, keeping client and orchestration in separate focused classes. +- Register typed HTTP clients in `Program.cs` via `builder.AddTypedHttpClient("ServiceName")` (or `builder.AddNamedHttpClient("ServiceName")`), which automatically wires base address configuration, `IdempotencyKeyHandler`, service discovery, and `Microsoft.Extensions.Http.Resilience` standard resilience handlers. +- Inside typed clients, use `client.Send(...)` or `client.SendValue(...)` returning `HttpRequestBuilder` for fluent request building (`WithQuery`, `WithPaging`, headers, `WithStatusResult`, `WithValidator`) and single-await execution with automatic response disposal. - Two adapter roles appear in Shopping: - **Synchronous adapter** (`IProductAdapter`) — real-time calls (e.g. inventory reservation at checkout); the HTTP client is called live inside the unit of work. - **Sync/replication adapter** (`IProductSyncAdapter`) — event-driven data replication; receives published domain events and maintains a local eventually-consistent copy in the domain's own store. diff --git a/.github/instructions/coreex-host-setup.instructions.md b/.github/instructions/coreex-host-setup.instructions.md index 8d2308981..064d22378 100644 --- a/.github/instructions/coreex-host-setup.instructions.md +++ b/.github/instructions/coreex-host-setup.instructions.md @@ -176,6 +176,7 @@ The scaffolded Relay `Program.cs` wiring is described in [Outbox Relay Host](#ou | Package | Key registrations | |---|---| +| `CoreEx` | `AddHostSettings()`, `AddNamedHttpClient()`, `AddTypedHttpClient()`, `AddPrecisionTimeProvider()` | | `CoreEx.AspNetCore` | `AddMvcWebApi()`, `AddHttpWebApi()`, `AddExecutionContext()`, `UseCoreExExceptionHandler()`, `UseExecutionContext()`, `UseIdempotencyKey()`, `MapHealthChecks()` | | `CoreEx.AspNetCore.NSwag` | `AddOpenApiDocument()`, `AddCoreExConfiguration()`, `UseOpenApi()`, `UseSwaggerUi()` | | `CoreEx.Caching.FusionCache` | `AddFusionCache()`, `AddFusionHybridCache()`, `AddDefaultCacheKeyProvider()`, `AddHybridCacheIdempotencyProvider()` | @@ -194,6 +195,7 @@ The scaffolded Relay `Program.cs` wiring is described in [Outbox Relay Host](#ou | Package | Key registrations | |---|---| +| `CoreEx` | `AddNamedHttpClient()`, `AddTypedHttpClient()` (when calling external APIs directly from subscribers or adapters) | | `CoreEx.AspNetCore` | `AddMvcWebApi()`, `AddHttpWebApi()`, `AddExecutionContext()`, `AddHostedServiceManager()`, `UseCoreExExceptionHandler()`, `UseExecutionContext()`, `MapHealthChecks()`, `MapHostedServices()` | | `CoreEx.Caching.FusionCache` | `AddFusionCache()`, `AddFusionHybridCache()`, `AddDefaultCacheKeyProvider()`, `AddHybridCacheIdempotencyProvider()` | | `CoreEx.Events` | `AddEventFormatter()`, `AddSubscribedManager()` | diff --git a/.github/instructions/coreex-repositories.instructions.md b/.github/instructions/coreex-repositories.instructions.md index 46bf7ad15..09c042dc7 100644 --- a/.github/instructions/coreex-repositories.instructions.md +++ b/.github/instructions/coreex-repositories.instructions.md @@ -23,7 +23,7 @@ tags: ["repositories", "infrastructure", "data-access", "efcore", "cosmos", "map | Package | Key types provided | |---|---| -| `CoreEx` | `[ScopedService]`, `.ThrowIfNull()`, `ItemsResult`, `Result`, `.GoAsync()`, `.ThenAs()`, `.ThenAsAsync()` | +| `CoreEx` | `[ScopedService]`, `.ThrowIfNull()`, `ItemsResult`, `Result`, `.GoAsync()`, `.ThenAs()`, `.ThenAsAsync()`, `HttpRequestBuilder`, `client.Send(...)`, `client.SendValue(...)`, `builder.AddNamedHttpClient(...)`, `builder.AddTypedHttpClient(...)` | | `CoreEx.Validation` | `Validator`, `Validator.Create()`, `.HasProperty(...)`, `.Mandatory()`, `.Email()` for inline response-contract rules | | `CoreEx.Events` | `EventData` | | `CoreEx.Data` | `IUnitOfWork`, `DataResult`, `QueryArgsConfig`, `QueryFilterOperator`, `.Where(parsed)`, `.OrderBy(parsed)` | @@ -444,11 +444,10 @@ public class ProductsHttpClient(HttpClient httpClient) { private readonly HttpClient _httpClient = httpClient.ThrowIfNull(); - public async Task CreateReservationAsync(MovementRequest request, CancellationToken cancellationToken = default) - { - var response = await _httpClient.PostAsJsonAsync("api/inventory/reserve", request, JsonDefaults.SerializerOptions, cancellationToken); - return await response.ToResultAsync(); - } + public Task CreateReservationAsync(MovementRequest request, CancellationToken cancellationToken = default) + => _httpClient + .SendValue(HttpMethod.Post, "api/inventory/reserve", request) + .ToResultAsync(cancellationToken); } // Infrastructure/Adapters/ProductAdapter.cs @@ -468,6 +467,10 @@ public class ProductAdapter(ShoppingEfDb ef, ProductsHttpClient client, IEventPu Keep the typed HTTP client and the adapter orchestration in separate, independently testable classes. +Register typed HTTP clients in the host `Program.cs` via `builder.AddTypedHttpClient("ProductsApi")` (or `builder.AddNamedHttpClient("ProductsApi")`). This automatically binds `BaseAddress` from configuration, adds `IdempotencyKeyHandler`, wires service discovery, and adds `Microsoft.Extensions.Http.Resilience` standard resilience handlers. + +Inside the client, use `_httpClient.Send(...)` and `_httpClient.SendValue(...)` to fluently construct requests (adding headers, queries, or paging) and execute them. Terminal methods on `HttpRequestBuilder` (`ToResultAsync`, `GetValueAsync`, `GetStringAsync`, etc.) execute the request and **automatically dispose** the underlying `HttpResponseMessage`. + ### Client response validation Keep response-contract rules in the typed client, not Application business/request validators. @@ -478,19 +481,39 @@ validator under `Clients/{ExternalDomain}/` only for substantial rules or genuin private static readonly Validator _validator = Validator.Create() .HasProperty(x => x.Email, c => c.Mandatory().Email()); -// Inside the asynchronous client method; handle expected statuses before conversion. -using var response = await _httpClient.GetAsync($"api/customers/{Uri.EscapeDataString(id)}", ct).ConfigureAwait(false); -return await response.WithValidator(_validator).ToResultAsync(ct).ConfigureAwait(false); +// Fluent builder execution with custom status mapping, validation, and automatic response disposal: +public Task> GetAsync(string id, CancellationToken ct = default) + => _httpClient + .Send(HttpMethod.Get, $"api/customers/{Uri.EscapeDataString(id)}") + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .WithValidator(_validator) + .ToResultAsync(ct); + +// Or when inspecting raw response directly (caller owns response disposal): +public async Task> GetAsync(string id, CancellationToken ct = default) +{ + using var response = await _httpClient + .Send(HttpMethod.Get, $"api/customers/{Uri.EscapeDataString(id)}") + .GetResponseAsync(ct).ConfigureAwait(false); + + return await response + .WithValidator(_validator) + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .ToResultAsync(ct).ConfigureAwait(false); +} ``` Validate only fields the consumer needs; do not impose create/update rules or identifier matching by default. Do not capture request state/scoped dependencies in a shared validator. +Matched status codes from `.WithStatusResult(...)` short-circuit execution before deserialization and validation. The wrapper validates successful non-null values and returns the validation result's `Value`. Reported errors become `HttpRequestException` with inner validation diagnostics: failed `ToResult*`, throwing `GetValue*`, standard internal 500 rather than caller 400. Required methods reject null; `OrDefault` methods allow null and skip validation for absent values. HTTP/ProblemDetails handling, deserialization exceptions, unexpected validator exceptions, and cancellation remain unchanged. -Dispose the response at the call site; the wrapper does not own it. +When executing terminal methods on `HttpRequestBuilder`, response disposal is automatic; when retrieving +raw responses via `GetResponseAsync()` or directly awaiting `HttpClient.SendAsync`, the caller retains +responsibility for disposing the response (`using var`). Test inline rules through the client, including invalid 2xx responses, exact inner messages, and valid minimal responses; do not expose private validators just for tests. diff --git a/.github/skills/coreex-adapter/SKILL.md b/.github/skills/coreex-adapter/SKILL.md index 7590b3d24..fca62513c 100644 --- a/.github/skills/coreex-adapter/SKILL.md +++ b/.github/skills/coreex-adapter/SKILL.md @@ -43,12 +43,13 @@ Guides you through adding or modifying an adapter — the boundary that isolates - Interface surface must be **domain-idiomatic** — never a mirror of the remote API's shape - Returns `Result` or `Result` — not plain values or thrown exceptions - `[ScopedService]` on the implementation — auto-discovered via `AddDynamicServicesUsing()` -- Typed HTTP client registered via `builder.AddTypedHttpClient("ServiceName")` in `Program.cs` +- Typed HTTP client registered via `builder.AddTypedHttpClient("ServiceName")` in `Program.cs` — auto-configures `BaseAddress`, `IdempotencyKeyHandler`, service discovery, and standard resilience handlers - Never call `HttpClient` directly in adapter methods — always delegate to the typed client class -- **Response validation belongs to the client**, not `Application/Validators/`. For small, client-only rules use a private static readonly `Validator` configured once with `Validator.Create().HasProperty(...)`, then `response.WithValidator(_validator).ToResultAsync(ct)`. Validate fields the consumer requires; do not copy create/update rules or add identifier matching by default. Extract a colocated validator only when complexity or genuine reuse warrants it; see the workflow. -- Dispose the response with `using var`; `WithValidator` does not own it. Reported validation errors become `HttpRequestException` failures with inner diagnostics, not caller validation errors. Test inline rules through the client, including invalid 2xx responses and success; cached adapters must not cache invalid responses. +- Inside the client, use `client.Send(...)` and `client.SendValue(...)` returning `HttpRequestBuilder` for fluent request building (headers, queries, paging) and terminal execution with automatic response disposal +- **Response validation belongs to the client**, not `Application/Validators/`. For small, client-only rules use a private static readonly `Validator` configured once with `Validator.Create().HasProperty(...)`, then chain `.WithValidator(_validator).ToResultAsync(ct)`. Validate fields the consumer requires; do not copy create/update rules or add identifier matching by default. Extract a colocated validator only when complexity or genuine reuse warrants it; see the workflow. +- Terminal methods on `HttpRequestBuilder` (`ToResultAsync`, `GetValueAsync`, etc.) **automatically dispose** the response. When inspecting raw responses (e.g. `builder.GetResponseAsync()`), dispose the response with `using var`. Reported validation errors become `HttpRequestException` failures with inner diagnostics, not caller validation errors. Test inline rules through the client, including invalid 2xx responses and success; cached adapters must not cache invalid responses. - `response.ToResultAsync()` maps HTTP status to `Result` (2xx → `Success`, error + ProblemDetails → `BusinessException`/`ProblemDetailsException`, plain error → `HttpRequestException`) -- `response.ToResultAsync()` is for responses that **must** carry a value — a null/empty body (including 204) is an `HttpRequestException` failure, so the result is `Result` with a non-null `T` and callers need no null handling. Use `ToResultOrDefaultAsync()` (→ `Result`) only where a missing body is genuinely valid; `GetValueAsync()`/`GetValueOrDefaultAsync()` are the throwing equivalents. Map a 404 yourself before calling (`Result.NotFoundError()`) when "not found" is an expected outcome +- `response.ToResultAsync()` is for responses that **must** carry a value — a null/empty body (including 204) is an `HttpRequestException` failure, so the result is `Result` with a non-null `T` and callers need no null handling. Use `ToResultOrDefaultAsync()` (→ `Result`) only where a missing body is genuinely valid; `GetValueAsync()`/`GetValueOrDefaultAsync()` are the throwing equivalents. Map specific status codes fluently using `.WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError())` (which short-circuits validation and deserialization) when "not found" is an expected outcome - **Real-time read adapters should cache successes** with `IHybridCache.GetOrCreateWithResultAsync(key, factory, options, ct)` — the adapter method becomes one line; only successful results are cached (failures such as not-found are never cached). Use short expiry via `HybridCacheEntryOptions.CreateFor(...)` (overridable through `CoreEx:Caching:{Type}:*`). Concurrent cache misses are collapsed to one factory call per key per node with `FusionHybridCache` (cross-node needs an opt-in FusionCache distributed locker; non-FusionCache implementations such as `MemoryOnlyHybridCache` don't collapse them) — keep the factory an idempotent read. Tests must then clear the cache per test and register a hybrid cache in the unit-test host (see `references/workflow.md` → "Caching a real-time adapter") - Two distinct adapter roles in the samples — **synchronous** (real-time HTTP + events) and **replication** (event-driven local-store write; EF in relational domains, `CosmosDb` containers in Cosmos domains); separate interfaces, separate implementations - **Generate `*.Test.Unit/Clients/{ExternalDomain}/{External}HttpClientTests.cs`** for a typed HTTP client used by a synchronous or replication adapter — cover success (2xx), server error (5xx), and business error (422/ProblemDetails) per endpoint; skip for store-only replication adapters (no HTTP) and for combined local-store+HTTP+events orchestration adapters (mock the client via `Test.ReplaceHttpClientFactory` in integration tests instead) diff --git a/.github/skills/coreex-adapter/references/workflow.md b/.github/skills/coreex-adapter/references/workflow.md index 3480b4950..f9dab1736 100644 --- a/.github/skills/coreex-adapter/references/workflow.md +++ b/.github/skills/coreex-adapter/references/workflow.md @@ -74,6 +74,8 @@ public interface IProductSyncAdapter Create in `Infrastructure/Clients/{ExternalDomain}/`. One class per external service; one method per remote endpoint. +Use `_httpClient.Send(...)` and `_httpClient.SendValue(...)` to fluently construct requests (adding headers, query parameters, or paging) and execute them. Terminal methods on `HttpRequestBuilder` (`ToResultAsync`, `GetValueAsync`, `GetStringAsync`, etc.) execute the request and **automatically dispose** the underlying `HttpResponseMessage`. + ### Client class ```csharp @@ -88,11 +90,10 @@ public class ProductsHttpClient(HttpClient httpClient) private readonly HttpClient _httpClient = httpClient.ThrowIfNull(); /// Creates a new inventory reservation. - public async Task CreateReservationAsync(MovementRequest request, CancellationToken ct = default) - { - using var response = await _httpClient.PostAsJsonAsync("api/inventory/reserve", request, JsonDefaults.SerializerOptions, ct).ConfigureAwait(false); - return await response.ToResultAsync(ct).ConfigureAwait(false); - } + public Task CreateReservationAsync(MovementRequest request, CancellationToken ct = default) + => _httpClient + .SendValue(HttpMethod.Post, "api/inventory/reserve", request) + .ToResultAsync(ct); } ``` @@ -107,17 +108,29 @@ public class ExternalHttpClient(HttpClient httpClient) private readonly HttpClient _httpClient = httpClient.ThrowIfNull(); - public async Task> GetAsync(string id, CancellationToken ct = default) - { - using var response = await _httpClient.GetAsync($"api/entities/{Uri.EscapeDataString(id)}", ct).ConfigureAwait(false); - if (response.StatusCode == HttpStatusCode.NotFound) - return Result.NotFoundError(); - - return await response.WithValidator(_validator).ToResultAsync(ct).ConfigureAwait(false); - } + // Fluent builder execution with custom status mapping, validation, and automatic response disposal: + public Task> GetAsync(string id, CancellationToken ct = default) + => _httpClient + .Send(HttpMethod.Get, $"api/entities/{Uri.EscapeDataString(id)}") + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .WithValidator(_validator) + .ToResultAsync(ct); } ``` +Matched status codes from `.WithStatusResult(...)` short-circuit execution before deserialization and validation. +Where raw response inspection is needed directly, use `GetResponseAsync` (caller owns response disposal): + +```csharp +using var response = await _httpClient + .Send(HttpMethod.Get, $"api/entities/{Uri.EscapeDataString(id)}") + .GetResponseAsync(ct).ConfigureAwait(false); + +return await response.WithValidator(_validator) + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .ToResultAsync(ct).ConfigureAwait(false); +``` + `ToResultAsync()` returns a non-null `T`: a successful response with a null/empty body (including 204) is an `HttpRequestException` failure, so no `?? throw` is needed. Use `ToResultOrDefaultAsync()` (→ `Result`) only where an absent body is a valid outcome. Map 404 yourself when "not found" is an expected business outcome (surface it as `Result.NotFoundError()`; the Application-layer policy decides how to present it). ### Response-contract validation @@ -143,8 +156,9 @@ Do not include raw response bodies or sensitive values in outer error messages. `ToResultAsync`/`GetValueAsync` reject missing content and null validated values. `ToResultOrDefaultAsync`/`GetValueOrDefaultAsync` allow null and skip validation for absent response values. Serializer options and cancellation tokens are supported; cancellation, deserialization -exceptions, and unexpected validator exceptions propagate unchanged. The wrapper does not dispose, -buffer, or cache the response; retain `using var` at the call site. +exceptions, and unexpected validator exceptions propagate unchanged. When using terminal methods +on `HttpRequestBuilder`, the response is disposed automatically. When operating on raw +`HttpResponseMessage` directly or via `GetResponseAsync()`, retain `using var` at the call site. Test private inline rules through `{External}HttpClientTests`, not reflection or a new public validator solely for testing. Cover missing required fields, invalid format, valid minimal/full diff --git a/CoreEx.Samples.Build.slnf b/CoreEx.Samples.Build.slnf index a16288d3b..a46569ad2 100644 --- a/CoreEx.Samples.Build.slnf +++ b/CoreEx.Samples.Build.slnf @@ -34,6 +34,7 @@ "samples/src/Contoso.Shopping.Database/Contoso.Shopping.Database.csproj", "samples/src/Contoso.Shopping.Domain/Contoso.Shopping.Domain.csproj", "samples/src/Contoso.Shopping.Infrastructure/Contoso.Shopping.Infrastructure.csproj", + "samples/src/Contoso.Mcp/Contoso.Mcp.csproj", "samples/src/Contoso.Shopping.Relay/Contoso.Shopping.Relay.csproj", "samples/src/Contoso.Shopping.Subscribe/Contoso.Shopping.Subscribe.csproj", "samples/tests/Contoso.Customers.Test.Api/Contoso.Customers.Test.Api.csproj", diff --git a/CoreEx.slnx b/CoreEx.slnx index 87696666d..dbc3472b5 100644 --- a/CoreEx.slnx +++ b/CoreEx.slnx @@ -104,6 +104,9 @@ + + + diff --git a/Directory.Packages.props b/Directory.Packages.props index b7889c434..54b94b35e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -13,6 +13,8 @@ + + diff --git a/consumer-instructions/.github/copilot-instructions.md b/consumer-instructions/.github/copilot-instructions.md index f4e23ae30..f6b5af10a 100644 --- a/consumer-instructions/.github/copilot-instructions.md +++ b/consumer-instructions/.github/copilot-instructions.md @@ -18,7 +18,7 @@ CoreEx is a modular .NET framework for enterprise back-end services. It provides | NuGet Package | Capability | |---|---| -| `CoreEx` | Core: exceptions, `Result`, execution context, DI helpers, entity markers, JSON, mapping, and Roslyn source generator for `[Contract]`-decorated and `[ReferenceData]`-decorated types | +| `CoreEx` | Core: exceptions, `Result`, execution context, DI helpers, HTTP client/builder extensions, entity markers, JSON, mapping, and Roslyn source generator for `[Contract]`-decorated and `[ReferenceData]`-decorated types | | `CoreEx.AspNetCore` | HTTP API: `WebApi` helpers, idempotency, health checks, middleware | | `CoreEx.AspNetCore.NSwag` | OpenAPI / NSwag integration | | `CoreEx.Validation` | Fluent validation: `Validator`, `AbstractValidator`, rules | diff --git a/samples/README.md b/samples/README.md index d248aeebf..6d3905a79 100644 --- a/samples/README.md +++ b/samples/README.md @@ -110,6 +110,7 @@ See [Patterns](docs/patterns.md) for the full catalog of architectural patterns |---|---| | `src/Contoso.Products.*` | Products domain — Contracts, Application, Infrastructure, API, Relay, Subscribe, CodeGen, Database | | `src/Contoso.Shopping.*` | Shopping domain — same layer split plus Domain aggregate | +| `src/Contoso.Mcp` | HTTP MCP facade that composes the Products and Shopping APIs | | `src/Contoso.Customers.*` | Customers domain (Cosmos DB) — Contracts, Application, Infrastructure, API, Relay, CodeGen, Database (Cosmos provisioning console); no Subscribe project | | `src/Contoso.Orders.*` | Orders domain (work in progress) | | `aspire/Contoso.Aspire` | Aspire AppHost — orchestrates all hosts for local development and E2E validation | @@ -166,7 +167,7 @@ See [Tooling](docs/tooling.md) for the full list of database commands and what e aspire run ``` -This starts every domain API, Outbox Relay, and Subscribe host as a single distributed application and opens the Aspire Dashboard for centralized logs, traces, and metrics. See [Aspire & E2E](docs/aspire.md) for a full walkthrough. +This starts every domain API, Outbox Relay, Subscribe host, and the Contoso MCP endpoint as a single distributed application and opens the Aspire Dashboard for centralized logs, traces, and metrics. See [Aspire & E2E](docs/aspire.md) for a full walkthrough and the [Contoso MCP demo](docs/aspire.md#contoso-mcp-demo) for connecting a local MCP client. ### 4 — Run Tests diff --git a/samples/aspire/Contoso.Aspire/AppHost.cs b/samples/aspire/Contoso.Aspire/AppHost.cs index 19bd3865a..754d9c3ca 100644 --- a/samples/aspire/Contoso.Aspire/AppHost.cs +++ b/samples/aspire/Contoso.Aspire/AppHost.cs @@ -31,7 +31,7 @@ // "https://localhost:7200" (Products) / "https://localhost:7320" (Customers); here the logical "https+http://products-api" / "https+http://customers-api" is injected instead, and WithReference populates the // "Services:products-api:*" / "Services:customers-api:*" configuration used by CoreEx's service-discovery-aware AddTypedHttpClient, so the resolved address // always matches whichever endpoint/port the target actually binds to (regardless of launch profile), rather than a static guess. -builder.AddProject("shopping-api") +var shoppingApi = builder.AddProject("shopping-api") .WithReference(sqlServer) .WithReference(redis) .WithReference(serviceBus) @@ -55,6 +55,13 @@ .AddHostedServiceSupport() .WithMockHostEnvironment("SendGrid__BaseAddress", mockhost, "http"); +builder.AddProject("contoso-mcp") + .WithReference(productsApi) + .WithEnvironment("ProductsApi__BaseAddress", "https+http://products-api") + .WithReference(shoppingApi) + .WithEnvironment("ShoppingApi__BaseAddress", "https+http://shopping-api") + .AddEndpoints("/health"); + // Orders domain. var orderWorkflowWorker = builder.AddProject("order-workflow-worker").AddEndpoints("/health").WithUrlForEndpoint("https", ep => { ep.Url = "http://localhost:8082"; ep.DisplayText = "DTS Dashboard"; }); builder.AddProject("orders-api").WithReference(sqlServer).WithReference(redis).WaitFor(orderWorkflowWorker).AddEndpoints("/health/ready/detailed"); diff --git a/samples/aspire/Contoso.Aspire/Contoso.Aspire.csproj b/samples/aspire/Contoso.Aspire/Contoso.Aspire.csproj index dad3ae73e..9ae30f9f3 100644 --- a/samples/aspire/Contoso.Aspire/Contoso.Aspire.csproj +++ b/samples/aspire/Contoso.Aspire/Contoso.Aspire.csproj @@ -55,6 +55,7 @@ + diff --git a/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj b/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj index ed3c0d2c9..4832a2dd6 100644 --- a/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj +++ b/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj @@ -19,6 +19,7 @@ + @@ -32,6 +33,7 @@ + diff --git a/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs b/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs new file mode 100644 index 000000000..9caf80c9e --- /dev/null +++ b/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs @@ -0,0 +1,103 @@ +namespace Contoso.Test.Aspire; + +public partial class E2ETest +{ + private string _mcpEndpoint = string.Empty; + + [Test] + public async Task Contoso_Mcp_FullBasketLifecycle() + { + Test.Checkpoint("Increase inventory for the seeded, active Product used by the MCP basket scenario."); + var productId = 28.ToGuid().ToString(); + Test.Http(_productsApi) + .Run(HttpMethod.Post, "/api/inventory/adjust", new MovementRequest + { + Id = Runtime.NewGuid().ToString(), + Products = new() { { productId, new MovementRequestProduct { Quantity = 5, UnitOfMeasureCode = "EA" } } } + }) + .AssertOK(); + + await using var client = await McpClient.CreateAsync(new HttpClientTransport(new HttpClientTransportOptions + { + Endpoint = new Uri($"{_mcpEndpoint}/mcp"), + EnableStandaloneGetStream = false + })).ConfigureAwait(false); + + var tools = await client.ListToolsAsync().ConfigureAwait(false); + string[] expectedToolNames = + [ + "search_products", "get_product", "get_basket", "list_customer_baskets", + "create_basket", "add_basket_item", "update_basket_item", "remove_basket_item", + "apply_basket_discount", "update_basket_shipping_address", "checkout_basket" + ]; + tools.Select(x => x.Name).Should().Contain(expectedToolNames); + tools.Should().OnlyContain(x => x.ProtocolTool.OutputSchema.HasValue); + + Test.Checkpoint("Search for and read a Product through MCP."); + var product = await CallToolAsync(client, "get_product", new Dictionary { ["productId"] = productId }).ConfigureAwait(false); + product.GetProperty("id").GetString().Should().Be(productId); + var productSearch = await CallToolAsync(client, "search_products", new Dictionary { ["searchTerm"] = "spec", ["take"] = 25 }).ConfigureAwait(false); + productSearch.EnumerateArray().Should().Contain(x => x.GetProperty("sku").GetString()!.StartsWith("SPEC", StringComparison.Ordinal)); + + Test.Checkpoint("Create a Basket for the seeded MCP demo customer, then read and list it through MCP."); + var customerId = 16.ToGuid().ToString(); + var basketCreated = await CallToolAsync(client, "create_basket", new Dictionary { ["customerId"] = customerId }).ConfigureAwait(false); + var basketId = basketCreated.GetProperty("id").GetString() ?? throw new InvalidOperationException("The MCP tool returned a Basket without an ID."); + + (await CallToolAsync(client, "get_basket", new Dictionary { ["basketId"] = basketId }).ConfigureAwait(false)).GetProperty("id").GetString().Should().Be(basketId); + (await CallToolAsync(client, "list_customer_baskets", new Dictionary { ["customerId"] = customerId }).ConfigureAwait(false)) + .EnumerateArray().Should().Contain(x => x.GetProperty("id").GetString() == basketId); + + Test.Checkpoint("Add a Product, update its quantity, and remove a second item through MCP."); + var basketAfterAdd = await CallToolAsync(client, "add_basket_item", new Dictionary { ["basketId"] = basketId, ["productId"] = productId, ["quantity"] = 1m }).ConfigureAwait(false); + var basketItemId = basketAfterAdd.GetProperty("items")[0].GetProperty("id").GetString() ?? throw new InvalidOperationException("The MCP tool returned a Basket item without an ID."); + + var secondProductId = 29.ToGuid().ToString(); + var basketWithSecondItem = await CallToolAsync(client, "add_basket_item", new Dictionary { ["basketId"] = basketId, ["productId"] = secondProductId, ["quantity"] = 1m }).ConfigureAwait(false); + var secondItemId = basketWithSecondItem.GetProperty("items").EnumerateArray().Single(x => x.GetProperty("productId").GetString() == secondProductId).GetProperty("id").GetString() + ?? throw new InvalidOperationException("The MCP tool returned the second Basket item without an ID."); + + var updatedBasket = await CallToolAsync(client, "update_basket_item", new Dictionary { ["basketId"] = basketId, ["basketItemId"] = basketItemId, ["quantity"] = 2m }).ConfigureAwait(false); + updatedBasket.GetProperty("items").EnumerateArray().Single(x => x.GetProperty("id").GetString() == basketItemId).GetProperty("quantity").GetDecimal().Should().Be(2m); + var basketAfterRemoval = await CallToolAsync(client, "remove_basket_item", new Dictionary { ["basketId"] = basketId, ["basketItemId"] = secondItemId }).ConfigureAwait(false); + basketAfterRemoval.GetProperty("items").EnumerateArray().Should().NotContain(x => x.GetProperty("productId").GetString() == secondProductId); + + Test.Checkpoint("Apply a coupon, set a shipping address, and check out through MCP."); + (await CallToolAsync(client, "apply_basket_discount", new Dictionary { ["basketId"] = basketId, ["couponCode"] = "SAVE10" }).ConfigureAwait(false)) + .GetProperty("pricing").GetProperty("discountPercentage").GetDecimal().Should().Be(10m); + var basketWithAddress = await CallToolAsync(client, "update_basket_shipping_address", new Dictionary + { + ["basketId"] = basketId, + ["shippingAddress"] = new Address { Street1 = "123 Main St", City = "Anytown", State = "CA", PostCode = "12345" } + }).ConfigureAwait(false); + basketWithAddress.GetProperty("shippingAddress").GetProperty("street1").GetString().Should().Be("123 Main St"); + + var checkedOutBasket = await CallToolAsync(client, "checkout_basket", new Dictionary { ["basketId"] = basketId }).ConfigureAwait(false); + checkedOutBasket.GetProperty("status").GetString().Should().Be("C"); + + Test.Checkpoint("Confirm inventory is asynchronously confirmed by the real Products subscriber."); + var movements = GetMovements(basketId); + for (var attempt = 1; movements.All(x => x.StatusCode != MovementStatus.Confirmed); attempt++) + { + if (attempt > 60) + Assert.Fail($"Inventory movement for MCP Basket '{basketId}' was not confirmed after 60 polls."); + + Test.Delay(1000, $"Waiting for the MCP Basket's inventory movement to be confirmed; iteration {attempt} of 60."); + movements = GetMovements(basketId); + } + } + + private static async Task CallToolAsync(McpClient client, string name, Dictionary arguments) + { + var result = await client.CallToolAsync(name, arguments).ConfigureAwait(false); + if (result.IsError == true) + Assert.Fail($"MCP tool '{name}' failed: {result.Content.OfType().FirstOrDefault()?.Text}"); + + return result.StructuredContent ?? throw new InvalidOperationException($"MCP tool '{name}' returned no structured content."); + } + + private Movement[] GetMovements(string basketId) => Test.Http(_productsApi) + .Run(HttpMethod.Get, "/api/inventory/movements", r => r.WithQuery($"referenceid eq '{basketId}'")) + .AssertOK() + .Value!; +} diff --git a/samples/aspire/Contoso.Test.Aspire/E2ETest.cs b/samples/aspire/Contoso.Test.Aspire/E2ETest.cs index 6bf3d6335..7e6a03f00 100644 --- a/samples/aspire/Contoso.Test.Aspire/E2ETest.cs +++ b/samples/aspire/Contoso.Test.Aspire/E2ETest.cs @@ -15,7 +15,7 @@ protected override async Task OnBeforeStartAsync(DistributedApplication app) // Migrate the Products, Shopping and Customers databases and seed each with test data. await app.MigratePostgresDataAsync("Postgres", ["mutate-data.seed.yaml"], Contoso.Products.Database.Program.ConfigureMigrationArgs); await app.MigrateSqlServerDataAsync("SqlServer", ["mutate-data.seed.yaml"], Contoso.Shopping.Database.Program.ConfigureMigrationArgs); - await app.MigrateCosmosDataAsync("Cosmos", "contoso", ["mutate-data.seed.yaml"], Contoso.Customers.Database.Program.ConfigureProvisionArgs); + await app.MigrateCosmosDataAsync("Cosmos", "contoso", ["mutate-data.seed.yaml", "mcp-demo.seed.yaml"], Contoso.Customers.Database.Program.ConfigureProvisionArgs); // Clear the Redis cache. await app.ClearRedisCacheAsync("redis"); @@ -27,10 +27,12 @@ protected override async Task OnBeforeStartAsync(DistributedApplication app) protected override async Task OnAfterStartAsync(DistributedApplication app) { // Wait for every host used by the asynchronous checkout flow before running the tests. - await app.WaitForResourceAsync([_productsApi, _shoppingApi, _customersApi, "products-relay", "products-subscribe", "shopping-relay", "shopping-subscribe", "customers-relay"]); + await app.WaitForResourceAsync([_productsApi, _shoppingApi, _customersApi, "products-relay", "products-subscribe", "shopping-relay", "shopping-subscribe", "customers-relay", "contoso-mcp"]); // Mock the SendGrid API so that the Shopping domain's Subscribe project can send emails without actually sending them. await app.HttpMock("mock-host", "http").Request(HttpMethod.Post, "/v3/mail/send").WithAnyBody().Respond.WithAsync(HttpStatusCode.Accepted); + + _mcpEndpoint = app.GetEndpoint("contoso-mcp", "http").ToString().TrimEnd('/'); } [Test] diff --git a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs index 36af7f410..5c07dcbae 100644 --- a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs +++ b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs @@ -1,5 +1,6 @@ global using Aspire.Hosting; global using Aspire.Hosting.ApplicationModel; +global using Aspire.Hosting.Testing; global using AwesomeAssertions; global using Azure.Messaging.ServiceBus; global using Azure.Messaging.ServiceBus.Administration; @@ -7,6 +8,8 @@ global using Contoso.Shopping.Contracts; global using CoreEx; global using Microsoft.Extensions.Configuration; +global using ModelContextProtocol.Client; +global using ModelContextProtocol.Protocol; global using NUnit.Framework; global using System.Net; global using System.Text.Json; diff --git a/samples/docs/aspire.md b/samples/docs/aspire.md index 14abc43ca..b10524cb8 100644 --- a/samples/docs/aspire.md +++ b/samples/docs/aspire.md @@ -21,6 +21,7 @@ The `Contoso.Aspire` AppHost (`samples/aspire/Contoso.Aspire/AppHost.cs`) regist | `products-relay` | `Contoso.Products.Relay` | HTTP + `/health/ready/detailed` + hosted-service controls | | `products-subscribe` | `Contoso.Products.Subscribe` | HTTP + `/health/ready/detailed` + hosted-service controls | | `shopping-api` | `Contoso.Shopping.Api` | HTTP + `/health/ready/detailed` | +| `contoso-mcp` | `Contoso.Mcp` | Streamable HTTP MCP at `/mcp` + `/health` (Products and Shopping API calls) | | `shopping-relay` | `Contoso.Shopping.Relay` | HTTP + `/health/ready/detailed` + hosted-service controls | | `shopping-subscribe` | `Contoso.Shopping.Subscribe` | HTTP + `/health/ready/detailed` + hosted-service controls | | `order-workflow-worker` | `Contoso.Order.Workflow.Worker` | HTTP + `/health` + DTS Dashboard link | @@ -78,6 +79,28 @@ dotnet run --project samples/aspire/Contoso.Aspire Both commands start the AppHost, which in turn launches all registered projects as child processes. The Aspire Dashboard opens automatically in the browser. +### Contoso MCP demo + +`Contoso.Mcp` is a local-development sample: it has no authentication and must not be exposed to an untrusted network. It is a cross-domain HTTP facade over the real Products and Shopping APIs. Aspire wires both API references through service discovery; it does not add MCP endpoints to either domain API. The server uses stateless Streamable HTTP at `/mcp` (not stdio). Copy the `contoso-mcp` HTTP endpoint from the Aspire Dashboard and append `/mcp`; the port can vary between runs. + +The tools support product search/detail and the complete basket lifecycle: list/create baskets, read a basket, add/update/remove items, apply `SAVE10`, update the shipping address, and checkout. Successful calls return typed structured data with a tool output schema; errors are reported as MCP tool errors, not success-shaped strings. The ordinary development database seed does not include customer or product test data. For an interactive basket demo, start `Contoso.E2E.Runner` and run **Database Migration and Base Data Refresh** followed by **Data Seeding for E2E Testing**; this resets the local sample databases to test data, seeds customer `16.ToGuid()` with an address, and stocks active products. Basket creation requires an existing Customers-domain ID. Checkout is consequential: it reserves stock and triggers asynchronous confirmation through the existing outbox, relay, Service Bus, and Products subscriber. + +To connect GitHub Copilot CLI while Aspire is running, register that endpoint from a terminal: + +```powershell +copilot mcp add --transport http coreex-contoso "http://localhost:/mcp" +``` + +Replace the URL with the `contoso-mcp` HTTP URL shown in the dashboard, including `/mcp`. In Copilot CLI, use `/mcp` to see the connected server and its tools. After completing the E2E setup above, ask it to search for a product, create a basket for customer `16.ToGuid()`, add an item, and inspect the basket. Treat checkout as a separate explicit step because it changes inventory and starts asynchronous processing. When finished, remove the connection with `copilot mcp remove coreex-contoso`. + +For protocol-level testing without Copilot CLI, run `npx @modelcontextprotocol/inspector`, choose Streamable HTTP, and enter the same `/mcp` URL. The Aspire E2E test exercises discovery and every lifecycle tool through the MCP client SDK: + +```powershell +dotnet test samples/aspire/Contoso.Test.Aspire --filter "FullyQualifiedName~Contoso_Mcp_FullBasketLifecycle" +``` + +This sample and test validate the local HTTP server and MCP protocol flow; they do not establish compatibility with Copilot App Sessions or provide a stdio transport. + ### Aspire Dashboard The dashboard (default: `http://localhost:15174`) provides: @@ -96,7 +119,7 @@ The hosted-service command buttons (Pause / Resume) are also surfaced here, maki `Contoso.Test.Aspire` (`samples/aspire/Contoso.Test.Aspire/E2ETest.cs`) is an NUnit project that provides an **automated, CI-friendly** counterpart to the interactive E2E Runner described below. Unlike the E2E Runner, it does not require Aspire to already be running — its single test class derives from `WithAspireTester` (from `CoreEx.UnitTesting`'s Aspire support), which starts the whole `Contoso.Aspire` AppHost itself for the duration of the test run. -Its `OnBeforeStartAsync` override migrates and seeds the Products (Postgres), Shopping (SQL Server) and Customers (Cosmos DB) databases, clears the Redis cache, and resets the Service Bus emulator's queues/topics/subscriptions to a known state (the topology is the code-based `ServiceBus` class), then `OnAfterStartAsync` waits for the three APIs, all three relays, and the Products and Shopping subscribers. The `CreateOrderAndConfirmAsync` test drives the same cross-domain flow as the E2E Runner's **Shopping Basket Lifecycle** scenario: checkout, async inventory confirmation, and the stubbed SendGrid email. Inventory confirmation uses up to 60 one-second polling delays to accommodate relay contention and lease recovery on slower CI runners; failure reports the basket, product, last movement status, and expected command path. +Its `OnBeforeStartAsync` override migrates and seeds the Products (Postgres), Shopping (SQL Server) and Customers (Cosmos DB) databases, clears the Redis cache, and resets the Service Bus emulator's queues/topics/subscriptions to a known state (the topology is the code-based `ServiceBus` class), then `OnAfterStartAsync` waits for the three APIs, all three relays, the Products and Shopping subscribers, and the MCP host. The `CreateOrderAndConfirmAsync` test drives the same cross-domain flow as the E2E Runner's **Shopping Basket Lifecycle** scenario: checkout, async inventory confirmation, and the stubbed SendGrid email. `Contoso_Mcp_FullBasketLifecycle` drives that basket lifecycle through the cross-domain MCP facade, verifies structured outputs and their schemas, and waits for the asynchronous inventory confirmation. Inventory confirmation uses up to 60 one-second polling delays to accommodate relay contention and lease recovery on slower CI runners; failure reports the basket, product, last movement status, and expected command path. `Customers_Api_Outbox_Relay_PublishesEvents` additionally drives customer create/update/delete and reference-data create/update/activate/deactivate/delete through the real Customers API. It receives the resulting events on the isolated, session-enabled `customers-relay` subscription using the Service Bus SDK and the AppHost's resolved connection string. Bounded polling correlates messages to this test's entity identifiers and event types; it does not drain Products/Shopping subscriptions or assume exactly-once delivery. No Customers subscriber or replication flow is introduced. diff --git a/samples/docs/infrastructure-layer.md b/samples/docs/infrastructure-layer.md index ef39c399c..950e32ed6 100644 --- a/samples/docs/infrastructure-layer.md +++ b/samples/docs/infrastructure-layer.md @@ -121,17 +121,18 @@ public class ProductMapper : BiDirectionMapper CreateReservationAsync(MovementRequest request) - { - var response = await _httpClient.PostAsJsonAsync("api/inventory/reserve", request, JsonDefaults.SerializerOptions); - return await response.ToResultAsync(); - } + private readonly HttpClient _httpClient = httpClient.ThrowIfNull(); + + public Task CreateReservationAsync(MovementRequest request, CancellationToken ct = default) + => _httpClient + .SendValue(HttpMethod.Post, "api/inventory/reserve", request) + .ToResultAsync(ct); } // samples/src/Contoso.Shopping.Infrastructure/Adapters/ProductAdapter.cs @@ -142,17 +143,16 @@ public class ProductAdapter(ShoppingEfDb ef, IEventPublisher eventPublisher, Pro } ``` -This split — client for transport, adapter for orchestration — keeps each class focused on a single responsibility and makes the HTTP interaction independently testable. +This split — client for transport, adapter for orchestration — keeps each class focused on a single responsibility and makes the HTTP interaction independently testable. Using `_httpClient.Send(...)` or `_httpClient.SendValue(...)` returns an `HttpRequestBuilder`, allowing fluent setup of headers, query parameters, and paging, with automatic response disposal when calling terminal methods (`ToResultAsync`, `GetValueAsync`, etc.). `CustomersHttpClient` / `CustomerAdapter` are the purely real-time variant: no local replica, the adapter delegates to the client (`GET api/customers/{id}`), mapping a 404 to `Result.NotFoundError()`. The lookup is wrapped in a single `IHybridCache.GetOrCreateWithResultAsync(...)` call: only successful results are cached (short, configurable expiry via `CoreEx:Caching:Customer:*`), failures such as not-found are never cached. The lookup is single-flight per key (per node) under FusionCache, so concurrent misses make one remote call; coalescing across nodes sharing Redis would need an opt-in FusionCache distributed locker, which isn't worth it for this cheap read. The client is registered in the Api and Subscribe hosts with `AddTypedHttpClient("CustomersApi")`, the `CustomersApi:BaseAddress` setting being overridden with service discovery under Aspire. -Before returning a successful lookup, `CustomersHttpClient` uses -`response.WithValidator(_validator).ToResultAsync(ct)` to require an email with valid format. +`CustomersHttpClient` chains `.WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError())` and `.WithValidator(_validator).ToResultAsync(ct)` directly on `_httpClient.Send(...)`: a 404 response short-circuits directly to `Result.NotFoundError()` before deserialization or validation runs, while a 200 response requires an email with valid format and automatically disposes the HTTP response. The response-specific rules are defined once in a private static readonly `Validator` inside the client using `Validator.Create().HasProperty(...)`, leaving other details unrestricted rather than applying create/update rules to a read response. Customer identity is trusted, not validated or compared with the requested identifier. Invalid successful responses become `HttpRequestException` failures with structured validation diagnostics, so they are not cached -or mistaken for caller validation errors. The client disposes the HTTP response after consumption. +or mistaken for caller validation errors. -> **See also**: [`ToResultAsync`](../../src/CoreEx/Extensions.HttpResponseMessage.cs) · [`JsonDefaults`](../../src/CoreEx/Json/JsonDefaults.cs) · [Strangler Fig / ACL patterns](https://learn.microsoft.com/en-us/azure/architecture/patterns/strangler-fig) +> **See also**: [`HttpRequestBuilder`](../../src/CoreEx/Http/HttpRequestBuilder.cs) · [`ToResultAsync`](../../src/CoreEx/Extensions.HttpResponseMessage.cs) · [`JsonDefaults`](../../src/CoreEx/Json/JsonDefaults.cs) · [Strangler Fig / ACL patterns](https://learn.microsoft.com/en-us/azure/architecture/patterns/strangler-fig) diff --git a/samples/src/Contoso.Mcp/Contoso.Mcp.csproj b/samples/src/Contoso.Mcp/Contoso.Mcp.csproj new file mode 100644 index 000000000..5cb649e45 --- /dev/null +++ b/samples/src/Contoso.Mcp/Contoso.Mcp.csproj @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/samples/src/Contoso.Mcp/GlobalUsing.cs b/samples/src/Contoso.Mcp/GlobalUsing.cs new file mode 100644 index 000000000..1e744a05e --- /dev/null +++ b/samples/src/Contoso.Mcp/GlobalUsing.cs @@ -0,0 +1,17 @@ +global using Contoso.Products.Contracts; +global using Contoso.Shopping.Contracts; +global using CoreEx; +global using CoreEx.Http; +global using CoreEx.Json; +global using CoreEx.Validation; +global using Microsoft.AspNetCore.Mvc; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Hosting; +global using ModelContextProtocol.AspNetCore; +global using ModelContextProtocol.Server; +global using System; +global using System.ComponentModel; +global using System.Net; +global using System.Net.Http.Json; +global using System.Text.Json; diff --git a/samples/src/Contoso.Mcp/Program.cs b/samples/src/Contoso.Mcp/Program.cs new file mode 100644 index 000000000..b25262a1c --- /dev/null +++ b/samples/src/Contoso.Mcp/Program.cs @@ -0,0 +1,24 @@ +namespace Contoso.Mcp; + +public class Program +{ + public static void Main(string[] args) + { + var builder = WebApplication.CreateBuilder(args); + + builder.AddNamedHttpClient("ProductsApi"); + builder.AddNamedHttpClient("ShoppingApi"); + + builder.Services + .AddMcpServer() + .WithHttpTransport(options => options.SessionMode = HttpServerSessionMode.Stateless) + .WithToolsFromAssembly(); + + var app = builder.Build(); + + app.MapGet("/health", () => Results.Ok("Healthy")); + app.MapMcp("/mcp"); + + app.Run(); + } +} diff --git a/samples/src/Contoso.Mcp/Properties/launchSettings.json b/samples/src/Contoso.Mcp/Properties/launchSettings.json new file mode 100644 index 000000000..48835068a --- /dev/null +++ b/samples/src/Contoso.Mcp/Properties/launchSettings.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "applicationUrl": "http://localhost:7235", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs new file mode 100644 index 000000000..62db6bd25 --- /dev/null +++ b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs @@ -0,0 +1,162 @@ +namespace Contoso.Mcp.Tools; + +[McpServerToolType] +public static class CommerceTools +{ + /// Searches products by text or SKU prefix. + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products whose description contains the search phrase or whose SKU starts with it. Returns up to the requested number of matching product summaries.")] + public static Task SearchProductsAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("A word or phrase to find in product descriptions, or the beginning of a product SKU.")] string searchTerm, + [Description("Maximum number of products to return (1-25).")] int take = 10, + CancellationToken cancellationToken = default) + { + searchTerm = searchTerm.Required(); + if (take is < 1 or > 25) + throw new ArgumentOutOfRangeException(nameof(take), "The result limit must be between 1 and 25."); + + var filterTerm = searchTerm.Replace("'", "''", StringComparison.Ordinal); + var filter = $"contains(Text, '{filterTerm}') or startswith(Sku, '{filterTerm}')"; + return ProductsApi(clientFactory) + .Send(HttpMethod.Get, "/api/products") + .WithQuery(filter) + .WithPaging(take: take) + .GetValueAsync(cancellationToken); + } + + /// Gets the full details for a product. + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Get product details by product ID.")] + public static Task GetProductAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The product ID.")] string productId, + CancellationToken cancellationToken = default) + => ProductsApi(clientFactory) + .Send(HttpMethod.Get, $"/api/products/{Uri.EscapeDataString(productId.Required())}") + .GetValueAsync(cancellationToken); + + /// Gets a basket by its ID. + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Get basket status, items, pricing, and shipping address by basket ID.")] + public static Task GetBasketAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The basket ID.")] string basketId, + CancellationToken cancellationToken = default) + => ShoppingApi(clientFactory) + .Send(HttpMethod.Get, $"/api/baskets/{Uri.EscapeDataString(basketId.Required())}") + .GetValueAsync(cancellationToken); + + /// Lists baskets owned by a customer. + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("List a customer's baskets. Requires a customer ID that exists in the Customers domain.")] + public static Task ListCustomerBasketsAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The existing customer ID. The sample E2E setup seeds customer 16.ToGuid() with an address.")] string customerId, + [Description("Maximum number of baskets to return (1-25).")] int take = 10, + CancellationToken cancellationToken = default) + { + if (take is < 1 or > 25) + throw new ArgumentOutOfRangeException(nameof(take), "The result limit must be between 1 and 25."); + + return ShoppingApi(clientFactory) + .Send(HttpMethod.Get, $"/api/customers/{Uri.EscapeDataString(customerId.Required())}/baskets") + .WithPaging(take: take) + .GetValueAsync(cancellationToken); + } + + /// Creates a basket for an existing customer. + [McpServerTool(UseStructuredContent = true), Description("Create a new basket for an existing customer. The customer's shipping address is copied when available.")] + public static Task CreateBasketAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The ID of an existing customer.")] string customerId, + CancellationToken cancellationToken = default) + => ShoppingApi(clientFactory) + .Send(HttpMethod.Post, $"/api/customers/{Uri.EscapeDataString(customerId.Required())}/baskets") + .GetValueAsync(cancellationToken); + + /// Adds a product to a basket. + [McpServerTool(UseStructuredContent = true), Description("Add a product and quantity to an active basket.")] + public static Task AddBasketItemAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The basket ID.")] string basketId, + [Description("The product ID.")] string productId, + [Description("The quantity to add.")] decimal quantity, + CancellationToken cancellationToken = default) + => ShoppingApi(clientFactory) + .SendValue( + HttpMethod.Post, + $"/api/baskets/{Uri.EscapeDataString(basketId.Required())}/items", + new BasketItemAddRequest { ProductId = productId.Required(), Quantity = quantity }) + .GetValueAsync(cancellationToken); + + /// Changes the quantity of a basket item. + [McpServerTool(UseStructuredContent = true), Description("Set an item's quantity in an active basket. The current item ETag is fetched and sent to the API for concurrency validation.")] + public static async Task UpdateBasketItemAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The basket ID.")] string basketId, + [Description("The basket item ID.")] string basketItemId, + [Description("The new quantity.")] decimal quantity, + CancellationToken cancellationToken = default) + { + var client = ShoppingApi(clientFactory); + var basket = await client + .Send(HttpMethod.Get, $"/api/baskets/{Uri.EscapeDataString(basketId.Required())}") + .GetValueAsync(cancellationToken).ConfigureAwait(false); + + var item = basket.Items?.SingleOrDefault(x => x.Id == basketItemId.Required()) + ?? throw new InvalidOperationException($"Basket item '{basketItemId}' was not found in basket '{basketId}'."); + + return await client + .SendValue( + HttpMethod.Put, + $"/api/baskets/{Uri.EscapeDataString(basketId)}/items/{Uri.EscapeDataString(basketItemId)}", + new BasketItemUpdateRequest { Quantity = quantity, ETag = item.ETag }) + .GetValueAsync(cancellationToken).ConfigureAwait(false); + } + + /// Removes an item from a basket. + [McpServerTool(UseStructuredContent = true, Destructive = true), Description("Remove a product from an active basket by basket item ID.")] + public static Task RemoveBasketItemAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The basket ID.")] string basketId, + [Description("The basket item ID.")] string basketItemId, + CancellationToken cancellationToken = default) + => ShoppingApi(clientFactory) + .Send(HttpMethod.Delete, $"/api/baskets/{Uri.EscapeDataString(basketId.Required())}/items/{Uri.EscapeDataString(basketItemId.Required())}") + .GetValueAsync(cancellationToken); + + /// Applies a discount coupon to a basket. + [McpServerTool(UseStructuredContent = true), Description("Apply a coupon code to an active basket.")] + public static Task ApplyBasketDiscountAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The basket ID.")] string basketId, + [Description("The discount coupon code.")] string couponCode, + CancellationToken cancellationToken = default) + => ShoppingApi(clientFactory) + .Send(HttpMethod.Put, $"/api/baskets/{Uri.EscapeDataString(basketId.Required())}/apply-discount/{Uri.EscapeDataString(couponCode.Required())}") + .GetValueAsync(cancellationToken); + + /// Changes the shipping address for a basket. + [McpServerTool(UseStructuredContent = true), Description("Set or replace the shipping address for an active basket.")] + public static Task UpdateBasketShippingAddressAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The basket ID.")] string basketId, + Address shippingAddress, + CancellationToken cancellationToken = default) + => ShoppingApi(clientFactory) + .SendValue(HttpMethod.Put, $"/api/baskets/{Uri.EscapeDataString(basketId.Required())}/shipping-address", shippingAddress) + .GetValueAsync(cancellationToken); + + /// Checks out a basket and starts inventory reservation. + [McpServerTool(UseStructuredContent = true, Destructive = true), Description("Check out the specified active basket. This is a consequential action: it reserves inventory and starts asynchronous confirmation through the outbox and Service Bus.")] + public static Task CheckoutBasketAsync( + [FromServices] IHttpClientFactory clientFactory, + [Description("The ID of the basket to check out.")] string basketId, + CancellationToken cancellationToken = default) + => ShoppingApi(clientFactory) + .Send(HttpMethod.Post, $"/api/baskets/{Uri.EscapeDataString(basketId.Required())}/checkout") + .GetValueAsync(cancellationToken); + + /// Creates the named client for the Products API. + private static HttpClient ProductsApi(IHttpClientFactory clientFactory) => clientFactory.CreateClient("ProductsApi"); + + /// Creates the named client for the Shopping API. + private static HttpClient ShoppingApi(IHttpClientFactory clientFactory) => clientFactory.CreateClient("ShoppingApi"); +} diff --git a/samples/src/Contoso.Mcp/appsettings.Development.json b/samples/src/Contoso.Mcp/appsettings.Development.json new file mode 100644 index 000000000..17c7c03c2 --- /dev/null +++ b/samples/src/Contoso.Mcp/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "ProductsApi": { + "BaseAddress": "https://localhost:7200" + }, + "ShoppingApi": { + "BaseAddress": "https://localhost:7219" + }, + "AllowedHosts": "localhost;127.0.0.1" +} diff --git a/samples/src/Contoso.Shopping.Infrastructure/Clients/Customers/CustomersHttpClient.cs b/samples/src/Contoso.Shopping.Infrastructure/Clients/Customers/CustomersHttpClient.cs index ac1dc8dee..e2363a2cd 100644 --- a/samples/src/Contoso.Shopping.Infrastructure/Clients/Customers/CustomersHttpClient.cs +++ b/samples/src/Contoso.Shopping.Infrastructure/Clients/Customers/CustomersHttpClient.cs @@ -15,12 +15,10 @@ public class CustomersHttpClient(HttpClient httpClient) /// Gets the customer, or returns a not-found result when the customer does not exist. /// /// The customer identifier. - public async Task> GetAsync(string id, CancellationToken ct = default) - { - using var response = await _httpClient.GetAsync($"api/customers/{Uri.EscapeDataString(id)}", ct).ConfigureAwait(false); - if (response.StatusCode == System.Net.HttpStatusCode.NotFound) - return Result.NotFoundError(); - - return await response.WithValidator(_validator).ToResultAsync(ct).ConfigureAwait(false); - } + public Task> GetAsync(string id, CancellationToken ct = default) + => _httpClient + .Send(HttpMethod.Get, $"api/customers/{Uri.EscapeDataString(id)}") + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .WithValidator(_validator) + .ToResultAsync(ct); } diff --git a/samples/src/Contoso.Shopping.Infrastructure/Clients/Products/ProductsHttpClient.cs b/samples/src/Contoso.Shopping.Infrastructure/Clients/Products/ProductsHttpClient.cs index 1b406bec7..f973948c9 100644 --- a/samples/src/Contoso.Shopping.Infrastructure/Clients/Products/ProductsHttpClient.cs +++ b/samples/src/Contoso.Shopping.Infrastructure/Clients/Products/ProductsHttpClient.cs @@ -12,9 +12,6 @@ public class ProductsHttpClient(HttpClient httpClient) /// Creates a new inventory reservation. /// /// The . - public async Task CreateReservationAsync(MovementRequest request, CancellationToken ct = default) - { - var response = await _httpClient.PostAsJsonAsync("api/inventory/reserve", request, JsonDefaults.SerializerOptions, ct).ConfigureAwait(false); - return await response.ToResultAsync(ct).ConfigureAwait(false); // Handles the response and returns errors/exceptions as expected. - } + public Task CreateReservationAsync(MovementRequest request, CancellationToken ct = default) + => _httpClient.SendValue(HttpMethod.Post, "api/inventory/reserve", request).ToResultAsync(ct); } diff --git a/samples/src/Contoso.Shopping.Infrastructure/Clients/SendGrid/SendGridHttpClient.cs b/samples/src/Contoso.Shopping.Infrastructure/Clients/SendGrid/SendGridHttpClient.cs index 649fa693e..3bdf499ac 100644 --- a/samples/src/Contoso.Shopping.Infrastructure/Clients/SendGrid/SendGridHttpClient.cs +++ b/samples/src/Contoso.Shopping.Infrastructure/Clients/SendGrid/SendGridHttpClient.cs @@ -13,9 +13,6 @@ public class SendGridHttpClient(HttpClient httpClient) /// /// The . /// A successful send returns (202) with no response body. - public async Task SendMailAsync(SendGridMailRequest request, CancellationToken ct = default) - { - var response = await _httpClient.PostAsJsonAsync("v3/mail/send", request, JsonDefaults.SerializerOptions, ct).ConfigureAwait(false); - return await response.ToResultAsync(ct).ConfigureAwait(false); // Handles the response and returns errors/exceptions as expected. - } + public Task SendMailAsync(SendGridMailRequest request, CancellationToken ct = default) + => _httpClient.SendValue(HttpMethod.Post, "v3/mail/send", request).ToResultAsync(ct); } diff --git a/samples/src/Contoso.Shopping.Infrastructure/GlobalUsing.cs b/samples/src/Contoso.Shopping.Infrastructure/GlobalUsing.cs index 57b3df860..587416ae8 100644 --- a/samples/src/Contoso.Shopping.Infrastructure/GlobalUsing.cs +++ b/samples/src/Contoso.Shopping.Infrastructure/GlobalUsing.cs @@ -31,6 +31,7 @@ global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; +global using System.Net; global using System.Net.Http.Json; global using System.Text.Json; global using System.Text.Json.Serialization; diff --git a/samples/tests/Contoso.Customers.Test.Common/Data/mcp-demo.seed.yaml b/samples/tests/Contoso.Customers.Test.Common/Data/mcp-demo.seed.yaml new file mode 100644 index 000000000..bc97aa807 --- /dev/null +++ b/samples/tests/Contoso.Customers.Test.Common/Data/mcp-demo.seed.yaml @@ -0,0 +1,3 @@ +# A dedicated local Aspire demo customer for the Contoso MCP sample. +customers: +- { id: ^16, firstName: MCP, lastName: Demo, email: mcp.demo@example.com, customerTypeCode: IND, contactMethodCode: EM, hasShopped: false, shippingAddress: { street1: "1 Demo Street", city: "Anytown", state: "CA", postCode: "12345" } } diff --git a/src/CoreEx/AGENTS.md b/src/CoreEx/AGENTS.md index 65e0e9762..bf895f604 100644 --- a/src/CoreEx/AGENTS.md +++ b/src/CoreEx/AGENTS.md @@ -53,18 +53,53 @@ return await GetOrderAsync(id) .ConfigureAwait(false); ``` +## HTTP Request Builder and Extensions + +Use `client.Send(...)` or `client.SendValue(...)` to build, configure, and execute HTTP requests fluently. Terminal methods (`GetValueAsync`, `GetValueOrDefaultAsync`, `GetStringAsync`, `ToResultAsync`, `ToProblemDetailsAsync`, `ThrowOnErrorAsync`) execute the request and automatically dispose the underlying `HttpResponseMessage`. + +```csharp +// Fluent request with query filter, paging, and typed response deserialization + disposal +var products = await _httpClient + .Send(HttpMethod.Get, "api/products") + .WithQuery("contains(Text, 'hammer')") + .WithPaging(take: 10) + .GetValueAsync(cancellationToken).ConfigureAwait(false); + +// POST JSON body and map response to CoreEx Result with automatic disposal +return await _httpClient + .SendValue(HttpMethod.Post, "api/inventory/reserve", request) + .ToResultAsync(cancellationToken).ConfigureAwait(false); + +// Intercept specific HTTP status codes (e.g. 404 -> Result.NotFoundError) +return await _httpClient + .Send(HttpMethod.Get, $"api/customers/{Uri.EscapeDataString(id)}") + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .ToResultAsync(cancellationToken).ConfigureAwait(false); +``` + +Register named or typed HTTP clients in `Program.cs` via `builder.AddNamedHttpClient(...)` or `builder.AddTypedHttpClient(...)`. These automatically configure the base address, `IdempotencyKeyHandler`, service discovery, and `Microsoft.Extensions.Http.Resilience` standard resilience handlers. + ## Validating HTTP Response Values -Use `response.WithValidator(validator).ToResultAsync(ct)` to validate a successful, -non-null deserialized dependency response. The typed wrapper returns the validation result's -`Value`; reported validation errors become failed results containing `HttpRequestException` -with structured inner validation diagnostics. `GetValueAsync(ct)` is the throwing equivalent. -Standard API handling treats these as internal 500 errors, not caller 400 validation errors. +Use `WithValidator(validator)` to validate a successful, non-null deserialized dependency response. +It can be chained directly onto `HttpRequestBuilder` (which manages request execution and automatic +response disposal) or used on an existing `HttpResponseMessage`. + +When combined with `.WithStatusResult(status, result)` (or `.WithErrorStatus(...)`), matched status +codes short-circuit before deserialization and validation, returning the custom result immediately. + +The typed wrapper returns the validation result's `Value`; reported validation errors become failed +results containing `HttpRequestException` with structured inner validation diagnostics. +`GetValueAsync(ct)` is the throwing equivalent. Standard API handling treats these as internal 500 +errors, not caller 400 validation errors. Required methods reject missing content and null validated values; `OrDefault` methods allow null and skip validation for absent response values. Existing HTTP/ProblemDetails handling, deserialization exceptions, unexpected validator exceptions, and cancellation are unchanged. -Dispose the response at the call site; the wrapper does not own it. + +When using terminal methods on `HttpRequestBuilder` (e.g. `ToResultAsync`, `GetValueAsync`), response disposal +is handled automatically. When inspecting raw responses via `builder.GetResponseAsync()` or directly awaiting +`client.SendAsync`, retain `using var response = ...` to dispose the response at the call site. For small client-only rules, configure a private static readonly `Validator` once with `Validator.Create().HasProperty(...)` from the `CoreEx.Validation` package. Keep these rules @@ -75,9 +110,21 @@ not create/update requirements or identifier matching by default. Test through t private static readonly Validator _validator = Validator.Create() .HasProperty(x => x.Email, c => c.Mandatory().Email()); -// Inside the asynchronous HTTP client method. -using var response = await _httpClient.GetAsync($"api/customers/{Uri.EscapeDataString(id)}", ct).ConfigureAwait(false); -return await response.WithValidator(_validator).ToResultAsync(ct).ConfigureAwait(false); +// Fluent builder execution with custom status mapping, validation, and automatic disposal: +return await _httpClient + .Send(HttpMethod.Get, $"api/customers/{Uri.EscapeDataString(id)}") + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .WithValidator(_validator) + .ToResultAsync(ct).ConfigureAwait(false); + +// Or when inspecting raw response directly (caller owns response disposal): +using var response = await _httpClient + .Send(HttpMethod.Get, $"api/customers/{Uri.EscapeDataString(id)}") + .GetResponseAsync(ct).ConfigureAwait(false); + +return await response.WithValidator(_validator) + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .ToResultAsync(ct).ConfigureAwait(false); ``` See [usage, inline examples, and semantics](./README.md#validating-http-response-values). diff --git a/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs b/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs index 471de27bf..53ffd5ebd 100644 --- a/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs +++ b/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs @@ -31,9 +31,8 @@ public static IHostApplicationBuilder AddHostSettings(this IHostApplicationBuild } /// - /// Adds an opinionated typed with idempotency key handler, service discovery, and standard resilience handlers. + /// Adds an opinionated named with idempotency key handler, service discovery, and standard resilience handlers. /// - /// The typed client. /// The . /// The name of the . /// An optional action to configure the . @@ -67,8 +66,7 @@ public static IHostApplicationBuilder AddHostSettings(this IHostApplicationBuild /// } /// /// - public static IHttpClientBuilder AddTypedHttpClient(this IHostApplicationBuilder builder, string name, Action? configureClient = null, Action? configureIdempotency = null) - where TClient : class + public static IHttpClientBuilder AddNamedHttpClient(this IHostApplicationBuilder builder, string name, Action? configureClient = null, Action? configureIdempotency = null) { var config = builder.ThrowIfNull().Configuration.GetSection(name) ?? throw new ArgumentException($"Unable to find configuration section for '{name}'."); @@ -77,9 +75,14 @@ public static IHttpClientBuilder AddTypedHttpClient(this IHostApplicati var cb = builder.Services.AddHttpClient(name, client => { - // Set the standard configured setting. - client.BaseAddress = config.GetValue("BaseAddress") ?? throw new ArgumentException($"Unable to find '{nameof(HttpClient.BaseAddress)}' configuration for '{name}'."); + var baseAddress = config.GetValue("BaseAddress"); + if (baseAddress is not null) + client.BaseAddress = baseAddress; + configureClient?.Invoke(client); + + if (client.BaseAddress is null) + throw new ArgumentException($"Unable to find '{nameof(HttpClient.BaseAddress)}' configuration for '{name}'."); }); cb.AddIdempotencyKeyHandler((sp, handler) => configureIdempotency?.Invoke(sp, handler)); @@ -90,11 +93,61 @@ public static IHttpClientBuilder AddTypedHttpClient(this IHostApplicati else cb.AddStandardResilienceHandler(); - cb.AddTypedClient(); - return cb; } + /// + /// Adds an opinionated named with idempotency key handler, service discovery, and standard resilience handlers. + /// + /// The . + /// The name of the . + /// An optional action to configure the . + /// An optional action to configure the . + /// The for fluent-style method-chaining. + public static IHttpClientBuilder AddHttpClient(this IHostApplicationBuilder builder, string name, Action? configureClient = null, Action? configureIdempotency = null) + => builder.AddNamedHttpClient(name, configureClient, configureIdempotency); + + /// + /// Adds an opinionated typed with idempotency key handler, service discovery, and standard resilience handlers. + /// + /// The typed client. + /// The . + /// The name of the . + /// An optional action to configure the . + /// An optional action to configure the . + /// The for fluent-style method-chaining. + /// The also represents the configuration section name for the ; as a minimum define the . + /// The handlers are added, in order specified, as follows: + /// + /// - Adds an idempotency key to outgoing HTTP requests. + /// Service discovery - Resolves the host as a logical service name (e.g. https://products-api) against a Services:{name}:{scheme} configuration section (or, on supporting platforms, a native resolver) before falling back unchanged to a literal address if no matching entry is found - see Microsoft.Extensions.ServiceDiscovery. This makes an Aspire-orchestrated BaseAddress (populated via the AppHost's WithReference) and a literal production URL both work unchanged through the same code path - no Aspire AppHost is required at runtime for the latter. + /// Standard resilience handlers - Adds standard resilience policies as enabled by Microsoft.Extensions.Http.Resilience. + /// + /// + /// Example configuration section for a named 'ProductsApi' pointing to a literal production address: + /// + /// { + /// "ProductsApi": { + /// "BaseAddress": "https://api.contoso.com/", + /// "Resilience": { + /// ... // Resilience configuration as per Microsoft.Extensions.Http.Resilience documentation. + /// } + /// } + /// } + /// + /// Or, when consumed from within an Aspire AppHost (local development/testing), pointing to the logical Aspire resource name (e.g. products-api) instead of a hardcoded host/port - the AppHost's WithReference(productsApi) then supplies the actual resolved endpoint(s) via the Services:products-api:* configuration section regardless of which port the referenced project ultimately binds to: + /// + /// { + /// "ProductsApi": { + /// "BaseAddress": "https+http://products-api" + /// } + /// } + /// + /// + public static IHttpClientBuilder AddTypedHttpClient(this IHostApplicationBuilder builder, string name, Action? configureClient = null, Action? configureIdempotency = null) + where TClient : class + => builder.AddNamedHttpClient(name, configureClient, configureIdempotency).AddTypedClient(); + /// /// Adds an opinionated typed with idempotency key handler and standard resilience handlers. /// @@ -135,29 +188,5 @@ public static IHttpClientBuilder AddTypedHttpClient(this IHostApplicati /// public static IHttpClientBuilder AddTypedHttpClient(this IHostApplicationBuilder builder, string name, Action? configureClient = null, Action? configureIdempotency = null) where TClient : class where TImplementation : class, TClient - { - var config = builder.ThrowIfNull().Configuration.GetSection(name) ?? throw new ArgumentException($"Unable to find configuration section for '{name}'."); - - // Registers the underlying service endpoint resolver providers (configuration-based + pass-through); safe/idempotent to call more than once per host. - builder.Services.AddServiceDiscovery(); - - var cb = builder.Services.AddHttpClient(name, client => - { - // Set the standard configured setting. - client.BaseAddress = config.GetValue("BaseAddress") ?? throw new ArgumentException($"Unable to find '{nameof(HttpClient.BaseAddress)}' configuration for '{name}'."); - configureClient?.Invoke(client); - }); - - cb.AddIdempotencyKeyHandler((sp, handler) => configureIdempotency?.Invoke(sp, handler)); - cb.AddServiceDiscovery(); - - if (config.GetSection("Resilience").Exists()) - cb.AddStandardResilienceHandler(config); - else - cb.AddStandardResilienceHandler(); - - cb.AddTypedClient(); - - return cb; - } -} \ No newline at end of file + => builder.AddNamedHttpClient(name, configureClient, configureIdempotency).AddTypedClient(); +} diff --git a/src/CoreEx/Extensions.HttpClient.cs b/src/CoreEx/Extensions.HttpClient.cs new file mode 100644 index 000000000..3668f706a --- /dev/null +++ b/src/CoreEx/Extensions.HttpClient.cs @@ -0,0 +1,165 @@ +namespace CoreEx; + +public static partial class Extensions +{ + /// + /// Creates an for the specified and . + /// + /// The . + /// The . + /// The request URI. + /// An . + public static HttpRequestBuilder Send(this HttpClient client, HttpMethod method, string? requestUri) + => Send(client, method, requestUri, null); + + /// + /// Creates an for the specified and . + /// + /// The . + /// The . + /// The request URI. + /// An optional action to configure the . + /// An . + public static HttpRequestBuilder Send(this HttpClient client, HttpMethod method, string? requestUri, Action? configureRequest) + => new(client, method, requestUri, configureRequest); + + /// + /// Creates an for the specified and , serializing the specified to JSON. + /// + /// The value . + /// The . + /// The . + /// The request URI. + /// The value to serialize to JSON. + /// An . + public static HttpRequestBuilder SendValue(this HttpClient client, HttpMethod method, string? requestUri, T value) + => SendValue(client, method, requestUri, value, null, null, null); + + /// + /// Creates an for the specified and , serializing the specified to JSON. + /// + /// The value . + /// The . + /// The . + /// The request URI. + /// The value to serialize to JSON. + /// An optional action to configure the . + /// An . + public static HttpRequestBuilder SendValue(this HttpClient client, HttpMethod method, string? requestUri, T value, Action? configureRequest) + => SendValue(client, method, requestUri, value, null, configureRequest, null); + + /// + /// Creates an for the specified and , serializing the specified to JSON. + /// + /// The value . + /// The . + /// The . + /// The request URI. + /// The value to serialize to JSON. + /// The media type. + /// An optional action to configure the . + /// An . + public static HttpRequestBuilder SendValue(this HttpClient client, HttpMethod method, string? requestUri, T value, string? mediaType, Action? configureRequest) + => SendValue(client, method, requestUri, value, mediaType, configureRequest, null); + + /// + /// Creates an for the specified and , serializing the specified to JSON. + /// + /// The value . + /// The . + /// The . + /// The request URI. + /// The value to serialize to JSON. + /// The media type. + /// An optional action to configure the . + /// An optional to use when serializing the value to JSON; defaults to where not specified. + /// An . + public static HttpRequestBuilder SendValue(this HttpClient client, HttpMethod method, string? requestUri, T value, string? mediaType = null, Action? configureRequest = null, JsonSerializerOptions? jsonSerializerOptions = null) + => Send(client, method, requestUri, request => + { + mediaType ??= MediaTypeNames.Application.Json; + request.Content = JsonContent.Create(value, new MediaTypeHeaderValue(mediaType), options: jsonSerializerOptions ?? JsonDefaults.SerializerOptions); + configureRequest?.Invoke(request); + }); + + /// + /// Sends an HTTP request with the specified and and returns the response. + /// + /// The . + /// The . + /// The request URI. + /// A cancellation token. + /// The . + public static Task SendAsync(this HttpClient client, HttpMethod method, string? requestUri, CancellationToken cancellationToken = default) + => SendAsync(client, method, requestUri, null, cancellationToken); + + /// + /// Sends an HTTP request with the specified and and returns the response. + /// + /// The . + /// The . + /// The request URI. + /// An optional action to configure the . + /// A cancellation token. + /// The . + public static Task SendAsync(this HttpClient client, HttpMethod method, string? requestUri, Action? configureRequest, CancellationToken cancellationToken = default) + => Send(client, method, requestUri, configureRequest).GetResponseAsync(cancellationToken); + + /// + /// Sends an HTTP request with the specified and and returns the response, serializing the specified to JSON. + /// + /// The value . + /// The . + /// The . + /// The request URI. + /// The value to serialize to JSON. + /// A cancellation token. + /// The . + public static Task SendValueAsync(this HttpClient client, HttpMethod method, string? requestUri, T value, CancellationToken cancellationToken = default) + => SendValueAsync(client, method, requestUri, value, null, null, null, cancellationToken); + + /// + /// Sends an HTTP request with the specified and and returns the response, serializing the specified to JSON. + /// + /// The value . + /// The . + /// The . + /// The request URI. + /// The value to serialize to JSON. + /// An optional action to configure the . + /// A cancellation token. + /// The . + public static Task SendValueAsync(this HttpClient client, HttpMethod method, string? requestUri, T value, Action? configureRequest, CancellationToken cancellationToken = default) + => SendValueAsync(client, method, requestUri, value, null, configureRequest, null, cancellationToken); + + /// + /// Sends an HTTP request with the specified and and returns the response, serializing the specified to JSON. + /// + /// The value . + /// The . + /// The . + /// The request URI. + /// The value to serialize to JSON. + /// The media type. + /// An optional action to configure the . + /// A cancellation token. + /// The . + public static Task SendValueAsync(this HttpClient client, HttpMethod method, string? requestUri, T value, string? mediaType, Action? configureRequest, CancellationToken cancellationToken = default) + => SendValueAsync(client, method, requestUri, value, mediaType, configureRequest, null, cancellationToken); + + /// + /// Sends an HTTP request with the specified and and returns the response, serializing the specified to JSON. + /// + /// The value . + /// The . + /// The . + /// The request URI. + /// The value to serialize to JSON. + /// The media type. + /// An optional action to configure the . + /// An optional to use when serializing the value to JSON; defaults to where not specified. + /// A cancellation token. + /// The . + public static Task SendValueAsync(this HttpClient client, HttpMethod method, string? requestUri, T value, string? mediaType, Action? configureRequest = null, JsonSerializerOptions? jsonSerializerOptions = null, CancellationToken cancellationToken = default) + => SendValue(client, method, requestUri, value, mediaType, configureRequest, jsonSerializerOptions).GetResponseAsync(cancellationToken); +} diff --git a/src/CoreEx/Extensions.HttpRequestMessage.cs b/src/CoreEx/Extensions.HttpRequestMessage.cs index d2aa1b16c..4bf01826c 100644 --- a/src/CoreEx/Extensions.HttpRequestMessage.cs +++ b/src/CoreEx/Extensions.HttpRequestMessage.cs @@ -1,3 +1,9 @@ +/* + * NOTE: When adding, updating, or removing request configuration extensions in this file, + * ensure the matching fluent methods on HttpRequestBuilder (CoreEx.Http.HttpRequestBuilder) + * are also added, updated, or removed to keep both HTTP request configuration APIs in sync. + */ + namespace CoreEx; public static partial class Extensions diff --git a/src/CoreEx/Extensions.HttpResponseMessage.cs b/src/CoreEx/Extensions.HttpResponseMessage.cs index 6241e52d1..55d6ecbfe 100644 --- a/src/CoreEx/Extensions.HttpResponseMessage.cs +++ b/src/CoreEx/Extensions.HttpResponseMessage.cs @@ -155,7 +155,7 @@ public static async Task> ToResultAsync(this HttpResponseMessage re return await ToResultAsync(response, cancellationToken).ConfigureAwait(false); // Where successful, attempt to read the content as JSON and return as the value; empty content (e.g. 204) is treated as no value as it is not valid JSON. - if (response.StatusCode == HttpStatusCode.NoContent || response.Content.Headers.ContentLength == 0) + if (response.StatusCode == HttpStatusCode.NoContent || response.Content is null || response.Content.Headers.ContentLength == 0) return default; var value = await response.Content.ReadFromJsonAsync(jsonSerializerOptions ?? JsonDefaults.SerializerOptions, cancellationToken).ConfigureAwait(false); @@ -172,6 +172,26 @@ public static async Task> ToResultAsync(this HttpResponseMessage re /// See . public static Task> ToResultOrDefaultAsync(this HttpResponseMessage response, CancellationToken cancellationToken = default) => ToResultOrDefaultAsync(response, null, cancellationToken); + /// + /// Converts the into a including the response content as a string where successful; a successful response with empty or content results in a value. + /// + /// The . + /// The . + /// The corresponding . + public static async Task> ToStringResultAsync(this HttpResponseMessage response, CancellationToken cancellationToken = default) + { + // Where not successful, reuse the non-generic version above to get the error details. + if (!response.ThrowIfNull().IsSuccessStatusCode) + return await ToResultAsync(response, cancellationToken).ConfigureAwait(false); + + // Where successful, attempt to read the content as a string; empty content (e.g. 204) is treated as no value. + if (response.StatusCode == HttpStatusCode.NoContent || response.Content is null || response.Content.Headers.ContentLength == 0) + return default; + + var value = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return string.IsNullOrEmpty(value) ? null : value; + } + /// /// Gets the deserialized JSON response value from the where successful; otherwise, will throw a corresponding exception. /// @@ -227,4 +247,16 @@ public async static Task GetValueAsync(this HttpResponseMessage response, /// The value where successful. /// See . public static Task GetValueOrDefaultAsync(this HttpResponseMessage response, CancellationToken cancellationToken = default) => GetValueOrDefaultAsync(response, null, cancellationToken); + + /// + /// Gets the response content as a string where successful, or where the content is empty or ; otherwise, will throw a corresponding exception. + /// + /// The . + /// The . + /// The response content as a string where successful, or where the content is empty or . + public static async Task GetStringAsync(this HttpResponseMessage response, CancellationToken cancellationToken = default) + { + var result = await response.ThrowIfNull().ToStringResultAsync(cancellationToken).ConfigureAwait(false); + return result.Value; + } } diff --git a/src/CoreEx/Http/HttpRequestBuilder.cs b/src/CoreEx/Http/HttpRequestBuilder.cs new file mode 100644 index 000000000..ea147615a --- /dev/null +++ b/src/CoreEx/Http/HttpRequestBuilder.cs @@ -0,0 +1,455 @@ +/* + * NOTE: When adding, updating, or removing request configuration methods in this file, + * ensure the matching extensions in Extensions.HttpRequestMessage (CoreEx.Extensions) + * are kept in sync, and vice versa, to preserve consistency across both HTTP request APIs. + */ + +namespace CoreEx.Http; + +/// +/// Builds and executes an HTTP request, providing fluent terminal methods that handle response consumption and disposal. +/// +/// The . +/// The . +/// The request URI. +/// An optional action to configure the . +public sealed class HttpRequestBuilder(HttpClient client, HttpMethod method, string? requestUri, Action? configureRequest = null) +{ + private readonly HttpClient _client = client.ThrowIfNull(); + private readonly HttpMethod _method = method.ThrowIfNull(); + private readonly string? _requestUri = requestUri; + private Action? _configureRequest = configureRequest; + private Dictionary>>? _statusResults; + + /// + /// Gets the . + /// + public HttpClient Client => _client; + + /// + /// Gets the . + /// + public HttpMethod Method => _method; + + /// + /// Gets the request URI. + /// + public string? RequestUri => _requestUri; + + /// + /// Adds an action to configure the before it is sent. + /// + /// The action to configure the request. + /// The instance to support fluent chaining. + public HttpRequestBuilder Configure(Action configureRequest) + { + configureRequest.ThrowIfNull(); + var existing = _configureRequest; + _configureRequest = existing is null ? configureRequest : (r => { existing(r); configureRequest(r); }); + return this; + } + + /// + /// Adds the header with the specified (where not ). + /// + /// The entity tag value. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithIfMatch(string? etag) => Configure(r => r.WithIfMatch(etag)); + + /// + /// Adds the header with the specified (where not ). + /// + /// The entity tag value. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithIfNoneMatch(string? etag) => Configure(r => r.WithIfNoneMatch(etag)); + + /// + /// Sets the request content type to . + /// + /// The instance to support fluent chaining. + public HttpRequestBuilder WithMergePatchJsonContentType() => Configure(r => r.WithMergePatchJsonContentType()); + + /// + /// Adds the header with the specified or generates a new one. + /// + /// The optional idempotency key; where not specified, a new is generated and used. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithIdempotencyKey(string? idempotencyKey = null) => Configure(r => r.WithIdempotencyKey(idempotencyKey)); + + /// + /// Adds to the request URI query string. + /// + /// The . + /// The instance to support fluent chaining. + public HttpRequestBuilder WithPaging(PagingArgs? paging) => Configure(r => r.WithPaging(paging)); + + /// + /// Adds paging query parameters to the request URI query string. + /// + /// The specified number of elements in a sequence to bypass. + /// The specified number of contiguous elements from the start of a sequence. + /// Indicates whether to get the total count when performing the underlying query. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithPaging(long? skip = null, long? take = null, bool count = false) => Configure(r => r.WithPaging(skip, take, count)); + + /// + /// Adds to the request URI query string. + /// + /// The . + /// The instance to support fluent chaining. + public HttpRequestBuilder WithQuery(QueryArgs? query) => Configure(r => r.WithQuery(query)); + + /// + /// Adds query parameters to the request URI query string. + /// + /// The basic dynamic OData-like $filter statement. + /// The basic dynamic OData-like $orderby statement. + /// The list of included fields. + /// The list of excluded fields. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithQuery(string? filter = null, string? orderBy = null, IEnumerable? include = null, IEnumerable? exclude = null) => Configure(r => r.WithQuery(filter, orderBy, include, exclude)); + + /// + /// Adds to the request URI query string. + /// + /// The instance to support fluent chaining. + public HttpRequestBuilder WithIncludeText() => Configure(r => r.WithIncludeText()); + + /// + /// Adds to the request URI query string. + /// + /// The instance to support fluent chaining. + public HttpRequestBuilder WithIncludeInactive() => Configure(r => r.WithIncludeInactive()); + + /// + /// Adds to the request URI query string. + /// + /// The list of included fields. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithFields(params IEnumerable fields) => Configure(r => r.WithFields(fields)); + + /// + /// Adds to the request URI query string. + /// + /// The list of excluded fields. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithoutFields(params IEnumerable fields) => Configure(r => r.WithoutFields(fields)); + + /// + /// Configures a to return when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The to return. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Result result) + { + _statusResults ??= []; + _statusResults[statusCode] = (_, _) => Task.FromResult(result); + return this; + } + + /// + /// Configures a factory to invoke when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The function returning the . + /// The instance to support fluent chaining. + public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Func result) + { + result.ThrowIfNull(); + _statusResults ??= []; + _statusResults[statusCode] = (_, _) => Task.FromResult(result()); + return this; + } + + /// + /// Configures a factory to invoke with the when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The function returning the for the response. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Func result) + { + result.ThrowIfNull(); + _statusResults ??= []; + _statusResults[statusCode] = (response, _) => Task.FromResult(result(response)); + return this; + } + + /// + /// Configures an asynchronous factory to invoke with the when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The asynchronous function returning the for the response. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Func> result) + { + result.ThrowIfNull(); + _statusResults ??= []; + _statusResults[statusCode] = result; + return this; + } + + /// + /// Configures a to return when the response status matches the specified . Alias for . + /// + public HttpRequestBuilder WithErrorStatus(HttpStatusCode statusCode, Result result) => WithStatusResult(statusCode, result); + + /// + /// Configures a factory to invoke when the response status matches the specified . Alias for . + /// + public HttpRequestBuilder WithErrorStatus(HttpStatusCode statusCode, Func result) => WithStatusResult(statusCode, result); + + /// + /// Configures a factory to invoke with the when the response status matches the specified . Alias for . + /// + public HttpRequestBuilder WithErrorStatus(HttpStatusCode statusCode, Func result) => WithStatusResult(statusCode, result); + + /// + /// Configures an asynchronous factory to invoke with the when the response status matches the specified . Alias for . + /// + public HttpRequestBuilder WithErrorStatus(HttpStatusCode statusCode, Func> result) => WithStatusResult(statusCode, result); + + /// + /// Attempts to get a registered custom for the response's status code. + /// + internal async Task TryGetStatusResultAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + if (_statusResults is not null && _statusResults.TryGetValue(response.StatusCode, out var handler)) + return await handler(response, cancellationToken).ConfigureAwait(false); + + return null; + } + + /// + /// Sends the HTTP request and returns the . The caller is responsible for disposing the returned response. + /// + /// The cancellation token. + /// The . + public async Task GetResponseAsync(CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(_method, _requestUri); + if (_client.BaseAddress is not null && request.RequestUri?.IsAbsoluteUri != true) + request.RequestUri = new Uri(_client.BaseAddress, _requestUri ?? string.Empty); + + _configureRequest?.Invoke(request); + return await _client.SendAsync(request, cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and returns the . The caller is responsible for disposing the returned response. + /// + /// The cancellation token. + /// The . + public Task SendAsync(CancellationToken cancellationToken = default) => GetResponseAsync(cancellationToken); + + /// + /// Sends the HTTP request and throws an exception where the response is not successful; disposes the response upon completion. + /// + /// The cancellation token. + /// The completed . + public async Task ThrowOnErrorAsync(CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + { + customResult.ThrowOnError(); + return; + } + + await response.ThrowOnErrorAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and converts the response into a where not successful and problem+json; disposes the response upon completion. + /// + /// The cancellation token. + /// The corresponding or . + public async Task ToProblemDetailsAsync(CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + { + if (customResult.IsFailure && customResult.Error is ProblemDetailsException pde) + return pde; + + return null; + } + + return await response.ToProblemDetailsAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and throws where considered a business exception; disposes the response upon completion. + /// + /// The cancellation token. + /// The corresponding or . + public async Task ThrowOnBusinessExceptionAsync(CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + { + if (customResult.IsFailure && customResult.Error is BusinessException) + customResult.ThrowOnError(); + + return null; + } + + return await response.ThrowOnBusinessExceptionAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and converts the response into a ; disposes the response upon completion. + /// + /// The cancellation token. + /// The corresponding . + public async Task ToResultAsync(CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + return customResult; + + return await response.ToResultAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and converts the response into a including the deserialized JSON response value; disposes the response upon completion. + /// + /// The response value . + /// The optional . + /// The cancellation token. + /// The corresponding . + public async Task> ToResultAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + return customResult.Bind(); + + return await response.ToResultAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and converts the response into a including the deserialized JSON response value; disposes the response upon completion. + /// + /// The response value . + /// The cancellation token. + /// The corresponding . + public Task> ToResultAsync(CancellationToken cancellationToken = default) => ToResultAsync(null, cancellationToken); + + /// + /// Sends the HTTP request and converts the response into a including the deserialized JSON response value or ; disposes the response upon completion. + /// + /// The response value . + /// The optional . + /// The cancellation token. + /// The corresponding . + public async Task> ToResultOrDefaultAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + return customResult.Bind(); + + return await response.ToResultOrDefaultAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and converts the response into a including the deserialized JSON response value or ; disposes the response upon completion. + /// + /// The response value . + /// The cancellation token. + /// The corresponding . + public Task> ToResultOrDefaultAsync(CancellationToken cancellationToken = default) => ToResultOrDefaultAsync(null, cancellationToken); + + /// + /// Sends the HTTP request and converts the response into a including the response content as a string; disposes the response upon completion. + /// + /// The cancellation token. + /// The corresponding . + public async Task> ToStringResultAsync(CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + return customResult.Bind(); + + return await response.ToStringResultAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and gets the deserialized JSON response value where successful; disposes the response upon completion. + /// + /// The response value . + /// The optional . + /// The cancellation token. + /// The deserialized response value. + public async Task GetValueAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + { + customResult.ThrowOnError(); + return customResult.Bind().Value; + } + + return await response.GetValueAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and gets the deserialized JSON response value where successful; disposes the response upon completion. + /// + /// The response value . + /// The cancellation token. + /// The deserialized response value. + public Task GetValueAsync(CancellationToken cancellationToken = default) => GetValueAsync(null, cancellationToken); + + /// + /// Sends the HTTP request and gets the deserialized JSON response value or where successful; disposes the response upon completion. + /// + /// The response value . + /// The optional . + /// The cancellation token. + /// The deserialized response value or . + public async Task GetValueOrDefaultAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + { + customResult.ThrowOnError(); + return customResult.Bind().Value; + } + + return await response.GetValueOrDefaultAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and gets the deserialized JSON response value or where successful; disposes the response upon completion. + /// + /// The response value . + /// The cancellation token. + /// The deserialized response value or . + public Task GetValueOrDefaultAsync(CancellationToken cancellationToken = default) => GetValueOrDefaultAsync(null, cancellationToken); + + /// + /// Sends the HTTP request and gets the response content as a string where successful; disposes the response upon completion. + /// + /// The cancellation token. + /// The response content as a string or . + public async Task GetStringAsync(CancellationToken cancellationToken = default) + { + using var response = await GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + { + customResult.ThrowOnError(); + return customResult.Bind().Value; + } + + return await response.GetStringAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Configures validation of a successful, non-null deserialized HTTP response value. + /// + /// The response value type. + /// The response value validator. + /// A typed wrapper that validates when a value is consumed. + public ValidatedHttpRequestBuilder WithValidator(Validation.IValidator validator) => new(this, validator); +} diff --git a/src/CoreEx/Http/README.md b/src/CoreEx/Http/README.md index a4b568658..6af16df89 100644 --- a/src/CoreEx/Http/README.md +++ b/src/CoreEx/Http/README.md @@ -25,19 +25,114 @@ The `Abstractions` sub-namespace defines additional lightweight types and interf | **[`ProblemDetails`](./Abstractions/ProblemDetails.cs)** | RFC 7807 problem-details record with `Type`, `Title`, `Status`, `Detail`, `Instance`, and an `Extensions` dictionary; includes `GetValidationErrors(string? key)`. | | **[`ProblemDetailsException`](./ProblemDetailsException.cs)** | Exception wrapping a `ProblemDetails`; provides `ToException()` to convert it back to a typed CoreEx semantic exception. | | **[`HttpNames`](./HttpNames.cs)** | Static class of configurable string properties for all standard paging query-string names, response-header names, and the idempotency-key header name. | +| **[`HttpRequestBuilder`](./HttpRequestBuilder.cs)** | Fluent HTTP request builder returned by `client.Send(...)` and `client.SendValue(...)`; executes requests and manages automatic response disposal across typed deserialization, ROP result conversion, validation, and error-handling methods. | +| **[`ValidatedHttpRequestBuilder`](./ValidatedHttpRequestBuilderT.cs)** | Wrapper returned by `builder.WithValidator(validator)` that executes the request, validates the deserialized response value, and automatically disposes the response. | | **[`ValidatedHttpResponse`](./ValidatedHttpResponseT.cs)** | Thin wrapper returned by `HttpResponseMessage.WithValidator(validator)`; validates successful non-null response values when consumed via `ToResult*` or `GetValue*`. Does not own the response. | | **[`HttpResult`](./HttpResult.cs)** | Readonly struct pairing an `HttpResponseMessage` with either a deserialized value or a `ProblemDetails` from the response; exposes `IsSuccess`, `StatusCode`, and `ThrowOnError()`. | | **[`HttpResult`](./HttpResultT.cs)** | Typed variant of `HttpResult` adding a strongly-typed `Value` property for the deserialized response body. | | [`IExtendedHttpClient`](./Abstractions/IExtendedHttpClient.cs) | Interface implemented by `TypedHttpClientBase` for type-safe HTTP client wrappers; exposes `SendAsync`, `GetAsync`, `PostAsync`, `PutAsync`, `PatchAsync`, `DeleteAsync` returning `HttpResult`. | +## Sending requests + +The `CoreEx` namespace provides fluent synchronous builder methods `HttpClient.Send(...)` and `SendValue(...)` +returning an `HttpRequestBuilder`, as well as direct asynchronous `SendAsync` and `SendValueAsync` overloads. +Relative URIs are resolved against `HttpClient.BaseAddress` before the configuration callback, so +`WithQuery`, `WithPaging`, and header helpers can be applied there. Absolute URIs do not require a base address. + +### Fluent request execution and automatic response disposal + +`client.Send(...)` and `client.SendValue(...)` return an `HttpRequestBuilder`. Calling any terminal consumption +method executes the request and automatically disposes the underlying `HttpResponseMessage`: + +```csharp +// Single await with automatic response disposal: +var product = await client.Send(HttpMethod.Get, "api/products/123") + .GetValueAsync(cancellationToken).ConfigureAwait(false); + +// Fluent query, paging, and headers directly chained: +var products = await client.Send(HttpMethod.Get, "api/products") + .WithQuery("startswith(sku, 'ABC')") + .WithPaging(take: 10) + .GetValueAsync(cancellationToken).ConfigureAwait(false); + +// Post JSON payload with typed response: +var created = await client.SendValue(HttpMethod.Post, "api/baskets", newBasket) + .GetValueAsync(cancellationToken).ConfigureAwait(false); + +// With custom status mapping (short-circuits validation/deserialization on match): +var customer = await client.Send(HttpMethod.Get, "api/customers/123") + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .WithValidator(customerValidator) + .ToResultAsync(cancellationToken).ConfigureAwait(false); + +// With response value validation: +var validated = await client.Send(HttpMethod.Get, "api/products/123") + .WithValidator(productValidator) + .GetValueAsync(cancellationToken).ConfigureAwait(false); + +// Where raw HttpResponseMessage is needed directly (caller owns disposal): +using var response = await client.Send(HttpMethod.Get, "api/products") + .GetResponseAsync(cancellationToken).ConfigureAwait(false); +``` + +### Direct asynchronous methods + +For direct response inspection, `SendAsync` and `SendValueAsync` return the raw `HttpResponseMessage` (which the caller must dispose): + +```csharp +using var response = await client.SendAsync(HttpMethod.Get, "api/products", + r => r.WithQuery("startswith(sku, 'ABC')").WithPaging(take: 10), cancellationToken).ConfigureAwait(false); +await response.ThrowOnErrorAsync(cancellationToken).ConfigureAwait(false); +``` + +## Reading response content + +`HttpResponseMessage` provides extensions for reading typed JSON or raw string content: + +- **Typed JSON**: `GetValueAsync` and `GetValueOrDefaultAsync` deserialize the body using `JsonDefaults.SerializerOptions` and throw on HTTP error (with `GetValueAsync` also rejecting empty or null bodies). Their ROP counterparts are `ToResultAsync` and `ToResultOrDefaultAsync`. +- **Raw string**: `GetStringAsync` reads the response as a string, returning `null` for empty content or 204 No Content, and throws on HTTP error. Its ROP counterpart is `ToStringResultAsync`, returning `Result`. + +```csharp +using var response = await client.SendAsync(HttpMethod.Get, "api/products/123", cancellationToken).ConfigureAwait(false); +var content = await response.GetStringAsync(cancellationToken).ConfigureAwait(false); +``` + ## Validating response values -Use `response.WithValidator(validator).ToResultAsync(ct)` for opt-in response-contract validation. -The wrapper returns the validation result's `Value`; reported failures contain an -`HttpRequestException` with inner validation diagnostics. `GetValueAsync(ct)` throws on failure. -Required methods reject null, while `OrDefault` methods allow null and skip validation for absent -values. HTTP status handling remains unchanged, and the caller retains response ownership. -See [the full usage guide and inline client example](../README.md#validating-http-response-values). +Use `WithValidator(validator)` with an `IValidator` for opt-in response-contract validation across two patterns: + +### 1. Fluent request builder (preferred) + +Chaining `.WithValidator(...)` onto `HttpRequestBuilder` returns a `ValidatedHttpRequestBuilder`. Calling any terminal method executes the request, validates the deserialized response, and **automatically disposes** the underlying `HttpResponseMessage`: + +```csharp +// Fluent request with validation and automatic response disposal: +var product = await client.Send(HttpMethod.Get, "api/products/123") + .WithValidator(productValidator) + .GetValueAsync(cancellationToken).ConfigureAwait(false); + +// ROP result pipeline variant: +var result = await client.Send(HttpMethod.Get, "api/products/123") + .WithValidator(productValidator) + .ToResultAsync(cancellationToken).ConfigureAwait(false); +``` + +### 2. Existing response message + +Calling `response.WithValidator(...)` on an existing `HttpResponseMessage` returns a `ValidatedHttpResponse`. The wrapper does not own or dispose the response — the caller **retains ownership** and must dispose it: + +```csharp +using var response = await client.SendAsync(HttpMethod.Get, "api/products/123", cancellationToken).ConfigureAwait(false); +var product = await response.WithValidator(productValidator).GetValueAsync(cancellationToken).ConfigureAwait(false); +``` + +### Validation semantics + +- **Returned value**: Both wrappers return the validation result's `Value`, preserving any value transformations or normalizations performed during validation. +- **Status interception**: Calling `.WithStatusResult(statusCode, result)` (or `.WithErrorStatus(...)`) on either builder or validated response short-circuits execution before deserialization and validation when the HTTP status matches, returning the mapped result directly. +- **Required vs optional**: `GetValueAsync` / `ToResultAsync` reject null or empty bodies both before and after validation. `GetValueOrDefaultAsync` / `ToResultOrDefaultAsync` allow null responses and skip validation when no body is present. +- **Error handling**: Reported validation errors are packaged into an `HttpRequestException` with the inner `ValidationException` preserving diagnostics. Under standard CoreEx exception handling, upstream validation failures map to internal **500** errors rather than caller **400** errors. +- **Full guide**: See [the full usage guide and inline client example](../README.md#validating-http-response-values). ## Related Namespaces diff --git a/src/CoreEx/Http/ValidatedHttpRequestBuilderT.cs b/src/CoreEx/Http/ValidatedHttpRequestBuilderT.cs new file mode 100644 index 000000000..82cf957db --- /dev/null +++ b/src/CoreEx/Http/ValidatedHttpRequestBuilderT.cs @@ -0,0 +1,185 @@ +namespace CoreEx.Http; + +/// +/// Enables opt-in validation when sending an HTTP request and consuming a deserialized response value. +/// +/// The response value type. +/// The . +/// The response value validator. +public sealed class ValidatedHttpRequestBuilder(HttpRequestBuilder builder, Validation.IValidator validator) +{ + private readonly HttpRequestBuilder _builder = builder.ThrowIfNull(); + private readonly Validation.IValidator _validator = validator.ThrowIfNull(); + + /// + /// Gets the . + /// + public HttpRequestBuilder Builder => _builder; + + /// + /// Gets the . + /// + public Validation.IValidator Validator => _validator; + + /// + /// Configures a to return when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The to return. + /// The instance to support fluent chaining. + public ValidatedHttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Result result) + { + _builder.WithStatusResult(statusCode, result); + return this; + } + + /// + /// Configures a factory to invoke when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The function returning the . + /// The instance to support fluent chaining. + public ValidatedHttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Func result) + { + _builder.WithStatusResult(statusCode, result); + return this; + } + + /// + /// Configures a factory to invoke with the when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The function returning the for the response. + /// The instance to support fluent chaining. + public ValidatedHttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Func result) + { + _builder.WithStatusResult(statusCode, result); + return this; + } + + /// + /// Configures an asynchronous factory to invoke with the when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The asynchronous function returning the for the response. + /// The instance to support fluent chaining. + public ValidatedHttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Func> result) + { + _builder.WithStatusResult(statusCode, result); + return this; + } + + /// + /// Configures a to return when the response status matches the specified . Alias for . + /// + public ValidatedHttpRequestBuilder WithErrorStatus(HttpStatusCode statusCode, Result result) => WithStatusResult(statusCode, result); + + /// + /// Configures a factory to invoke when the response status matches the specified . Alias for . + /// + public ValidatedHttpRequestBuilder WithErrorStatus(HttpStatusCode statusCode, Func result) => WithStatusResult(statusCode, result); + + /// + /// Configures a factory to invoke with the when the response status matches the specified . Alias for . + /// + public ValidatedHttpRequestBuilder WithErrorStatus(HttpStatusCode statusCode, Func result) => WithStatusResult(statusCode, result); + + /// + /// Configures an asynchronous factory to invoke with the when the response status matches the specified . Alias for . + /// + public ValidatedHttpRequestBuilder WithErrorStatus(HttpStatusCode statusCode, Func> result) => WithStatusResult(statusCode, result); + + /// + /// Sends the HTTP request and gets a result containing the validated value; missing content or a null validated value is a failure. Disposes the response upon completion. + /// + /// The cancellation token. + /// The validated result. + public Task> ToResultAsync(CancellationToken cancellationToken = default) => ToResultAsync(null, cancellationToken); + + /// + /// Sends the HTTP request and gets a result containing the validated value; missing content or a null validated value is a failure. Disposes the response upon completion. + /// + /// The optional JSON serializer options. + /// The cancellation token. + /// The validated result. + public async Task> ToResultAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) + { + using var response = await _builder.GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await _builder.TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + return customResult.Bind(); + + return await response.WithValidator(_validator).ToResultAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and gets a result containing the validated value, allowing null; absent response values are not validated. Disposes the response upon completion. + /// + /// The cancellation token. + /// The validated result. + public Task> ToResultOrDefaultAsync(CancellationToken cancellationToken = default) => ToResultOrDefaultAsync(null, cancellationToken); + + /// + /// Sends the HTTP request and gets a result containing the validated value, allowing null; absent response values are not validated. Disposes the response upon completion. + /// + /// The optional JSON serializer options. + /// The cancellation token. + /// The validated result. + public async Task> ToResultOrDefaultAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) + { + using var response = await _builder.GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await _builder.TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + return customResult.Bind(); + + return await response.WithValidator(_validator).ToResultOrDefaultAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and gets the validated value, throwing on failure, including missing content or a null validated value. Disposes the response upon completion. + /// + /// The cancellation token. + /// The validated value. + public Task GetValueAsync(CancellationToken cancellationToken = default) => GetValueAsync(null, cancellationToken); + + /// + /// Sends the HTTP request and gets the validated value, throwing on failure, including missing content or a null validated value. Disposes the response upon completion. + /// + /// The optional JSON serializer options. + /// The cancellation token. + /// The validated value. + public async Task GetValueAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) + { + using var response = await _builder.GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await _builder.TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + { + customResult.ThrowOnError(); + return customResult.Bind().Value; + } + + return await response.WithValidator(_validator).GetValueAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Sends the HTTP request and gets the validated value, allowing null and throwing on failure; absent response values are not validated. Disposes the response upon completion. + /// + /// The cancellation token. + /// The validated value or null. + public Task GetValueOrDefaultAsync(CancellationToken cancellationToken = default) => GetValueOrDefaultAsync(null, cancellationToken); + + /// + /// Sends the HTTP request and gets the validated value, allowing null and throwing on failure; absent response values are not validated. Disposes the response upon completion. + /// + /// The optional JSON serializer options. + /// The cancellation token. + /// The validated value or null. + public async Task GetValueOrDefaultAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) + { + using var response = await _builder.GetResponseAsync(cancellationToken).ConfigureAwait(false); + if (await _builder.TryGetStatusResultAsync(response, cancellationToken).ConfigureAwait(false) is { } customResult) + { + customResult.ThrowOnError(); + return customResult.Bind().Value; + } + + return await response.WithValidator(_validator).GetValueOrDefaultAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/CoreEx/Http/ValidatedHttpResponseT.cs b/src/CoreEx/Http/ValidatedHttpResponseT.cs index 6c0f8b399..41d98423b 100644 --- a/src/CoreEx/Http/ValidatedHttpResponseT.cs +++ b/src/CoreEx/Http/ValidatedHttpResponseT.cs @@ -12,6 +12,93 @@ public sealed class ValidatedHttpResponse(HttpResponseMessage response, Valid { private readonly HttpResponseMessage _response = response.ThrowIfNull(); private readonly Validation.IValidator _validator = validator.ThrowIfNull(); + private Dictionary>>? _statusResults; + + /// + /// Configures a to return when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The to return. + /// The instance to support fluent chaining. + public ValidatedHttpResponse WithStatusResult(HttpStatusCode statusCode, Result result) + { + _statusResults ??= []; + _statusResults[statusCode] = (_, _) => Task.FromResult(result); + return this; + } + + /// + /// Configures a factory to invoke when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The function returning the . + /// The instance to support fluent chaining. + public ValidatedHttpResponse WithStatusResult(HttpStatusCode statusCode, Func result) + { + result.ThrowIfNull(); + _statusResults ??= []; + _statusResults[statusCode] = (_, _) => Task.FromResult(result()); + return this; + } + + /// + /// Configures a factory to invoke with the when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The function returning the for the response. + /// The instance to support fluent chaining. + public ValidatedHttpResponse WithStatusResult(HttpStatusCode statusCode, Func result) + { + result.ThrowIfNull(); + _statusResults ??= []; + _statusResults[statusCode] = (response, _) => Task.FromResult(result(response)); + return this; + } + + /// + /// Configures an asynchronous factory to invoke with the when the response status matches the specified . + /// + /// The HTTP status code to match. + /// The asynchronous function returning the for the response. + /// The instance to support fluent chaining. + public ValidatedHttpResponse WithStatusResult(HttpStatusCode statusCode, Func> result) + { + result.ThrowIfNull(); + _statusResults ??= []; + _statusResults[statusCode] = result; + return this; + } + + /// + /// Configures a to return when the response status matches the specified . Alias for . + /// + public ValidatedHttpResponse WithErrorStatus(HttpStatusCode statusCode, Result result) => WithStatusResult(statusCode, result); + + /// + /// Configures a factory to invoke when the response status matches the specified . Alias for . + /// + public ValidatedHttpResponse WithErrorStatus(HttpStatusCode statusCode, Func result) => WithStatusResult(statusCode, result); + + /// + /// Configures a factory to invoke with the when the response status matches the specified . Alias for . + /// + public ValidatedHttpResponse WithErrorStatus(HttpStatusCode statusCode, Func result) => WithStatusResult(statusCode, result); + + /// + /// Configures an asynchronous factory to invoke with the when the response status matches the specified . Alias for . + /// + public ValidatedHttpResponse WithErrorStatus(HttpStatusCode statusCode, Func> result) => WithStatusResult(statusCode, result); + + /// + /// Attempts to get a registered custom for the response's status code. + /// + private async Task TryGetStatusResultAsync(CancellationToken cancellationToken) + { + if (_statusResults is not null && _statusResults.TryGetValue(_response.StatusCode, out var handler)) + return await handler(_response, cancellationToken).ConfigureAwait(false); + + return null; + } /// /// Gets a result containing the validated value; missing content or a null validated value is a failure. @@ -28,6 +115,9 @@ public sealed class ValidatedHttpResponse(HttpResponseMessage response, Valid /// The validated result. public async Task> ToResultAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) { + if (await TryGetStatusResultAsync(cancellationToken).ConfigureAwait(false) is { } customResult) + return customResult.Bind(); + var result = await _response.ToResultAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); var validated = await ValidateAsync(result.AsResult(), true, cancellationToken).ConfigureAwait(false); return validated.AsResult(); @@ -48,6 +138,9 @@ public async Task> ToResultAsync(JsonSerializerOptions? jsonSerializer /// The validated result. public async Task> ToResultOrDefaultAsync(JsonSerializerOptions? jsonSerializerOptions, CancellationToken cancellationToken = default) { + if (await TryGetStatusResultAsync(cancellationToken).ConfigureAwait(false) is { } customResult) + return customResult.Bind(); + var result = await _response.ToResultOrDefaultAsync(jsonSerializerOptions, cancellationToken).ConfigureAwait(false); return await ValidateAsync(result, false, cancellationToken).ConfigureAwait(false); } diff --git a/src/CoreEx/README.md b/src/CoreEx/README.md index d9328da39..e49dd71ed 100644 --- a/src/CoreEx/README.md +++ b/src/CoreEx/README.md @@ -33,6 +33,7 @@ All other CoreEx packages (`CoreEx.Events`, `CoreEx.Database`, `CoreEx.AspNetCor - 🃏 **Wildcard parsing**: `Wildcard` provides standardized `*` and `?` wildcard parsing, validation, and `Regex` compilation for consistent wildcard-based filtering. - 🏷️ **Reference data interfaces**: Core `IReferenceData`, `IReferenceDataCollection`, and `ReferenceDataOrchestrator` types establish the reference data contract used by `CoreEx.RefData` and data-layer packages. - 📐 **Schema versioning**: `SchemaAttribute` and `Schema` utilities for associating a semantic version with entity types, supporting schema-aware messaging and validation. +- 🌐 **HTTP client and request builder**: `client.Send(...)` and `client.SendValue(...)` return an `HttpRequestBuilder` providing fluent request setup (`WithQuery`, `WithPaging`, headers) and terminal execution with automatic response disposal (`GetValueAsync`, `ToResultAsync`, `GetStringAsync`, etc.). `builder.AddNamedHttpClient` / `AddTypedHttpClient` configure service discovery, resilience, and idempotency key handling. ## Key types @@ -41,6 +42,7 @@ All other CoreEx packages (`CoreEx.Events`, `CoreEx.Database`, `CoreEx.AspNetCor | **[`ExecutionContext`](./ExecutionContext.cs)** | `AsyncLocal`-scoped ambient context carrying user identity, tenant ID, operation type, timestamp, and attributes for the lifetime of a request. | | **[`Result`](./Results/Result.cs)** | Value type representing a successful or failed operation with no return value; the basis of railway-oriented programming in CoreEx. | | **[`Result`](./Results/ResultT.cs)** | Value type representing a successful operation with a value, or a typed error — used for functional pipeline-style error propagation. | +| **[`HttpRequestBuilder`](./Http/HttpRequestBuilder.cs)** | Fluent HTTP request builder for configuring requests, chaining extensions, and executing terminal deserialization and result mapping with automatic response disposal. | | **[`DataConsistencyException`](./DataConsistencyException.cs)** | Non-error exception used to signal a potential data consistency issue without treating it as an application error. | | **[`PrecisionTimeProvider`](./PrecisionTimeProvider.cs)** | `TimeProvider` implementation that truncates timestamps to a configurable fractional-second precision for database compatibility. | | **[`OperationType`](./OperationType.cs)** | Enum representing the CRUD operation type (Get, Create, Update, Delete, Query) carried on the `ExecutionContext`. | @@ -52,14 +54,23 @@ See the [Extended Exceptions](#extended-exceptions) section below for the full l ## Validating HTTP response values HTTP success and JSON deserialization do not guarantee a valid dependency response. Use -`WithValidator` with a `CoreEx.Validation.IValidator` to opt into response-contract validation: +`WithValidator` with a `CoreEx.Validation.IValidator` to opt into response-contract validation. +This can be chained directly onto `HttpRequestBuilder` (which manages execution and automatic response disposal) +or applied directly to an `HttpResponseMessage`: ```csharp +// Fluent request with validator and automatic response disposal: +return await _httpClient.Send(HttpMethod.Get, $"api/customers/{Uri.EscapeDataString(id)}") + .WithValidator(customerValidator) + .ToResultAsync(ct).ConfigureAwait(false); + +// Or on an existing HttpResponseMessage (caller owns response disposal): using var response = await _httpClient.GetAsync($"api/customers/{Uri.EscapeDataString(id)}", ct).ConfigureAwait(false); return await response.WithValidator(customerValidator).ToResultAsync(ct).ConfigureAwait(false); ``` -The [typed wrapper](./Http/ValidatedHttpResponseT.cs) reuses the existing response converters and +The typed wrappers ([`ValidatedHttpRequestBuilder`](./Http/ValidatedHttpRequestBuilderT.cs) and +[`ValidatedHttpResponse`](./Http/ValidatedHttpResponseT.cs)) reuse the existing response converters and validates only successful, non-null deserialized values. It returns the **validation result's Value**, including replacements or normalization. `ToResultAsync` requires a non-null value both before and after validation; `ToResultOrDefaultAsync` allows null and skips validation when the @@ -82,9 +93,11 @@ For small client-only rules, prefer a private static readonly `Validator` con private static readonly Validator _validator = Validator.Create() .HasProperty(x => x.Email, c => c.Mandatory().Email()); -// Inside the asynchronous HTTP client method. -using var response = await _httpClient.GetAsync($"api/customers/{Uri.EscapeDataString(id)}", ct).ConfigureAwait(false); -return await response.WithValidator(_validator).ToResultAsync(ct).ConfigureAwait(false); +// Fluent builder with custom status mapping and automatic response disposal: +return await _httpClient.Send(HttpMethod.Get, $"api/customers/{Uri.EscapeDataString(id)}") + .WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError()) + .WithValidator(_validator) + .ToResultAsync(ct).ConfigureAwait(false); ``` Extract a colocated validator only for substantial rules or genuine reuse. Validate consumed @@ -94,9 +107,10 @@ rules through client unit tests rather than exposing them solely for testing. In must not be cached. See the [Customers client sample](../../samples/src/Contoso.Shopping.Infrastructure/Clients/Customers/CustomersHttpClient.cs) for a complete implementation. -Validators can normalize values or perform I/O, so consider their effects explicitly. The wrapper -does not own or dispose the response, cache validation results, or support concurrent response -consumption. Keep responsibility for disposing the response at the call site. +Validators can normalize values or perform I/O, so consider their effects explicitly. +When using `HttpRequestBuilder` terminal methods, the response is automatically disposed. When using +raw `HttpResponseMessage.WithValidator`, the wrapper does not own or dispose the response — retain +responsibility for disposing the response via `using var` at the call site. Like the underlying converters, a terminal operation consumes the content stream; creating a wrapper does not buffer the body for repeated reads. diff --git a/src/CoreEx/Validation/ValidatorExtensions.Requires.cs b/src/CoreEx/Validation/ValidatorExtensions.Requires.cs index 989d336e0..04137b8ef 100644 --- a/src/CoreEx/Validation/ValidatorExtensions.Requires.cs +++ b/src/CoreEx/Validation/ValidatorExtensions.Requires.cs @@ -2,6 +2,18 @@ namespace CoreEx.Validation; public static partial class ValidatorExtensions { + /// + /// Validates (requires) that the is non-null, non-empty, and non-whitespace and continues; otherwise, will throw a . + /// + /// The value to validate. + /// The value name. + /// The friendly text name used in validation messages (defaults to as sentence case where not specified). + /// The value where non-null, non-empty, and non-whitespace. + /// Thrown where the value is null, empty, or whitespace. + [return: NotNull()] + public static string Required(this string? value, [CallerArgumentExpression(nameof(value))] string? name = null, LText? text = null) + => string.IsNullOrWhiteSpace(value) ? throw Validation.CreateRequiredValueResult(name, text).Error : value; + /// /// Validates (requires) that the is non-default and continues; otherwise, will throw a . /// diff --git a/tests/CoreEx.Test.Unit/Http/HttpClientExtensionsTests.cs b/tests/CoreEx.Test.Unit/Http/HttpClientExtensionsTests.cs new file mode 100644 index 000000000..2078b915e --- /dev/null +++ b/tests/CoreEx.Test.Unit/Http/HttpClientExtensionsTests.cs @@ -0,0 +1,55 @@ +namespace CoreEx.Test.Unit.Http; + +[TestFixture] +public class HttpClientExtensionsTests +{ + [Test] + public async Task SendAsync_ResolvesRelativeUriBeforeConfiguration() + { + using var handler = new InspectHandler(async (request, ct) => + { + await Task.Yield(); + request.RequestUri.Should().Be(new Uri("https://example.com/api/products?$filter=name%20eq%20%27A%26B%27&$take=5")); + return new HttpResponseMessage(System.Net.HttpStatusCode.OK); + }); + using var client = new HttpClient(handler) { BaseAddress = new Uri("https://example.com/") }; + using var response = await client.SendAsync(HttpMethod.Get, "/api/products", r => r.WithQuery("name eq 'A&B'").WithPaging(take: 5)); + response.StatusCode.Should().Be(System.Net.HttpStatusCode.OK); + } + + [Test] + public async Task SendAsync_AbsoluteUriDoesNotRequireBaseAddress() + { + using var handler = new InspectHandler((request, ct) => + { + request.RequestUri.Should().Be(new Uri("https://example.com/api")); + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK)); + }); + using var client = new HttpClient(handler); + using var response = await client.SendAsync(HttpMethod.Get, "https://example.com/api"); + response.StatusCode.Should().Be(System.Net.HttpStatusCode.OK); + } + + [TestCase(null, "application/json")] + [TestCase("application/merge-patch+json", "application/merge-patch+json")] + public async Task SendValue_PreservesContentUntilCompletionAndUsesCoreExJson(string? mediaType, string expectedMediaType) + { + var payload = new { ProductId = "123", Quantity = 2 }; + using var handler = new InspectHandler(async (request, ct) => + { + await Task.Yield(); + var content = request.Content ?? throw new InvalidOperationException("Request content is required."); + content.Headers.ContentType?.MediaType.Should().Be(expectedMediaType); + (await content.ReadAsStringAsync(ct)).Should().Be(System.Text.Json.JsonSerializer.Serialize(payload, CoreEx.Json.JsonDefaults.SerializerOptions)); + return new HttpResponseMessage(System.Net.HttpStatusCode.OK); + }); + using var client = new HttpClient(handler) { BaseAddress = new Uri("https://example.com/") }; + using var response = await client.SendValueAsync(HttpMethod.Post, "/api", payload, mediaType: mediaType); + response.StatusCode.Should().Be(System.Net.HttpStatusCode.OK); + } + + private sealed class InspectHandler(Func> inspect) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => inspect(request, cancellationToken); + } +} diff --git a/tests/CoreEx.Test.Unit/Http/HttpClientTests.cs b/tests/CoreEx.Test.Unit/Http/HttpClientTests.cs index 17ed71c90..73dc4941d 100644 --- a/tests/CoreEx.Test.Unit/Http/HttpClientTests.cs +++ b/tests/CoreEx.Test.Unit/Http/HttpClientTests.cs @@ -106,12 +106,7 @@ public class EntryPoint { public static void ConfigureApplication(IHostApplicationBuilder builder) { - builder.Services.AddHttpClient("UnitTest", static client => - { - client.BaseAddress = new Uri("http://unittest"); - }) - .AddIdempotencyKeyHandler() - .AddStandardResilienceHandler(); // https://learn.microsoft.com/en-us/dotnet/core/resilience/http-resilience?tabs=dotnet-cli#standard-resilience-handler-defaults + builder.AddNamedHttpClient("UnitTest", client => client.BaseAddress = new Uri("http://unittest")); } } -} \ No newline at end of file +} diff --git a/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs b/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs new file mode 100644 index 000000000..12f1be770 --- /dev/null +++ b/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs @@ -0,0 +1,640 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using CoreEx.Data; +using CoreEx.Http; + +namespace CoreEx.Test.Unit.Http; + +[TestFixture] +public class HttpRequestBuilderTests +{ + private sealed class Widget + { + public string? Name { get; set; } + } + + private sealed class DisposableResponse : HttpResponseMessage + { + public DisposableResponse(HttpStatusCode statusCode, HttpContent? content = null) : base(statusCode) + { + if (content is not null) + Content = content; + } + + public bool IsDisposed { get; private set; } + + protected override void Dispose(bool disposing) + { + IsDisposed = true; + base.Dispose(disposing); + } + } + + private sealed class InspectHandler(Func> inspect) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => inspect(request, cancellationToken); + } + + private sealed class WidgetValidator(Func>> validate) : CoreEx.Validation.IValidator + { + public int Calls { get; private set; } + + public Task> ValidateAsync(Widget value, CancellationToken cancellationToken = default) + { + Calls++; + return validate(value, cancellationToken); + } + } + + private sealed class WidgetValidationResult(Widget? value, bool hasErrors = false, Exception? error = null) : CoreEx.Validation.IValidationResult + { + public Widget? Value => value; + public bool HasErrors => hasErrors; + public CoreEx.Entities.MessageItemCollection? Messages => null; + public Exception? ToException() => error; + public CoreEx.Validation.IValidationResult ThrowOnError() => error is not null ? throw error : this; + public CoreEx.Results.Result ToResult() => error is null ? CoreEx.Results.Result.Success : CoreEx.Results.Result.Fail(error); + } + + private static HttpClient CreateClient(HttpMessageHandler handler) => new(handler) { BaseAddress = new Uri("https://example.com/") }; + + [Test] + public void Properties_And_ConfigureChaining() + { + using var client = new HttpClient(); + var builder = client.Send(HttpMethod.Post, "/api/test", r => r.Headers.Add("X-First", "1")); + + builder.Client.Should().BeSameAs(client); + builder.Method.Should().Be(HttpMethod.Post); + builder.RequestUri.Should().Be("/api/test"); + + builder.Configure(r => r.Headers.Add("X-Second", "2")); + + using var handler = new InspectHandler((req, _) => + { + req.Headers.GetValues("X-First").Single().Should().Be("1"); + req.Headers.GetValues("X-Second").Single().Should().Be("2"); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); + }); + + using var testClient = CreateClient(handler); + var configuredBuilder = new HttpRequestBuilder(testClient, HttpMethod.Post, "/api/test", r => r.Headers.Add("X-First", "1")) + .Configure(r => r.Headers.Add("X-Second", "2")); + + using var response = configuredBuilder.GetResponseAsync().GetAwaiter().GetResult(); + response.StatusCode.Should().Be(HttpStatusCode.OK); + } + + [Test] + public async Task GetResponseAsync_And_SendAsync_DoNotDisposeResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.OK); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var response1 = await client.Send(HttpMethod.Get, "/test").GetResponseAsync(); + lastResponse!.IsDisposed.Should().BeFalse(); + response1.Dispose(); + lastResponse.IsDisposed.Should().BeTrue(); + + var response2 = await client.Send(HttpMethod.Get, "/test").SendAsync(); + lastResponse!.IsDisposed.Should().BeFalse(); + response2.Dispose(); + lastResponse.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task GetValueAsync_ReturnsDeserializedValue_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.OK, new StringContent("{\"name\":\"WidgetA\"}", Encoding.UTF8, "application/json")); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var widget = await client.Send(HttpMethod.Get, "/widgets/1").GetValueAsync(); + widget.Name.Should().Be("WidgetA"); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task GetValueOrDefaultAsync_ReturnsNullForNoContent_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.NoContent); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var widget = await client.Send(HttpMethod.Get, "/widgets/1").GetValueOrDefaultAsync(); + widget.Should().BeNull(); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task GetStringAsync_ReturnsString_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.OK, new StringContent("hello world", Encoding.UTF8, "text/plain")); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var str = await client.Send(HttpMethod.Get, "/greeting").GetStringAsync(); + str.Should().Be("hello world"); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task ThrowOnErrorAsync_ThrowsOnFailure_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.InternalServerError); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + Func act = async () => await client.Send(HttpMethod.Get, "/fail").ThrowOnErrorAsync(); + await act.Should().ThrowAsync(); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task ToResultAsync_ReturnsResult_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.OK); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/ok").ToResultAsync(); + result.IsSuccess.Should().BeTrue(); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task ToResultAsync_Typed_ReturnsValue_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.OK, new StringContent("{\"name\":\"test\"}", Encoding.UTF8, "application/json")); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/item").ToResultAsync(); + result.IsSuccess.Should().BeTrue(); + result.Value.Name.Should().Be("test"); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task ToResultOrDefaultAsync_Typed_ReturnsNullForNoContent_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.NoContent); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/item").ToResultOrDefaultAsync(); + result.IsSuccess.Should().BeTrue(); + result.Value.Should().BeNull(); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task ToStringResultAsync_ReturnsString_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.OK, new StringContent("result string", Encoding.UTF8, "text/plain")); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/str").ToStringResultAsync(); + result.IsSuccess.Should().BeTrue(); + result.Value.Should().Be("result string"); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task ToProblemDetailsAsync_ReturnsProblemDetails_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + var pdJson = "{\"status\":400,\"title\":\"Bad Request\",\"type\":\"https://tools.ietf.org/html/rfc7231#section-6.5.1\"}"; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.BadRequest, new StringContent(pdJson, Encoding.UTF8, "application/problem+json")); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var pde = await client.Send(HttpMethod.Get, "/bad").ToProblemDetailsAsync(); + pde.Should().NotBeNull(); + pde!.ProblemDetails.Title.Should().Be("Bad Request"); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task ThrowOnBusinessExceptionAsync_ThrowsOnBusinessException_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + var pdJson = "{\"status\":400,\"title\":\"Business Error\",\"errorType\":\"business\"}"; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.BadRequest, new StringContent(pdJson, Encoding.UTF8, "application/problem+json")); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + Func act = async () => await client.Send(HttpMethod.Get, "/biz").ThrowOnBusinessExceptionAsync(); + await act.Should().ThrowAsync(); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task SendValue_SerializesPayload_AndSupportsFluentChaining() + { + var payload = new Widget { Name = "PayloadWidget" }; + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler(async (req, ct) => + { + req.Method.Should().Be(HttpMethod.Post); + req.RequestUri.Should().Be(new Uri("https://example.com/api/widgets")); + req.Content.Should().NotBeNull(); + req.Content!.Headers.ContentType?.MediaType.Should().Be("application/json"); + var body = await req.Content.ReadAsStringAsync(ct); + body.Should().Contain("PayloadWidget"); + + lastResponse = new DisposableResponse(HttpStatusCode.Created, new StringContent("{\"name\":\"CreatedWidget\"}", Encoding.UTF8, "application/json")); + return lastResponse; + }); + + using var client = CreateClient(handler); + var created = await client.SendValue(HttpMethod.Post, "/api/widgets", payload) + .GetValueAsync(); + + created.Name.Should().Be("CreatedWidget"); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task WithValidator_ValidatesResponse_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.OK, new StringContent("{\"name\":\"ValidatedWidget\"}", Encoding.UTF8, "application/json")); + return Task.FromResult(lastResponse); + }); + + var replacement = new Widget { Name = "ReplacedWidget" }; + var validator = new WidgetValidator((w, _) => Task.FromResult>(new WidgetValidationResult(replacement))); + + using var client = CreateClient(handler); + var validatedBuilder = client.Send(HttpMethod.Get, "/api/widgets/1").WithValidator(validator); + + validatedBuilder.Builder.Should().NotBeNull(); + validatedBuilder.Validator.Should().BeSameAs(validator); + + var widget = await validatedBuilder.GetValueAsync(); + widget.Should().BeSameAs(replacement); + validator.Calls.Should().Be(1); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task WithValidator_ValidationFailure_ThrowsHttpRequestException_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.OK, new StringContent("{\"name\":\"BadWidget\"}", Encoding.UTF8, "application/json")); + return Task.FromResult(lastResponse); + }); + + var error = ValidationException.Create("name", "Name is invalid."); + var validator = new WidgetValidator((_, _) => Task.FromResult>(new WidgetValidationResult(null, true, error))); + + using var client = CreateClient(handler); + Func act = async () => await client.Send(HttpMethod.Get, "/api/widgets/1").WithValidator(validator).GetValueAsync(); + var ex = (await act.Should().ThrowAsync()).Which; + ex.InnerException.Should().BeSameAs(error); + validator.Calls.Should().Be(1); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task WithValidator_ToResultAsync_ReturnsFailureOnValidationError_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.OK, new StringContent("{\"name\":\"BadWidget\"}", Encoding.UTF8, "application/json")); + return Task.FromResult(lastResponse); + }); + + var error = ValidationException.Create("name", "Name is invalid."); + var validator = new WidgetValidator((_, _) => Task.FromResult>(new WidgetValidationResult(null, true, error))); + + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/api/widgets/1").WithValidator(validator).ToResultAsync(); + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType().Which.InnerException.Should().BeSameAs(error); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task WithValidator_GetValueOrDefaultAsync_ReturnsNullOnNoContent_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.NoContent); + return Task.FromResult(lastResponse); + }); + + var validator = new WidgetValidator((w, _) => Task.FromResult>(new WidgetValidationResult(w))); + + using var client = CreateClient(handler); + var widget = await client.Send(HttpMethod.Get, "/api/widgets/1").WithValidator(validator).GetValueOrDefaultAsync(); + widget.Should().BeNull(); + validator.Calls.Should().Be(0); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task FluentWithMethods_ConfigureRequestMessageCorrectly() + { + HttpRequestMessage? captured = null; + using var handler = new InspectHandler((req, _) => + { + captured = req; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); + }); + + using var client = CreateClient(handler); + await client.Send(HttpMethod.Patch, "/api/widgets/1") + .WithIfMatch("abc") + .WithMergePatchJsonContentType() + .WithIdempotencyKey("idemp-123") + .WithQuery("name eq 'test'", "name asc") + .WithIncludeText() + .WithIncludeInactive() + .WithFields("id", "name") + .WithoutFields("internalNotes") + .WithPaging(skip: 10, take: 20, count: true) + .GetResponseAsync(); + + captured.Should().NotBeNull(); + captured!.Headers.IfMatch.Should().ContainSingle().Which.Tag.Should().Be("\"abc\""); + captured.Headers.GetValues(HttpNames.IdempotencyKeyHeaderName).Should().ContainSingle().Which.Should().Be("idemp-123"); + captured.Content.Should().BeNull(); // MergePatchJsonContentType applies to Content if present, but here content is not set yet + var query = captured.RequestUri!.Query; + query.Should().Contain("$filter=name%20eq%20%27test%27"); + query.Should().Contain("$orderby=name%20asc"); + query.Should().Contain("$text=true"); + query.Should().Contain("$inactive=true"); + query.Should().Contain("$fields=id%2Cname"); + query.Should().Contain("$exclude=internalNotes"); + query.Should().Contain("$skip=10"); + query.Should().Contain("$take=20"); + query.Should().Contain("$count=true"); + } + + [Test] + public async Task FluentWithMethods_WithPayloadAndIfNoneMatch() + { + HttpRequestMessage? captured = null; + using var handler = new InspectHandler((req, _) => + { + captured = req; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); + }); + + using var client = CreateClient(handler); + await client.SendValue(HttpMethod.Put, "/api/widgets/1", new Widget { Name = "test" }) + .WithIfNoneMatch("xyz") + .WithMergePatchJsonContentType() + .WithPaging(new PagingArgs(5, 15, true)) + .WithQuery(new QueryArgs { Filter = "status eq 'active'" }) + .GetResponseAsync(); + + captured.Should().NotBeNull(); + captured!.Headers.IfNoneMatch.Should().ContainSingle().Which.Tag.Should().Be("\"xyz\""); + captured.Content!.Headers.ContentType?.MediaType.Should().Be(HttpNames.MergePatchJsonMediaTypeName); + var query = captured.RequestUri!.Query; + query.Should().Contain("$skip=5"); + query.Should().Contain("$take=15"); + query.Should().Contain("$count=true"); + query.Should().Contain("$filter=status%20eq%20%27active%27"); + } + + [Test] + public void NullArguments_ThrowArgumentNullException() + { + using var client = new HttpClient(); + Action nullClient = () => Extensions.Send(null!, HttpMethod.Get, "/"); + Action nullMethod = () => client.Send(null!, "/"); + Action nullValidator = () => client.Send(HttpMethod.Get, "/").WithValidator(null!); + Action nullConfigure = () => client.Send(HttpMethod.Get, "/").Configure(null!); + + nullClient.Should().Throw(); + nullMethod.Should().Throw(); + nullValidator.Should().Throw(); + nullConfigure.Should().Throw(); + } + + [Test] + public async Task WithStatusResult_DirectResult_InterceptsStatus_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.NotFound); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/items/123") + .WithStatusResult(HttpStatusCode.NotFound, CoreEx.Results.Result.NotFoundError()) + .ToResultAsync(); + + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType(); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task WithStatusResult_FactoryOverloads_InterceptStatus_AndDisposeResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.Conflict); + return Task.FromResult(lastResponse); + }); + + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/items/123") + .WithStatusResult(HttpStatusCode.Conflict, () => CoreEx.Results.Result.ConflictError()) + .ToResultAsync(); + + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType(); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task WithStatusResult_ResponseAwareFactory_InterceptsStatus() + { + using var handler = new InspectHandler((_, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest))); + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/items/123") + .WithStatusResult(HttpStatusCode.BadRequest, res => CoreEx.Results.Result.BusinessError($"Status was {(int)res.StatusCode}")) + .ToResultAsync(); + + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType().Which.Message.Should().Be("Status was 400"); + } + + [Test] + public async Task WithStatusResult_AsyncFactory_InterceptsStatus() + { + using var handler = new InspectHandler((_, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.PreconditionFailed))); + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/items/123") + .WithStatusResult(HttpStatusCode.PreconditionFailed, async (res, ct) => + { + await Task.Yield(); + return CoreEx.Results.Result.ConcurrencyError(); + }) + .ToResultAsync(); + + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType(); + } + + [Test] + public async Task WithStatusResult_ShortCircuitsValidator_AndDisposesResponse() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.NotFound); + return Task.FromResult(lastResponse); + }); + + var validator = new WidgetValidator((w, _) => Task.FromResult>(new WidgetValidationResult(w))); + + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/items/123") + .WithStatusResult(HttpStatusCode.NotFound, () => CoreEx.Results.Result.NotFoundError()) + .WithValidator(validator) + .ToResultAsync(); + + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType(); + validator.Calls.Should().Be(0); // Short-circuited! + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task WithStatusResult_ChainedAfterValidator_ShortCircuitsValidator() + { + DisposableResponse? lastResponse = null; + using var handler = new InspectHandler((_, _) => + { + lastResponse = new DisposableResponse(HttpStatusCode.NotFound); + return Task.FromResult(lastResponse); + }); + + var validator = new WidgetValidator((w, _) => Task.FromResult>(new WidgetValidationResult(w))); + + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/items/123") + .WithValidator(validator) + .WithStatusResult(HttpStatusCode.NotFound, () => CoreEx.Results.Result.NotFoundError()) + .ToResultAsync(); + + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType(); + validator.Calls.Should().Be(0); + lastResponse!.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task WithStatusResult_GetValueAsync_ThrowsMappedException() + { + using var handler = new InspectHandler((_, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound))); + using var client = CreateClient(handler); + Func act = async () => await client.Send(HttpMethod.Get, "/items/123") + .WithStatusResult(HttpStatusCode.NotFound, () => CoreEx.Results.Result.NotFoundError()) + .GetValueAsync(); + + await act.Should().ThrowAsync(); + } + + [Test] + public async Task WithStatusResult_ThrowOnErrorAsync_ThrowsMappedException() + { + using var handler = new InspectHandler((_, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound))); + using var client = CreateClient(handler); + Func act = async () => await client.Send(HttpMethod.Get, "/items/123") + .WithStatusResult(HttpStatusCode.NotFound, () => CoreEx.Results.Result.NotFoundError()) + .ThrowOnErrorAsync(); + + await act.Should().ThrowAsync(); + } + + [Test] + public async Task WithErrorStatus_Alias_BehavesIdentically() + { + using var handler = new InspectHandler((_, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound))); + using var client = CreateClient(handler); + var result = await client.Send(HttpMethod.Get, "/items/123") + .WithErrorStatus(HttpStatusCode.NotFound, _ => CoreEx.Results.Result.NotFoundError()) + .ToResultAsync(); + + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType(); + } + + [Test] + public async Task ValidatedHttpResponse_WithStatusResult_InterceptsStatus() + { + using var response = new HttpResponseMessage(HttpStatusCode.NotFound); + var validator = new WidgetValidator((w, _) => Task.FromResult>(new WidgetValidationResult(w))); + + var result = await response.WithValidator(validator) + .WithStatusResult(HttpStatusCode.NotFound, () => CoreEx.Results.Result.NotFoundError()) + .ToResultAsync(); + + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType(); + validator.Calls.Should().Be(0); + } +} diff --git a/tests/CoreEx.Test.Unit/Http/HttpResponseMessageExtensionsTests.cs b/tests/CoreEx.Test.Unit/Http/HttpResponseMessageExtensionsTests.cs index 72999e74a..6e4934d34 100644 --- a/tests/CoreEx.Test.Unit/Http/HttpResponseMessageExtensionsTests.cs +++ b/tests/CoreEx.Test.Unit/Http/HttpResponseMessageExtensionsTests.cs @@ -105,6 +105,91 @@ public async Task ToResultOrDefaultAsync_NotSuccess_IsFailure() r.IsFailure.Should().BeTrue(); } + [Test] + public async Task ToResultOrDefaultAsync_NullContent_IsSuccessWithNull() + { + var r = await Empty(HttpStatusCode.OK).ToResultOrDefaultAsync(); + r.IsSuccess.Should().BeTrue(); + r.Value.Should().BeNull(); + } + + [Test] + public async Task ToStringResultAsync_Value_ReturnsValue() + { + var r = await new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("hello world") }.ToStringResultAsync(); + r.IsSuccess.Should().BeTrue(); + r.Value.Should().Be("hello world"); + } + + [Test] + public async Task ToStringResultAsync_NullContent_IsSuccessWithNull() + { + var r = await Empty(HttpStatusCode.OK).ToStringResultAsync(); + r.IsSuccess.Should().BeTrue(); + r.Value.Should().BeNull(); + } + + [Test] + public async Task ToStringResultAsync_EmptyString_IsSuccessWithNull() + { + var r = await new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(string.Empty) }.ToStringResultAsync(); + r.IsSuccess.Should().BeTrue(); + r.Value.Should().BeNull(); + } + + [Test] + public async Task ToStringResultAsync_NoContent_IsSuccessWithNull() + { + var r = await Empty(HttpStatusCode.NoContent).ToStringResultAsync(); + r.IsSuccess.Should().BeTrue(); + r.Value.Should().BeNull(); + } + + [Test] + public async Task ToStringResultAsync_NotSuccess_IsFailure() + { + var r = await Empty(HttpStatusCode.InternalServerError).ToStringResultAsync(); + r.IsFailure.Should().BeTrue(); + r.Error.Should().BeOfType(); + } + + [Test] + public async Task GetStringAsync_Value_ReturnsValue() + { + var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("test-data") }; + var value = await response.GetStringAsync(); + value.Should().Be("test-data"); + } + + [Test] + public async Task GetStringAsync_NullContent_ReturnsNull() + { + var value = await Empty(HttpStatusCode.OK).GetStringAsync(); + value.Should().BeNull(); + } + + [Test] + public async Task GetStringAsync_EmptyString_ReturnsNull() + { + var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(string.Empty) }; + var value = await response.GetStringAsync(); + value.Should().BeNull(); + } + + [Test] + public async Task GetStringAsync_NoContent_ReturnsNull() + { + var value = await Empty(HttpStatusCode.NoContent).GetStringAsync(); + value.Should().BeNull(); + } + + [Test] + public async Task GetStringAsync_NotSuccess_Throws() + { + var act = async () => await Empty(HttpStatusCode.InternalServerError).GetStringAsync(); + await act.Should().ThrowAsync(); + } + [Test] public async Task GetValueAsync_Value_ReturnsValue() => (await Json("{\"name\":\"abc\"}").GetValueAsync()).Name.Should().Be("abc"); diff --git a/tests/CoreEx.Test.Unit/Validation/ValidationTests.cs b/tests/CoreEx.Test.Unit/Validation/ValidationTests.cs index 6491c6061..de6da3544 100644 --- a/tests/CoreEx.Test.Unit/Validation/ValidationTests.cs +++ b/tests/CoreEx.Test.Unit/Validation/ValidationTests.cs @@ -46,6 +46,29 @@ public void Required_NonDefault_ReturnsValue() result.Should().Be(123); } + [TestCase("value")] + [TestCase(" value ")] + public void Required_StringNonWhitespace_ReturnsValue(string value) + => value.Required().Should().Be(value); + + [TestCase(null)] + [TestCase("")] + [TestCase(" ")] + public void Required_StringNullOrWhitespace_ThrowsValidationException(string? value) + { + Action act = () => value.Required(); + act.Should().Throw(); + } + + [Test] + public void Required_StringNullOrWhitespace_CustomNameAndText() + { + string? value = ""; + Action act = () => value.Required("myValue", new("custom", "Custom")); + act.Should().Throw() + .Which.Messages.Should().ContainSingle().Which.ToString().Should().Be("MessageItem { Type = Error, Text = Custom is required., Property = myValue }"); + } + [Test] public void Required_Default_ThrowsValidationException() { @@ -131,4 +154,4 @@ public void Requires_ResultT_Func_EmptyName_Throws() Action act = () => result.Requires(() => 1, ""); act.Should().Throw(); } -} \ No newline at end of file +} From d7f8001b3eb19182ebea5d6b46a20b778d0f2f1d Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 13 Oct 2026 08:08:12 -0700 Subject: [PATCH 02/25] Add QueryArgs free-text search support Wire configurable keyword and ranked search through querying, HTTP, OpenAPI and GraphQL. Update the Products and MCP samples, tests and querying documentation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- CHANGELOG.md | 4 + .../src/Contoso.Mcp/Tools/CommerceTools.cs | 10 +- .../Controllers/ProductReadController.cs | 2 +- .../Repositories/ProductQueryArgsConfig.cs | 7 + .../ReadTests.ProductQuery.cs | 49 ++++ .../ReadTests/Product_Query_Schema.res.json | 6 + .../NSwagOpenApiOperationProcessor.cs | 3 + src/CoreEx.AspNetCore.NSwag/README.md | 2 +- .../Abstractions/WebApiOptionsBase.cs | 1 + .../Http/AspNetCoreHttpExtensions.cs | 10 + src/CoreEx.AspNetCore/Mvc/QueryAttribute.cs | 6 + src/CoreEx.AspNetCore/OpenApiOptions.cs | 5 + src/CoreEx.Data.GraphQL/GraphQLEngine.cs | 1 + src/CoreEx.Data.GraphQL/GraphQLQueryRoot.cs | 4 +- .../Internal/GraphQLArgsMapper.cs | 9 +- .../GraphQLIntrospectionSchemaBuilder.cs | 3 + src/CoreEx.Data.GraphQL/README.md | 6 +- src/CoreEx.Data/Querying/QueryArgsConfig.cs | 56 ++++- .../Querying/QueryArgsParseResult.cs | 15 +- src/CoreEx.Data/Querying/QueryExtensions.cs | 54 +++- .../Querying/QuerySearchFieldConfig.cs | 144 +++++++++++ src/CoreEx.Data/Querying/QuerySearchMode.cs | 30 +++ src/CoreEx.Data/Querying/QuerySearchParser.cs | 230 ++++++++++++++++++ .../Querying/QuerySearchParserException.cs | 13 + .../Querying/QuerySearchParserResult.cs | 112 +++++++++ src/CoreEx.Data/Querying/README.md | 113 ++++++++- src/CoreEx/Data/QueryArgs.cs | 20 +- src/CoreEx/Extensions.HttpRequestMessage.cs | 20 +- src/CoreEx/Http/HttpNames.cs | 5 + src/CoreEx/Http/HttpRequestBuilder.cs | 7 + .../Querying/QuerySearchParserTests.cs | 209 ++++++++++++++++ .../Http/HttpRequestBuilderTests.cs | 2 + .../Http/HttpRequestMessageExtensionsTests.cs | 27 ++ 33 files changed, 1147 insertions(+), 38 deletions(-) create mode 100644 src/CoreEx.Data/Querying/QuerySearchFieldConfig.cs create mode 100644 src/CoreEx.Data/Querying/QuerySearchMode.cs create mode 100644 src/CoreEx.Data/Querying/QuerySearchParser.cs create mode 100644 src/CoreEx.Data/Querying/QuerySearchParserException.cs create mode 100644 src/CoreEx.Data/Querying/QuerySearchParserResult.cs create mode 100644 tests/CoreEx.Data.Test.Unit/Querying/QuerySearchParserTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index e8528441c..0c23e9b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Represents the **NuGet** versions. +## v4.2.0 +- *Enhancement:* Added OData-style free-text `$search` support to the `QueryArgs` querying pipeline: `QueryArgs.Search`; `QueryArgsConfig.WithSearch(Action)` with `QuerySearchMode` (`Keyword` executed by `QueryExtensions.Where` as a provider-agnostic predicate where every term must match a configured keyword field; `FullText`/`Semantic`/`Hybrid` are relevance-ranked and executed by the repository via `QueryArgsParseResult.IsRankedSearch`/`SearchResult.Text`, with `$orderby` rejected); `QuerySearchParserException`; `WithSearch` on `HttpRequestMessage`/`HttpRequestBuilder`; `QueryAttribute.SupportsSearch` (NSwag `$search` parameter); and a GraphQL-lite `search` argument on list roots. `$search` is rejected where not configured. The Products sample (and its MCP `search_products` tool) now uses `$search`. +- *Fixed:* `HttpRequestMessage.WithQuery(QueryArgs)` (and therefore `HttpRequestBuilder.WithQuery(QueryArgs)`) previously ignored `IsIncludeText` and `IsIncludeInactive`; these are now sent as `$text=true` and `$inactive=true` respectively. + ## v4.1.0 - *Enhancement:* Added `NamedDestinationProvider` (`AddNamedDestinationProvider()`); events are sent to the shared destination (topic), whereas commands are sent to a per-domain queue named `{Destination}-{domain}` so that only the addressed domain receives them. A `null` or empty command domain is self-addressed to the `IHostSettings.DomainName`. - *Enhancement:* Added keyed `WithReceiver`/`WithSessionReceiver` and `WithKeyedSubscribedSubscriber` support to `AzureServiceBusReceiving` to enable multiple receivers (e.g. events topic and commands queue) within a single Subscribe host. diff --git a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs index 62db6bd25..ab5e816a5 100644 --- a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs +++ b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs @@ -3,11 +3,11 @@ namespace Contoso.Mcp.Tools; [McpServerToolType] public static class CommerceTools { - /// Searches products by text or SKU prefix. - [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products whose description contains the search phrase or whose SKU starts with it. Returns up to the requested number of matching product summaries.")] + /// Searches products by free-text keywords. + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products using free-text keywords; each word must appear in the product description or be the beginning of the product SKU. Returns up to the requested number of matching product summaries.")] public static Task SearchProductsAsync( [FromServices] IHttpClientFactory clientFactory, - [Description("A word or phrase to find in product descriptions, or the beginning of a product SKU.")] string searchTerm, + [Description("One or more keywords to find in product descriptions, or the beginning of a product SKU.")] string searchTerm, [Description("Maximum number of products to return (1-25).")] int take = 10, CancellationToken cancellationToken = default) { @@ -15,11 +15,9 @@ public static Task SearchProductsAsync( if (take is < 1 or > 25) throw new ArgumentOutOfRangeException(nameof(take), "The result limit must be between 1 and 25."); - var filterTerm = searchTerm.Replace("'", "''", StringComparison.Ordinal); - var filter = $"contains(Text, '{filterTerm}') or startswith(Sku, '{filterTerm}')"; return ProductsApi(clientFactory) .Send(HttpMethod.Get, "/api/products") - .WithQuery(filter) + .WithSearch(searchTerm) .WithPaging(take: take) .GetValueAsync(cancellationToken); } diff --git a/samples/src/Contoso.Products.Api/Controllers/ProductReadController.cs b/samples/src/Contoso.Products.Api/Controllers/ProductReadController.cs index 447fc267a..321bf2878 100644 --- a/samples/src/Contoso.Products.Api/Controllers/ProductReadController.cs +++ b/samples/src/Contoso.Products.Api/Controllers/ProductReadController.cs @@ -13,7 +13,7 @@ public class ProductReadController(WebApi webApi, IProductReadService service) : [HttpGet] [ProducesResponseType(typeof(ProductLite[]), 200)] - [Query(supportsOrderBy: true), Paging(supportsCount: true)] + [Query(supportsOrderBy: true, SupportsSearch = true), Paging(supportsCount: true)] public Task QueryAsync(CancellationToken cancellationToken = default) => _webApi.GetAsync(Request, (ro, ct) => _service.QueryAsync(ro.QueryArgs, ro.PagingArgs, ct), cancellationToken: cancellationToken); [HttpGet("$query")] diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs index 413c05fb2..39e5e7b3d 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs @@ -23,5 +23,12 @@ public ProductQueryArgsConfig() .AddField(nameof(Contracts.ProductBase.Sku), c => c.WithDefault().WithAlwaysInclude()) .AddField(nameof(Contracts.ProductBase.Text)) .AddField(nameof(Contracts.ProductBase.Brand))); + + // Configure the free-text $search; keyword mode (unranked) where each term must match the Text (contains) or Sku (starts-with). A semantic/hybrid (vector) search would + // instead use WithMode(QuerySearchMode.Semantic) and be executed by the repository (see QueryArgsParseResult.IsRankedSearch). + WithSearch(search => search + .WithDefaultModelPrefix("Product") + .AddKeywordField(nameof(Contracts.ProductBase.Text), c => c.AsUpperCase()) + .AddKeywordField(nameof(Contracts.ProductBase.Sku), c => c.WithOperator(QueryFilterOperator.StartsWith).AsUpperCase())); } } diff --git a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs index dfaa32733..4ce25c479 100644 --- a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs +++ b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs @@ -97,4 +97,53 @@ public void Product_Query_FilterByBrandAndSubCategory() r.Should().NotBeNull().And.HaveCount(2); } + + [Test] + public void Product_Query_Search_MultipleTerms() + { + var r = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=rear derailleur") + .AssertOK() + .Value; + + r.Should().NotBeNull() + .And.HaveCount(3) + .And.BeInAscendingOrder(x => x.Sku) + .And.OnlyContain(p => p.Text!.Contains("Rear Derailleur")); + } + + [Test] + public void Product_Query_Search_SkuPrefix() + { + var r = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=sram-xx1") + .AssertOK() + .Value; + + r.Should().NotBeNull() + .And.ContainSingle() + .Which.Sku.Should().Be("SRAM-XX1-EAGLE-CASSETTE"); + } + + [Test] + public void Product_Query_Search_WithFilterAndOrderBy() + { + var r = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=cleaner&$filter=brand eq 'MUCOFF'&$orderby=text desc") + .AssertOK() + .Value; + + r.Should().NotBeNull() + .And.HaveCount(2) + .And.BeInDescendingOrder(x => x.Text); + } + + [Test] + public void Product_Query_Search_TooManyTerms() + { + Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=a b c d e f g h i") + .AssertBadRequest() + .AssertErrors(new ApiError("$search", "Statement must not contain more than 8 distinct terms.")); + } } diff --git a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json index f5d287731..f308c3f91 100644 --- a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json +++ b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json @@ -66,5 +66,11 @@ } }, "default": "sku asc" + }, + "search": { + "mode": "keyword", + "ranked": false, + "maxLength": 200, + "maxTerms": 8 } } \ No newline at end of file diff --git a/src/CoreEx.AspNetCore.NSwag/NSwagOpenApiOperationProcessor.cs b/src/CoreEx.AspNetCore.NSwag/NSwagOpenApiOperationProcessor.cs index 54bd1bd3a..38b3b1065 100644 --- a/src/CoreEx.AspNetCore.NSwag/NSwagOpenApiOperationProcessor.cs +++ b/src/CoreEx.AspNetCore.NSwag/NSwagOpenApiOperationProcessor.cs @@ -67,6 +67,9 @@ private void HandleQueryAttribute(AspNetCoreOperationProcessorContext context) if (query.SupportsOrderBy) context.OperationDescription.Operation.Parameters.Add(CreateParameter(HttpNames.QueryOrderByQueryStringName, Options.QueryOrderByText, JsonObjectType.String, $"{nameof(QueryArgs)}{nameof(QueryArgs.OrderBy)}")); + + if (query.SupportsSearch) + context.OperationDescription.Operation.Parameters.Add(CreateParameter(HttpNames.QuerySearchQueryStringName, Options.QuerySearchText, JsonObjectType.String, $"{nameof(QueryArgs)}{nameof(QueryArgs.Search)}")); } /// diff --git a/src/CoreEx.AspNetCore.NSwag/README.md b/src/CoreEx.AspNetCore.NSwag/README.md index fb82ea490..23a4d08d5 100644 --- a/src/CoreEx.AspNetCore.NSwag/README.md +++ b/src/CoreEx.AspNetCore.NSwag/README.md @@ -13,7 +13,7 @@ ## Key capabilities - 📋 **Paging parameters**: Reads `[PagingAttribute]` and adds `$skip`, `$take`, and optionally `$count` query-string parameters with descriptions controlled by `OpenApiOptions.PagingSkipText`, `PagingTakeText`, etc. -- 🔍 **Query parameters**: Reads `[QueryAttribute]` and adds `$filter` and/or `$orderby` query-string parameters when `SupportsFilter` / `SupportsOrderBy` are set. +- 🔍 **Query parameters**: Reads `[QueryAttribute]` and adds `$filter`, `$orderby` and/or `$search` query-string parameters when `SupportsFilter` / `SupportsOrderBy` / `SupportsSearch` are set. - 📦 **Request body content types**: Reads `[AcceptsAttribute]` and populates the operation `RequestBody` with the declared content type(s) and NSwag-inferred JSON schema for the body type. - 🔑 **Idempotency-key header**: Reads `[IdempotencyKeyAttribute]` and adds an `Idempotency-Key` header parameter to the operation. - ✅ **Non-nullable properties**: Walks the request/response types of each operation and marks any property decorated with `CoreEx.Entities.NonNullableAttribute` as `nullable: false` in the schema, regardless of whether it is declared nullable in code (e.g. `string?`, `int?`). diff --git a/src/CoreEx.AspNetCore/Abstractions/WebApiOptionsBase.cs b/src/CoreEx.AspNetCore/Abstractions/WebApiOptionsBase.cs index 3199a4e11..a2811481e 100644 --- a/src/CoreEx.AspNetCore/Abstractions/WebApiOptionsBase.cs +++ b/src/CoreEx.AspNetCore/Abstractions/WebApiOptionsBase.cs @@ -201,6 +201,7 @@ private QueryArgs GetQueryArgs() { Filter = GetNamedQueryString(Request.Query, HttpNames.QueryFilterQueryStringName), OrderBy = GetNamedQueryString(Request.Query, HttpNames.QueryOrderByQueryStringName), + Search = GetNamedQueryString(Request.Query, HttpNames.QuerySearchQueryStringName), IncludeFields = IncludeFields, ExcludeFields = ExcludeFields, IsIncludeText = IsIncludeText, diff --git a/src/CoreEx.AspNetCore/Http/AspNetCoreHttpExtensions.cs b/src/CoreEx.AspNetCore/Http/AspNetCoreHttpExtensions.cs index 8c4ac001b..bb4034514 100644 --- a/src/CoreEx.AspNetCore/Http/AspNetCoreHttpExtensions.cs +++ b/src/CoreEx.AspNetCore/Http/AspNetCoreHttpExtensions.cs @@ -22,6 +22,16 @@ public static partial class AspNetCoreHttpExtensions /// The to support fluent-style method-chaining. public static RouteHandlerBuilder WithQuery(this RouteHandlerBuilder builder, bool supportsFilter = true, bool supportsOrderBy = false) => builder.ThrowIfNull().WithMetadata(new QueryAttribute(supportsFilter, supportsOrderBy)); + /// + /// Adds metadata to the that the action/operation supports including . + /// + /// The . + /// Indicates whether is supported/enabled. + /// Indicates whether is supported/enabled. + /// Indicates whether is supported/enabled. + /// The to support fluent-style method-chaining. + public static RouteHandlerBuilder WithQuery(this RouteHandlerBuilder builder, bool supportsFilter, bool supportsOrderBy, bool supportsSearch) => builder.ThrowIfNull().WithMetadata(new QueryAttribute(supportsFilter, supportsOrderBy) { SupportsSearch = supportsSearch }); + /// /// Adds to for all endpoints produced by the defaulting the content type to . /// diff --git a/src/CoreEx.AspNetCore/Mvc/QueryAttribute.cs b/src/CoreEx.AspNetCore/Mvc/QueryAttribute.cs index bbfd798e7..15468494a 100644 --- a/src/CoreEx.AspNetCore/Mvc/QueryAttribute.cs +++ b/src/CoreEx.AspNetCore/Mvc/QueryAttribute.cs @@ -20,4 +20,10 @@ public sealed class QueryAttribute(bool supportsFilter = true, bool supportsOrde /// /// Defaults to . public bool SupportsOrderBy { get; } = supportsOrderBy; + + /// + /// Gets or sets whether the (free-text $search) is supported. + /// + /// Defaults to . + public bool SupportsSearch { get; set; } } \ No newline at end of file diff --git a/src/CoreEx.AspNetCore/OpenApiOptions.cs b/src/CoreEx.AspNetCore/OpenApiOptions.cs index 0616e7476..e646e1ce3 100644 --- a/src/CoreEx.AspNetCore/OpenApiOptions.cs +++ b/src/CoreEx.AspNetCore/OpenApiOptions.cs @@ -40,6 +40,11 @@ public class OpenApiOptions /// public string QueryOrderByText { get; set; } = "The OData-like query order-by expression."; + /// + /// Gets or sets the descriptive text. + /// + public string QuerySearchText { get; set; } = "The free-text search expression; interpretation (keyword, full-text, semantic) is endpoint-specific and where ranked results are ordered by relevance."; + /// /// Gets or sets the descriptive text. /// diff --git a/src/CoreEx.Data.GraphQL/GraphQLEngine.cs b/src/CoreEx.Data.GraphQL/GraphQLEngine.cs index f80b76a0c..fc5f159c5 100644 --- a/src/CoreEx.Data.GraphQL/GraphQLEngine.cs +++ b/src/CoreEx.Data.GraphQL/GraphQLEngine.cs @@ -364,6 +364,7 @@ private Task ExecuteItemRootAsync(GraphQLItemRoot root, GraphQLField field, stri KeyNotFoundException => NewError(ex.Message, [alias], "ARGUMENT_ERROR"), QueryFilterParserException => NewError(ex.Message, [alias], "FILTER_PARSE_ERROR"), QueryOrderByParserException => NewError(ex.Message, [alias], "ORDERBY_PARSE_ERROR"), + QuerySearchParserException => NewError(ex.Message, [alias], "SEARCH_PARSE_ERROR"), ValidationException vex => MapValidationException(vex, alias), NotFoundException => MapKnownExtendedException(ex, alias, "NOT_FOUND"), ConflictException => MapKnownExtendedException(ex, alias, "CONFLICT_ERROR"), diff --git a/src/CoreEx.Data.GraphQL/GraphQLQueryRoot.cs b/src/CoreEx.Data.GraphQL/GraphQLQueryRoot.cs index c4dddde70..0a42220ad 100644 --- a/src/CoreEx.Data.GraphQL/GraphQLQueryRoot.cs +++ b/src/CoreEx.Data.GraphQL/GraphQLQueryRoot.cs @@ -62,8 +62,8 @@ public async Task InvokeAsync(QueryArgs? queryArgs, PagingArgs? pa if (_options.EnableSensitiveDataLogging) logger.LogDebug("Invoking GraphQL query root '{Name}' with:\n QueryArgs: [{QueryArgs}]\n PagingArgs: [{PagingArgs}].", Name, queryArgs, pagingArgs); else - logger.LogDebug("Invoking GraphQL query root '{Name}' with: HasFilter={HasFilter}, HasOrderBy={HasOrderBy}, PagingArgs: [{PagingArgs}].", - Name, queryArgs?.Filter is not null, queryArgs?.OrderBy is not null, pagingArgs); + logger.LogDebug("Invoking GraphQL query root '{Name}' with: HasFilter={HasFilter}, HasOrderBy={HasOrderBy}, HasSearch={HasSearch}, PagingArgs: [{PagingArgs}].", + Name, queryArgs?.Filter is not null, queryArgs?.OrderBy is not null, queryArgs?.Search is not null, pagingArgs); } } diff --git a/src/CoreEx.Data.GraphQL/Internal/GraphQLArgsMapper.cs b/src/CoreEx.Data.GraphQL/Internal/GraphQLArgsMapper.cs index fdaac4ca3..4836bb192 100644 --- a/src/CoreEx.Data.GraphQL/Internal/GraphQLArgsMapper.cs +++ b/src/CoreEx.Data.GraphQL/Internal/GraphQLArgsMapper.cs @@ -1,7 +1,7 @@ namespace CoreEx.Data.GraphQL.Internal; /// -/// Maps the GraphQL-native where/orderBy/first/after/includeText/includeInactive root field arguments onto / +/// Maps the GraphQL-native where/orderBy/search/first/after/includeText/includeInactive root field arguments onto / /// , translating the structured where/orderBy input objects to the equivalent OData-esque filter/orderby strings consumed by /// the underlying (see /). /// @@ -19,7 +19,8 @@ public static QueryArgs BuildQueryArgs(IReadOnlyDictionary args var queryArgs = new QueryArgs { Filter = GraphQLFilterTranslator.Translate(args.TryGetValue("where", out var where) ? where : null, filterParser), - OrderBy = GraphQLOrderByTranslator.Translate(args.TryGetValue("orderBy", out var orderBy) ? orderBy : null) + OrderBy = GraphQLOrderByTranslator.Translate(args.TryGetValue("orderBy", out var orderBy) ? orderBy : null), + Search = args.GetString("search") }; if (args.GetBool("includeText") is true) @@ -47,8 +48,8 @@ public static QueryArgs BuildQueryArgs(IReadOnlyDictionary args /// Thrown where where/orderBy are specified; these are only applicable to list query roots. public static void ApplyItemRootFlags(IReadOnlyDictionary args) { - if (args.ContainsKey("where") || args.ContainsKey("orderBy")) - throw new GraphQLArgumentTranslationException("'where'/'orderBy' are not supported on single-item root fields; they are only applicable to list query roots."); + if (args.ContainsKey("where") || args.ContainsKey("orderBy") || args.ContainsKey("search")) + throw new GraphQLArgumentTranslationException("'where'/'orderBy'/'search' are not supported on single-item root fields; they are only applicable to list query roots."); if (args.GetBool("includeText") is true && ExecutionContext.HasCurrent && !ExecutionContext.Current.IncludeRelatedText) ExecutionContext.Current.IncludeRelatedText = true; diff --git a/src/CoreEx.Data.GraphQL/Internal/GraphQLIntrospectionSchemaBuilder.cs b/src/CoreEx.Data.GraphQL/Internal/GraphQLIntrospectionSchemaBuilder.cs index 1fa2ccaab..c286d2891 100644 --- a/src/CoreEx.Data.GraphQL/Internal/GraphQLIntrospectionSchemaBuilder.cs +++ b/src/CoreEx.Data.GraphQL/Internal/GraphQLIntrospectionSchemaBuilder.cs @@ -120,6 +120,9 @@ private static JsonObject BuildQueryRootField(GraphQLQueryRoot root, Dictionary< if (orderByTypeName is not null) args.Add(NewArg("orderBy", ListOf(NonNullOf(NamedRef(orderByTypeName, "INPUT_OBJECT"))))); + if (root.QueryArgsConfig.HasSearchParser) + args.Add(NewArg("search", NamedRef("String", "SCALAR"))); + args.Add(NewArg("includeText", NamedRef("Boolean", "SCALAR"))); args.Add(NewArg("includeInactive", NamedRef("Boolean", "SCALAR"))); diff --git a/src/CoreEx.Data.GraphQL/README.md b/src/CoreEx.Data.GraphQL/README.md index 89346c189..07cb6e512 100644 --- a/src/CoreEx.Data.GraphQL/README.md +++ b/src/CoreEx.Data.GraphQL/README.md @@ -38,7 +38,9 @@ The engine is deliberately **transport-agnostic**: it references only `CoreEx.Da `GraphQLOrderByTranslator`) onto the OData-esque `filter`/`orderby` strings; the translated string is always parsed/validated by the entity's own, unmodified `QueryArgsConfig` (`QueryFilterParser`/ `QueryOrderByParser`) — so whatever operators and fields a `QueryArgsConfig` already exposes for the REST - `$filter`/`$orderby` query strings are supported **exactly**, with no separate allow-list to maintain. + `$filter`/`$orderby` query strings are supported **exactly**, with no separate allow-list to maintain. Where the + `QueryArgsConfig` has a `SearchParser` configured, a free-text `search: String` argument is also advertised and + passed through as-is to `QueryArgs.Search` (the REST `$search` equivalent). - 🔗 **Relay Cursor Connections paging**: list query roots return the spec-shaped `edges { node cursor } pageInfo { hasNextPage hasPreviousPage startCursor endCursor } totalCount` response via `first`/`after` forward pagination (backward pagination — `last`/`before` — is out of scope for v1 and rejected with an @@ -47,7 +49,7 @@ The engine is deliberately **transport-agnostic**: it references only `CoreEx.Da object graph (e.g. `node { address { street city } }`) since projection is performed over one already-materialized result via `JsonFilter`, not via per-field resolvers. - 🧾 **GraphQL-shaped errors**: `GraphQLArgumentTranslationException`, `QueryFilterParserException`, - `QueryOrderByParserException`, `ValidationException`, `NotFoundException`, and unknown-field errors are + `QueryOrderByParserException`, `QuerySearchParserException`, `ValidationException`, `NotFoundException`, and unknown-field errors are mapped to `{ message, path, extensions.code }` error objects. - 🔍 **Spec-compliant introspection**: `__schema`/`__type(name:)` (plus `__typename`) implement the official GraphQL introspection schema, built once from the registered roots (see `Internal.GraphQLIntrospectionSchemaBuilder`) diff --git a/src/CoreEx.Data/Querying/QueryArgsConfig.cs b/src/CoreEx.Data/Querying/QueryArgsConfig.cs index 6cb43badd..b71da125a 100644 --- a/src/CoreEx.Data/Querying/QueryArgsConfig.cs +++ b/src/CoreEx.Data/Querying/QueryArgsConfig.cs @@ -7,6 +7,7 @@ public class QueryArgsConfig { private QueryFilterParser? _filterParser; private QueryOrderByParser? _orderByParser; + private QuerySearchParser? _searchParser; /// /// Creates a new . @@ -39,6 +40,16 @@ public class QueryArgsConfig /// public bool HasOrderByParser => _orderByParser is not null; + /// + /// Gets the . + /// + public QuerySearchParser SearchParser => _searchParser ??= new QuerySearchParser(this); + + /// + /// Indicates whether there is a . + /// + public bool HasSearchParser => _searchParser is not null; + /// /// Enables fluent-style method-chaining configuration for the . /// @@ -62,10 +73,23 @@ public QueryArgsConfig WithOrderBy(Action orderBy) } /// - /// Parses and converts the and to dynamic LINQ. + /// Enables fluent-style method-chaining configuration for the . + /// + /// The . + /// The instance to support fluent-style method-chaining. + public QueryArgsConfig WithSearch(Action search) + { + search.ThrowIfNull()(SearchParser); + return this; + } + + /// + /// Parses and converts the , and to dynamic LINQ. /// /// The . /// The . + /// Where the is specified and the is ranked, then an explicit is not supported, + /// and no default ordering is applied, as the results are expected to be ordered by relevance. public QueryArgsParseResult Parse(QueryArgs? queryArgs) { var result = new QueryArgsParseResult(this); @@ -85,8 +109,29 @@ public QueryArgsParseResult Parse(QueryArgs? queryArgs) return result.Adjust(x => x.Error = new QueryFilterParserException("Filter statement is not currently supported.")); } + QuerySearchParserResult? searchParserResult = null; + if (!string.IsNullOrWhiteSpace(queryArgs.Search)) + { + if (HasSearchParser) + { + searchParserResult = SearchParser.Parse(queryArgs.Search); + if (searchParserResult.HasError) + return result.Adjust(x => x.Error = searchParserResult.Error); + + if (!searchParserResult.HasText) + searchParserResult = null; + } + else + return result.Adjust(x => x.Error = new QuerySearchParserException("Search statement is not currently supported.")); + } + QueryOrderByParserResult? orderByParserResult = null; - if (HasOrderByParser) + if (searchParserResult is not null && searchParserResult.IsRanked) + { + if (!string.IsNullOrEmpty(queryArgs.OrderBy)) + return result.Adjust(x => x.Error = new QueryOrderByParserException("OrderBy statement is not supported in combination with a search; results are ordered by relevance.")); + } + else if (HasOrderByParser) { orderByParserResult = OrderByParser.Parse(queryArgs.OrderBy); if (orderByParserResult.HasError) @@ -94,8 +139,8 @@ public QueryArgsParseResult Parse(QueryArgs? queryArgs) } else if (!string.IsNullOrEmpty(queryArgs.OrderBy)) return result.Adjust(x => x.Error = new QueryOrderByParserException("OrderBy statement is not currently supported.")); - - return new QueryArgsParseResult(this, filterParserResult, orderByParserResult); + + return new QueryArgsParseResult(this, filterParserResult, orderByParserResult, searchParserResult); } /// @@ -132,6 +177,9 @@ public JsonElement ToJsonSchema() if (HasOrderByParser) dict["orderby"] = OrderByParser.ToJsonSchema(); + if (HasSearchParser) + dict["search"] = SearchParser.ToJsonSchema(); + return JsonSerializer.SerializeToElement(dict); } } diff --git a/src/CoreEx.Data/Querying/QueryArgsParseResult.cs b/src/CoreEx.Data/Querying/QueryArgsParseResult.cs index 55e831d61..6b3abfce4 100644 --- a/src/CoreEx.Data/Querying/QueryArgsParseResult.cs +++ b/src/CoreEx.Data/Querying/QueryArgsParseResult.cs @@ -11,11 +11,13 @@ public sealed class QueryArgsParseResult : IQueryParseError, IToResult /// The owning . /// The . /// The . - internal QueryArgsParseResult(QueryArgsConfig config, QueryFilterParserResult? filterResult = null, QueryOrderByParserResult? orderByResult = null) + /// The . + internal QueryArgsParseResult(QueryArgsConfig config, QueryFilterParserResult? filterResult = null, QueryOrderByParserResult? orderByResult = null, QuerySearchParserResult? searchResult = null) { Config = config; FilterResult = filterResult; OrderByResult = orderByResult; + SearchResult = searchResult; } /// @@ -33,6 +35,17 @@ internal QueryArgsParseResult(QueryArgsConfig config, QueryFilterParserResult? f /// public QueryOrderByParserResult? OrderByResult { get; } + /// + /// Gets the ; where no search text was specified. + /// + public QuerySearchParserResult? SearchResult { get; } + + /// + /// Indicates whether a ranked search was specified; as such, the repository is responsible for executing the search (using the ) + /// and ordering the results by relevance. + /// + public bool IsRankedSearch => SearchResult is not null && SearchResult.IsRanked; + /// public bool HasError => Error is not null; diff --git a/src/CoreEx.Data/Querying/QueryExtensions.cs b/src/CoreEx.Data/Querying/QueryExtensions.cs index 079dc7c16..8881651a8 100644 --- a/src/CoreEx.Data/Querying/QueryExtensions.cs +++ b/src/CoreEx.Data/Querying/QueryExtensions.cs @@ -6,14 +6,48 @@ namespace CoreEx.Data.Querying; public static class QueryExtensions { /// - /// Adds a dynamic query (basic dynamic OData-like $filter statement). + /// Adds a dynamic query (basic dynamic OData-like $filter statement) and (free-text $search statement). /// /// The element . /// The sequence of elements. /// The . /// The . /// The query. - public static IQueryable Where(this IQueryable source, QueryArgsConfig queryConfig, QueryArgs? queryArgs = null) => Where(source, queryConfig, queryArgs?.Filter); + /// A ranked search is not applied; this is the responsibility of the repository. + public static IQueryable Where(this IQueryable source, QueryArgsConfig queryConfig, QueryArgs? queryArgs = null) => WhereSearch(Where(source, queryConfig, queryArgs?.Filter), queryConfig, queryArgs?.Search); + + /// + /// Adds a dynamic query (free-text $search statement). + /// + /// The element . + /// The sequence of elements. + /// The . + /// The free-text $search statement. + /// The query. + /// A ranked search is not applied; this is the responsibility of the repository. + public static IQueryable WhereSearch(this IQueryable source, QueryArgsConfig queryConfig, string? search) + { + queryConfig.ThrowIfNull(); + if (string.IsNullOrWhiteSpace(search)) + return source; + + if (!queryConfig.HasSearchParser) + throw new QuerySearchParserException("Search statement is not currently supported."); + + return WhereSearch(source, queryConfig, queryConfig.SearchParser.Parse(search).ThrowOnError()); + } + + /// + /// Applies the search predicate (where applicable). + /// + private static IQueryable WhereSearch(IQueryable source, QueryArgsConfig queryConfig, QuerySearchParserResult? result) + { + if (result is null) + return source; + + var linq = result.ToLinqString(out var args); + return string.IsNullOrEmpty(linq) ? source : source.Where(queryConfig.ParsingConfig, linq, [.. args]); + } /// /// Adds a dynamic query (basic dynamic OData-like $filter statement). @@ -38,20 +72,24 @@ public static IQueryable Where(this IQueryable source } /// - /// Adds a dynamic query filter (basic dynamic OData-like $filter statement). + /// Adds a dynamic query filter (basic dynamic OData-like $filter statement) and search (free-text $search statement). /// /// The element . /// The sequence of elements. - /// The that contains the parsed dynamic OData-like $filter statement. + /// The that contains the parsed dynamic OData-like $filter and $search statements. /// The query. + /// A ranked search is not applied; this is the responsibility of the repository. public static IQueryable Where(this IQueryable source, QueryArgsParseResult result) { result.ThrowIfNull().ThrowOnError(); - if (result.FilterResult is null) - return source; + if (result.FilterResult is not null) + { + var linq = result.FilterResult.ToLinqString(out var args); + if (!string.IsNullOrEmpty(linq)) + source = source.Where(result.Config.ParsingConfig, linq, [.. args]); + } - var linq = result.FilterResult.ToLinqString(out var args); - return string.IsNullOrEmpty(linq) ? source : source.Where(result.Config.ParsingConfig, linq, [.. args]); + return WhereSearch(source, result.Config, result.SearchResult); } /// diff --git a/src/CoreEx.Data/Querying/QuerySearchFieldConfig.cs b/src/CoreEx.Data/Querying/QuerySearchFieldConfig.cs new file mode 100644 index 000000000..396fe086b --- /dev/null +++ b/src/CoreEx.Data/Querying/QuerySearchFieldConfig.cs @@ -0,0 +1,144 @@ +namespace CoreEx.Data.Querying; + +/// +/// Provides the field configuration. +/// +/// The owning . +/// The model name to be used for the dynamic LINQ expression. +public sealed class QuerySearchFieldConfig(QuerySearchParser parser, string model) +{ + /// + /// Gets the owning . + /// + public QuerySearchParser Parser { get; } = parser.ThrowIfNull(); + + /// + /// Gets the model name to be used for the dynamic LINQ expression. + /// + /// The model must be of . + public string Model { get; } = model.ThrowIfNullOrEmpty(nameof(model)); + + /// + /// Gets the optional prefix to be used where referencing the underlying model. + /// + /// This will default from when instantiated. + public string? ModelPrefix { get; private set; } = parser.DefaultModelPrefix; + + /// + /// Gets the fully-qualified name (including any where specified). + /// + public string FullyQualifiedModelName => (ModelPrefix is null ? string.Empty : ModelPrefix + ".") + Model; + + /// + /// Gets the used to match a search term against the field. + /// + /// Defaults to . + public QueryFilterOperator Operator { get; private set; } = QueryFilterOperator.Contains; + + /// + /// Indicates whether the field and search term are to be converted to upper case before comparison. + /// + public bool IsToUpper { get; private set; } + + /// + /// Indicates whether the field and search term are to be converted to lower case before comparison. + /// + public bool IsToLower { get; private set; } + + /// + /// Indicates whether a != null check is to be included before the comparison. + /// + /// Defaults to . + public bool IsCheckNotNull { get; private set; } = true; + + /// + /// Sets (overrides) the . + /// + /// The ; must be one of , , or . + /// The to support fluent-style method-chaining. + public QuerySearchFieldConfig WithOperator(QueryFilterOperator @operator) + { + Operator = @operator.ThrowWhen(o => o is not (QueryFilterOperator.Contains or QueryFilterOperator.StartsWith or QueryFilterOperator.EndsWith or QueryFilterOperator.Equal), + $"Operator must be one of {QueryFilterOperator.Contains}, {QueryFilterOperator.StartsWith}, {QueryFilterOperator.EndsWith} or {QueryFilterOperator.Equal}."); + + return this; + } + + /// + /// Indicates that the field and search term are to be converted to upper case before comparison (see ). + /// + /// The to support fluent-style method-chaining. + public QuerySearchFieldConfig AsUpperCase() + { + IsToUpper = true; + IsToLower = false; + return this; + } + + /// + /// Indicates that the field and search term are to be converted to lower case before comparison (see ). + /// + /// The to support fluent-style method-chaining. + public QuerySearchFieldConfig AsLowerCase() + { + IsToLower = true; + IsToUpper = false; + return this; + } + + /// + /// Indicates that the != null check is to be excluded (see ). + /// + /// The to support fluent-style method-chaining. + /// Only use where the underlying field is guaranteed to be non-, or the data source handles comparisons natively. + public QuerySearchFieldConfig WithNoNullCheck() + { + IsCheckNotNull = false; + return this; + } + + /// + /// Sets (overrides) the optional to be used where referencing the underlying model. + /// + /// The model prefix. + /// The to support fluent-style method-chaining. + public QuerySearchFieldConfig WithModelPrefix(string modelPrefix) + { + ModelPrefix = modelPrefix.ThrowIfNullOrEmpty(); + return this; + } + + /// + /// Clears (overrides) the optional to be used where referencing the underlying model. + /// + /// The to support fluent-style method-chaining. + public QuerySearchFieldConfig WithNoModelPrefix() + { + ModelPrefix = null; + return this; + } + + /// + /// Writes the dynamic LINQ comparison for the specified . + /// + internal void WriteAsDynamicLinq(StringBuilder sb, List args, string term) + { + var field = FullyQualifiedModelName + (IsToUpper ? ".ToUpper()" : IsToLower ? ".ToLower()" : string.Empty); + var placeholder = $"@{args.Count}"; + args.Add(IsToUpper ? term.ToUpperInvariant() : IsToLower ? term.ToLowerInvariant() : term); + + if (IsCheckNotNull) + sb.Append('(').Append(FullyQualifiedModelName).Append(" != null && "); + + sb.Append(Operator switch + { + QueryFilterOperator.StartsWith => $"{field}.StartsWith({placeholder})", + QueryFilterOperator.EndsWith => $"{field}.EndsWith({placeholder})", + QueryFilterOperator.Equal => $"{field} == {placeholder}", + _ => $"{field}.Contains({placeholder})" + }); + + if (IsCheckNotNull) + sb.Append(')'); + } +} diff --git a/src/CoreEx.Data/Querying/QuerySearchMode.cs b/src/CoreEx.Data/Querying/QuerySearchMode.cs new file mode 100644 index 000000000..b24f6471d --- /dev/null +++ b/src/CoreEx.Data/Querying/QuerySearchMode.cs @@ -0,0 +1,30 @@ +namespace CoreEx.Data.Querying; + +/// +/// Defines the search mode; i.e. how the free-text $search is to be interpreted and executed. +/// +/// Only is executed by CoreEx itself (as a provider-agnostic dynamic LINQ predicate). All other modes are ranked and are executed by the +/// repository using the underlying data source capabilities (e.g. PostgreSQL tsvector/pgvector, SQL Server full-text, Azure Cosmos DB VectorDistance/FullTextScore, Azure AI Search). +/// Where ranked, results are ordered by relevance and an explicit $orderby is not supported. +public enum QuerySearchMode +{ + /// + /// Indicates a keyword search: each term must match (AND) at least one of the configured fields (OR); executed by CoreEx as a dynamic LINQ predicate (unranked). + /// + Keyword = 0, + + /// + /// Indicates a lexical full-text search (e.g. BM25); executed by the repository (ranked). + /// + FullText = 1, + + /// + /// Indicates a semantic (vector/embedding similarity) search; executed by the repository (ranked). + /// + Semantic = 2, + + /// + /// Indicates a hybrid search that fuses lexical and semantic results (e.g. reciprocal rank fusion); executed by the repository (ranked). + /// + Hybrid = 3 +} diff --git a/src/CoreEx.Data/Querying/QuerySearchParser.cs b/src/CoreEx.Data/Querying/QuerySearchParser.cs new file mode 100644 index 000000000..130bad116 --- /dev/null +++ b/src/CoreEx.Data/Querying/QuerySearchParser.cs @@ -0,0 +1,230 @@ +namespace CoreEx.Data.Querying; + +/// +/// Represents a basic free-text query search ($search) parser with explicitly defined behavior. +/// +/// The search text is opaque; i.e. no grammar (boolean operators, phrases, etc.) is defined or interpreted. The parser normalizes and validates the text (length, terms), and where the +/// is provides a provider-agnostic dynamic LINQ predicate across the configured keyword fields. +/// All other modes are ranked and are not executed by CoreEx; the repository is responsible for executing using the +/// (e.g. generating an embedding and performing a vector similarity search), with results ordered by relevance. +/// This is not intended to be a replacement for a dedicated search engine; use $filter for precise, structured filtering. +/// The owning . +public sealed class QuerySearchParser(QueryArgsConfig owner) +{ + private readonly List _fields = []; + private Action? _validator; + private string? _helpText; + + /// + /// Gets the default maximum search text length. + /// + public const int DefaultMaxLength = 200; + + /// + /// Gets the default maximum number of search terms. + /// + public const int DefaultMaxTerms = 8; + + /// + /// Gets the owning . + /// + public QueryArgsConfig Owner => owner.ThrowIfNull(); + + /// + /// Gets the . + /// + /// Defaults to . + public QuerySearchMode Mode { get; private set; } = QuerySearchMode.Keyword; + + /// + /// Indicates whether the search is ranked (i.e. is not ) and is to be executed by the repository with results ordered by relevance. + /// + public bool IsRanked => Mode != QuerySearchMode.Keyword; + + /// + /// Gets the maximum (normalized) search text length. + /// + /// Defaults to . + public int MaxLength { get; private set; } = DefaultMaxLength; + + /// + /// Gets the maximum number of distinct search terms; only enforced where the is . + /// + /// Defaults to . Each term results in an additional predicate per keyword field, so this limits the resulting query complexity. + public int MaxTerms { get; private set; } = DefaultMaxTerms; + + /// + /// Gets the default model prefix (if any). + /// + /// This will be automatically applied to all subsequent field additions; for example . + public string? DefaultModelPrefix { get; private set; } + + /// + /// Gets the configured fields. + /// + public IReadOnlyList KeywordFields => _fields; + + /// + /// Sets (overrides) the . + /// + /// The . + /// The to support fluent-style method-chaining. + public QuerySearchParser WithMode(QuerySearchMode mode) + { + Mode = mode; + return this; + } + + /// + /// Sets (overrides) the . + /// + /// The maximum (normalized) search text length. + /// The to support fluent-style method-chaining. + public QuerySearchParser WithMaxLength(int maxLength) + { + MaxLength = maxLength.ThrowWhen(v => v <= 0, "Maximum length must be greater than zero."); + return this; + } + + /// + /// Sets (overrides) the . + /// + /// The maximum number of distinct search terms. + /// The to support fluent-style method-chaining. + public QuerySearchParser WithMaxTerms(int maxTerms) + { + MaxTerms = maxTerms.ThrowWhen(v => v <= 0, "Maximum terms must be greater than zero."); + return this; + } + + /// + /// Sets (overrides) the (if any). + /// + /// The model prefix. + /// The to support fluent-style method-chaining. + public QuerySearchParser WithDefaultModelPrefix(string? modelPrefix = null) + { + DefaultModelPrefix = modelPrefix; + return this; + } + + /// + /// Adds a for the specified . + /// + /// The model name (must be of ) to be used for the dynamic LINQ expression. + /// The optional action enabling further field configuration. + /// The to support fluent-style method-chaining. + /// The model names are not exposed externally (e.g. via ). + public QuerySearchParser AddKeywordField(string model, Action? configure = null) + { + var config = new QuerySearchFieldConfig(this, model); + configure?.Invoke(config); + config.FullyQualifiedModelName.ThrowWhen(name => _fields.Any(f => f.FullyQualifiedModelName == name), $"The search keyword field '{config.FullyQualifiedModelName}' has already been added and must be unique."); + _fields.Add(config); + return this; + } + + /// + /// Sets (override) the additional help text. + /// + /// The additional help text. + /// The to support fluent-style method-chaining. + public QuerySearchParser WithHelpText(string text) + { + _helpText = text; + return this; + } + + /// + /// Sets (overrides) a that can be used to further validate the parsed search. + /// + /// The validator action. + /// The to support fluent-style method-chaining. + /// Throw a to have the validation message formatted correctly and consistently. + public QuerySearchParser WithValidator(Action? validator) + { + _validator = validator; + return this; + } + + /// + /// Parses (normalizes and validates) the free-text . + /// + /// The free-text $search statement. + /// The . + public QuerySearchParserResult Parse(string? search) + { + if (Mode == QuerySearchMode.Keyword && _fields.Count == 0) + throw new InvalidOperationException($"A {nameof(QuerySearchMode.Keyword)} search requires at least one field to be configured; see {nameof(AddKeywordField)}."); + + var terms = string.IsNullOrWhiteSpace(search) ? [] : search.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (terms.Length == 0) + return new QuerySearchParserResult(this, null, []); + + var text = string.Join(' ', terms); + + try + { + if (text.Length > MaxLength) + throw new QuerySearchParserException($"Statement must not exceed {MaxLength} characters in length."); + + var distinct = terms.Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + if (Mode == QuerySearchMode.Keyword && distinct.Length > MaxTerms) + throw new QuerySearchParserException($"Statement must not contain more than {MaxTerms} distinct terms."); + + var result = new QuerySearchParserResult(this, text, distinct); + _validator?.Invoke(result); + return result; + } + catch (QuerySearchParserException qspex) + { + qspex.WithExtension("schema", owner.ToJsonSchema()); + return new QuerySearchParserResult(qspex); + } + } + + /// + public override string ToString() + { + var sb = new StringBuilder(); + sb.Append("Search is free-text (no operators or phrases); "); + sb.Append(Mode switch + { + QuerySearchMode.FullText => "full-text search with results ordered by relevance (order-by is not supported).", + QuerySearchMode.Semantic => "semantic (meaning-based) search with results ordered by relevance (order-by is not supported).", + QuerySearchMode.Hybrid => "hybrid (full-text and semantic) search with results ordered by relevance (order-by is not supported).", + _ => "keyword search where each term must match." + }); + + sb.AppendLine().Append("Maximum length: ").Append(MaxLength); + if (Mode == QuerySearchMode.Keyword) + sb.AppendLine().Append("Maximum terms: ").Append(MaxTerms); + + if (!string.IsNullOrEmpty(_helpText)) + sb.AppendLine().AppendLine("---").Append("Note: ").Append(_helpText); + + return sb.ToString(); + } + + /// + /// Produces the JSON schema for the configuration. + /// + /// The . + public JsonElement ToJsonSchema() + { + var root = new Dictionary + { + { "mode", Mode.ToString().ToLowerInvariant() }, + { "ranked", IsRanked }, + { "maxLength", MaxLength } + }; + + if (Mode == QuerySearchMode.Keyword) + root["maxTerms"] = MaxTerms; + + if (!string.IsNullOrEmpty(_helpText)) + root["description"] = _helpText; + + return JsonSerializer.SerializeToElement(root); + } +} diff --git a/src/CoreEx.Data/Querying/QuerySearchParserException.cs b/src/CoreEx.Data/Querying/QuerySearchParserException.cs new file mode 100644 index 000000000..27981e543 --- /dev/null +++ b/src/CoreEx.Data/Querying/QuerySearchParserException.cs @@ -0,0 +1,13 @@ +namespace CoreEx.Data.Querying; + +/// +/// Represents a . +/// +/// The error message. +public sealed class QuerySearchParserException(string message) : ValidationException(new MessageItem(MessageType.Error, message, HttpNames.QuerySearchQueryStringName), FallbackMessage) +{ + /// + /// Gets the default/fallback + /// + internal const string FallbackMessage = "A query search parsing error occurred."; +} diff --git a/src/CoreEx.Data/Querying/QuerySearchParserResult.cs b/src/CoreEx.Data/Querying/QuerySearchParserResult.cs new file mode 100644 index 000000000..468c9cc60 --- /dev/null +++ b/src/CoreEx.Data/Querying/QuerySearchParserResult.cs @@ -0,0 +1,112 @@ +namespace CoreEx.Data.Querying; + +/// +/// Represents the result of . +/// +public sealed class QuerySearchParserResult : IQueryParseError +{ + private readonly QuerySearchParser? _parser; + + /// + /// Initializes a new instance of the class. + /// + /// The owning . + /// The normalized search text. + /// The distinct search terms. + internal QuerySearchParserResult(QuerySearchParser parser, string? text, string[] terms) + { + _parser = parser.ThrowIfNull(); + Mode = parser.Mode; + Text = text; + Terms = terms; + } + + /// + /// Initializes a new instance of the class with an error. + /// + /// The error . + internal QuerySearchParserResult(QuerySearchParserException error) => Error = error.ThrowIfNull(); + + /// + /// Gets the . + /// + public QuerySearchMode Mode { get; } + + /// + /// Indicates whether the search is ranked (i.e. is not ) and is to be executed by the repository with results ordered by relevance. + /// + public bool IsRanked => Mode != QuerySearchMode.Keyword; + + /// + /// Gets the normalized search text (trimmed with internal white space collapsed); where not specified. + /// + /// This is the value that should be passed to a ranked search capability (e.g. used to generate an embedding for a semantic search). + public string? Text { get; } + + /// + /// Indicates whether there is search . + /// + public bool HasText => !string.IsNullOrEmpty(Text); + + /// + /// Gets the distinct (case-insensitive) white-space separated search terms. + /// + public string[] Terms { get; } = []; + + /// + public bool HasError => Error is not null; + + /// + ExtendedException? IQueryParseError.Error => Error; + + /// + /// Gets the error represented as an that occurred during parsing, if any. + /// + public QuerySearchParserException? Error { get; } + + /// + /// Throws the where ; otherwise, does nothing. + /// + public QuerySearchParserResult ThrowOnError() => HasError ? throw Error! : this; + + /// + /// Provides the resulting dynamic LINQ predicate. + /// + /// The resulting dynamic LINQ arguments. + /// The dynamic LINQ predicate; where there is no or the search (as must be executed by the repository). + /// Each term must match (AND) at least one of the configured fields (OR). + public string? ToLinqString(out object?[] args) + { + ThrowOnError(); + args = []; + if (_parser is null || IsRanked || Terms.Length == 0) + return null; + + var sb = new StringBuilder(); + var list = new List(); + foreach (var term in Terms) + { + if (sb.Length > 0) + sb.Append(" && "); + + sb.Append('('); + var first = true; + foreach (var field in _parser.KeywordFields) + { + if (!first) + sb.Append(" || "); + + field.WriteAsDynamicLinq(sb, list, term); + first = false; + } + + sb.Append(')'); + } + + args = [.. list]; + return sb.ToString(); + } + + /// + public override string ToString() => Text ?? string.Empty; +} diff --git a/src/CoreEx.Data/Querying/README.md b/src/CoreEx.Data/Querying/README.md index f206defe1..8d3a39b48 100644 --- a/src/CoreEx.Data/Querying/README.md +++ b/src/CoreEx.Data/Querying/README.md @@ -1,12 +1,12 @@ # CoreEx.Data.Querying -> Provides the safe, explicitly-configured OData-like `$filter` and `$orderby` LINQ query translation pipeline: `QueryArgsConfig`, `QueryFilterParser`, `QueryOrderByParser`, field configurations, expression AST types, and the `IQueryable.Where` and `OrderBy` extensions. +> Provides the safe, explicitly-configured OData-like `$filter`, `$orderby` and `$search` LINQ query translation pipeline: `QueryArgsConfig`, `QueryFilterParser`, `QueryOrderByParser`, `QuerySearchParser`, field configurations, expression AST types, and the `IQueryable.Where` and `OrderBy` extensions. ## Overview -`CoreEx.Data.Querying` implements a constrained, secure dynamic query layer. Unlike full OData or GraphQL, it does not expose arbitrary model properties: every filterable and sortable field must be explicitly registered in a `QueryArgsConfig` instance. This design prevents accidental data leakage, index misuse, and injection-style attacks while still delivering flexible client-driven queries via the `$filter` and `$orderby` query-string arguments from `QueryArgs`. +`CoreEx.Data.Querying` implements a constrained dynamic query layer. It does not expose arbitrary model properties: every filterable and sortable field must be explicitly registered in a `QueryArgsConfig` instance. This allows the data owner to control the query surface while delivering flexible client-driven queries via the `$filter`, `$orderby` and `$search` query-string arguments from `QueryArgs`. Allow-listing and parameterized predicates are guardrails, not substitutes for authorization, appropriate indexes or query-cost controls. -The parse pipeline is: `QueryArgs.Filter` string → `QueryFilterParser.Parse()` → expression token AST → LINQ `QueryStatement` (a `System.Linq.Dynamic.Core`-compatible predicate string + parameter array). `QueryExtensions.Where` and `OrderBy` drives both parsers, validates errors, and applies the resulting predicates to the `IQueryable`. +The filter pipeline is: `QueryArgs.Filter` string → `QueryFilterParser.Parse()` → expression token AST → LINQ `QueryStatement` (a `System.Linq.Dynamic.Core`-compatible predicate string + parameter array). `QueryArgsConfig.Parse()` combines filter, search and order-by validation into a `QueryArgsParseResult`. The `QueryExtensions.Where` and `OrderBy` extensions apply its filter, keyword search and ordering to the `IQueryable`; ranked search is executed by the repository. Field configurations (`QueryFilterFieldConfig`, `QueryFilterParseableFieldConfig`, `QueryFilterNullFieldConfig`, `QueryFilterReferenceDataFieldConfig`, `QueryFilterEnumFieldConfig`) map each registered field name to its underlying model property, allowed operators, value transformations, and custom statement generators. @@ -15,22 +15,121 @@ Field configurations (`QueryFilterFieldConfig`, `QueryFilterParseableFieldCon - ⚙️ **Explicit field allow-listing**: Only fields registered via `QueryFilterParser.AddField` or `QueryOrderByParser.AddField` are accepted; all others produce a structured parse error (`IQueryParseError`) rather than a runtime exception. - 🔍 **Filter operator support**: String fields support `eq`, `ne`, `startswith`, `endswith`, `contains`, `in`; numeric/date fields support `eq`, `ne`, `lt`, `le`, `gt`, `ge`, `in`; null fields support `eq null` / `ne null`. - 📐 **Order-by support**: `$orderby` accepts comma-separated `field asc|desc` pairs; each field's model name and direction are validated and translated to LINQ `.OrderBy()`/`.ThenBy()`. -- 🔠 **Value normalization**: `UseUpperCase()` / `UseLowerCase()` on string fields apply `ToUpper`/`ToLower` transforms to both the filter value and the model field in the LINQ predicate, enabling case-insensitive matching at the data layer. +- 🔠 **Value normalization**: `AsUpperCase()` / `AsLowerCase()` on string fields apply `ToUpper`/`ToLower` transforms to both the filter value and the model field in the LINQ predicate, enabling case-insensitive matching at the data layer. - 🏷️ **Reference data fields**: `AddReferenceDataField` resolves a `Code` string in the filter to the reference data `Id`, so callers filter by code while the query runs against the numeric/GUID identifier column. - 🔢 **Enum fields**: `AddEnumField` parses enum name strings to their underlying numeric value for persistence queries. - 🔲 **Null fields**: `AddNullField` maps an abstract nullable concept (e.g. `terminated`) to a model property null-check expression, hiding physical schema detail from the caller. - 🛑 **Structured parse errors**: `IQueryParseError` carries the error message and field name; `QueryFilterParserException` / `QueryOrderByParserException` are thrown when parsing fails, translating cleanly to `400 Bad Request` in `WebApi`. - 🔗 **Dynamic LINQ integration**: Generated `QueryStatement` objects are applied via `System.Linq.Dynamic.Core` `.Where(predicate, args)` and `.OrderBy(statement)`, keeping the translation generic across EF Core, Cosmos, and in-memory queryables, etc. +- 🔎 **Free-text search (`$search`)**: `WithSearch(Action)` enables the OData-style `$search` argument (`QueryArgs.Search`). The text is opaque (no grammar); it is normalized and validated (`MaxLength`, `MaxTerms`). See [Free-text search](#free-text-search-search). + +## Structured filtering (`$filter`) + +`$filter` answers **"which records satisfy these conditions?"** It is an explicit boolean expression, not a search phrase. For example (shown without URL encoding): + +```text +GET /api/products?$filter=startswith(sku, 'SPEC') and contains(text, 'Comp') +``` + +Enable filtering with `WithFilter` and register each public field, its type and its permitted operators: + +```csharp +var config = QueryArgsConfig.Create() + .WithFilter(filter => filter + .AddField("sku", nameof(Product.Sku), c => c + .WithOperators(QueryFilterOperator.EqualityOperators | QueryFilterOperator.StartsWith) + .AsUpperCase()) + .AddField("text", nameof(Product.Text), c => c + .WithOperators(QueryFilterOperator.StringFunctions) + .AsUpperCase())); +``` + +Public names can differ from model properties; `WithDefaultModelPrefix` supports queries over nested projections. The registered type determines how constants are parsed, and `WithOperators` restricts what callers can do with that field. Expressions support `and`, `or` and parentheses, comparisons such as `eq`/`ne`, and configured string functions such as `startswith`, `contains` and `endswith`. String literals use single quotes; an embedded quote is escaped by doubling it (`'O''Brien'`). Reference-data and enum field configurations translate public values into their persistence representation. + +The parser validates the expression and emits a Dynamic LINQ predicate with separate parameter values; client text is not executed as arbitrary LINQ. Unknown fields, unsupported operators and invalid values produce `QueryFilterParserException`, surfaced as `400 Bad Request` by `WebApi`. A filter supplied without a configured filter parser is also rejected. `ToJsonSchema()` describes the allowed fields and operators. + +Filtering and search are complementary: when both are supplied, records must satisfy the filter **and** the search. The repository remains responsible for applying mandatory authorization and tenant restrictions independently of optional client criteria. + +## Ordering (`$orderby`) + +`$orderby` answers **"in what sequence should matching records be returned?"** It accepts comma-separated fields with an optional `asc` or `desc` direction; omitting the direction means ascending: + +```text +GET /api/products?$filter=startswith(sku, 'SPEC')&$orderby=text desc, sku +``` + +Enable ordering with `WithOrderBy` and register the sortable fields: + +```csharp +var config = QueryArgsConfig.Create() + .WithOrderBy(orderby => orderby + .AddField("sku", nameof(Product.Sku), c => c.WithDefault().WithAlwaysInclude()) + .AddField("text", nameof(Product.Text))); +``` + +`WithDefault` defines the ordering when the client supplies none; multiple default fields are applied in registration order. `WithAlwaysInclude` appends a field when it is missing from the chosen ordering. Using a unique field as the final tie-breaker gives deterministic ordering for paging (but does not prevent page drift when records change between requests). `WithDirection` can restrict a field to ascending or descending ordering, and model names/prefixes can be mapped independently of public field names. + +Unknown fields and disallowed directions produce `QueryOrderByParserException` (`400` in `WebApi`); an explicit order-by without a configured parser is also rejected. `ToJsonSchema()` describes the allowed fields, directions and default ordering. + +For non-ranked queries, parse once and apply the validated criteria before materializing or paging: + +```csharp +var parsed = config.Parse(queryArgs).ThrowOnError(); +var ordered = products.Where(parsed).OrderBy(parsed); +``` + +Keyword search can use client/default ordering. Ranked search instead owns relevance ordering: an explicit `$orderby` is rejected and configured default/always-included ordering is not applied. + +## Free-text search (`$search`) + +`$search` is deliberately separate from `$filter`. `$filter` is a precise boolean predicate; `$search` is a *"find me things like this"* request whose execution strategy belongs to the data owner. Clients always send **text**, never vectors; embedding the text (where applicable) is a server-side Infrastructure concern. + +The `QuerySearchMode` determines who executes the search: + +| Mode | Executed by | Ordering | +|------|-------------|----------| +| `Keyword` (default) | The framework: `QueryExtensions.Where` applies a provider-agnostic Dynamic LINQ predicate where **every term** must match **at least one** configured keyword field (`AddKeywordField`). | `$orderby` (or default) as per usual. | +| `FullText`, `Semantic`, `Hybrid` | The repository: ranked modes are *never* applied by `Where`; the repository checks `QueryArgsParseResult.IsRankedSearch` and uses `SearchResult.Text` (e.g. PostgreSQL `tsvector`, pgvector `<=>`, Cosmos `VectorDistance`, reciprocal rank fusion). | Relevance; an explicit `$orderby` is rejected, and the default order-by is not applied. | + +```csharp +public class ProductQueryArgsConfig : QueryArgsConfig +{ + public ProductQueryArgsConfig() + { + WithFilter(...); + WithOrderBy(...); + WithSearch(search => search + .WithDefaultModelPrefix("Product") + .AddKeywordField(nameof(Product.Text), c => c.AsUpperCase()) + .AddKeywordField(nameof(Product.Sku), c => c.WithOperator(QueryFilterOperator.StartsWith).AsUpperCase())); + } +} + +// Repository: keyword search is applied alongside the filter. +var parsed = ProductQueryArgsConfig.Default.Parse(query).ThrowOnError(); +var q = products.Where(parsed); +q = parsed.IsRankedSearch ? /* rank by relevance using parsed.SearchResult!.Text */ q : q.OrderBy(parsed); +``` + +A `$search` against a `QueryArgsConfig` without a configured search parser is rejected (`400`) with `QuerySearchParserException`. The `ToJsonSchema` output includes a `search` entry (`mode`, `ranked`, `maxLength`, `maxTerms`) without exposing model names. Clients send it via `HttpRequestBuilder.WithSearch`/`HttpRequestMessage.WithSearch` (or `QueryArgs.Search` with `WithQuery(QueryArgs)`); API endpoints advertise it with `[Query(SupportsSearch = true)]`; GraphQL-lite list roots expose a `search` argument. + +The defaults of **200 characters** (`WithMaxLength`) and **8 distinct terms** (`WithMaxTerms`) are configurable CoreEx policy choices, not OData or search-industry standards. Length is measured after whitespace normalization using .NET `string.Length` (UTF-16 code units), in every mode. The term limit applies only to keyword mode: with `T` terms and `F` fields, the generated predicate contains `T × F` field comparisons. These defaults suit short catalog queries; longer natural-language requests may warrant a larger length limit for semantic/hybrid search, chosen against the repository's actual costs and usage. + +`SearchResult.Text` preserves word order, repetitions and casing, collapsing whitespace only. `SearchResult.Terms` is separately deduplicated case-insensitively so repeated keyword terms do not generate redundant predicates or consume the term allowance. Deduplication deliberately happens **after** constructing and length-checking `Text`: removing repeated words from a natural-language query can change its meaning or ranking, and repetition must not bypass the length limit. ## Key types | Type | Description | |------|-------------| -| **[`QueryArgsConfig`](./QueryArgsConfig.cs)** | Root builder: `Create()`, `WithFilter(Action)`, `WithOrderBy(Action)`; holds `FilterParser`, `OrderByParser`, and `ParsingConfig`. | +| **[`QueryArgsConfig`](./QueryArgsConfig.cs)** | Root builder: `Create()`, `WithFilter(Action)`, `WithOrderBy(Action)`, `WithSearch(Action)`; holds `FilterParser`, `OrderByParser`, `SearchParser`, and `ParsingConfig`. | +| **[`QuerySearchParser`](./QuerySearchParser.cs)** | Configures and parses `$search`: `WithMode`, `WithMaxLength`, `WithMaxTerms`, `AddKeywordField`, `WithValidator`, `Parse(string)`. | +| [`QuerySearchParserResult`](./QuerySearchParserResult.cs) | `$search` parse output: `Mode`, `IsRanked`, normalized `Text`, distinct `Terms`, `ToLinqString` (keyword only), and `Error`. | +| [`QuerySearchFieldConfig`](./QuerySearchFieldConfig.cs) | Keyword field config: model name/prefix, `WithOperator` (`Contains`, `StartsWith`, `EndsWith`, `Equal`), `AsUpperCase`/`AsLowerCase`, `WithNoNullCheck`. | +| [`QuerySearchMode`](./QuerySearchMode.cs) | `Keyword` (framework-executed), `FullText`, `Semantic`, `Hybrid` (repository-executed, relevance-ranked). | | **[`QueryFilterParser`](./QueryFilterParser.cs)** | Registers filterable fields and parses `$filter` strings: `AddField`, `AddNullField`, `AddReferenceDataField`, `AddEnumField`, `WithDefault`, `OnQuery`, `Parse(string)`. | | **[`QueryOrderByParser`](./QueryOrderByParser.cs)** | Registers sortable fields and parses `$orderby` strings: `AddField`, `WithDefault`, `Parse(string)`. | | **[`QueryExtensions`](./QueryExtensions.cs)** | `IQueryable.Where` and `OrderBy` — drives both parsers, applies filter and order-by to the queryable, and throws on parse errors. | -| **[`QueryArgsParseResult`](./QueryArgsParseResult.cs)** | Combined parse output: `FilterResult`, `OrderByResult`, `HasErrors` and resulting `Error` (where applicable). | +| **[`QueryArgsParseResult`](./QueryArgsParseResult.cs)** | Combined parse output: `FilterResult`, `OrderByResult`, `SearchResult`, `IsRankedSearch`, `HasErrors` and resulting `Error` (where applicable). | | **[`QueryStatement`](./QueryStatement.cs)** | Carries a Dynamic LINQ predicate string and its parameter array, applied to `IQueryable` via `.Where(statement.Predicate, statement.Args)`. | | [`QueryFilterParserResult`](./QueryFilterParserResult.cs) | `$filter` parse output: `QueryStatement`, per-field parsed values dictionary, and `ParseErrors`. | | [`QueryOrderByParserResult`](./QueryOrderByParserResult.cs) | `$orderby` parse output: `QueryStatement` and `ParseErrors`. | @@ -49,4 +148,4 @@ Field configurations (`QueryFilterFieldConfig`, `QueryFilterParseableFieldCon - **[`CoreEx.Data`](../README.md)** - Root package; `QueryArgs` (filter/orderby strings) and `PagingArgs` are defined there and passed into the `IQueryable.Where` and `OrderBy` extensions. - **[`CoreEx.RefData`](../../CoreEx/RefData/README.md)** - `QueryFilterReferenceDataFieldConfig` calls `ReferenceDataOrchestrator` to resolve codes to IDs during filter parsing. - **[`CoreEx.Database`](../../CoreEx.Database/README.md)** - ADO.NET query builders call `Where`/`OrderBy` to apply filter/orderby to SQL-backed `IQueryable`. -- **[`CoreEx.EntityFrameworkCore`](../../CoreEx.EntityFrameworkCore/README.md)** - EF Core query extensions consume `QueryArgsConfig` identically via the same `Where`/`OrderBy` extension methods. \ No newline at end of file +- **[`CoreEx.EntityFrameworkCore`](../../CoreEx.EntityFrameworkCore/README.md)** - EF Core query extensions consume `QueryArgsConfig` identically via the same `Where`/`OrderBy` extension methods. diff --git a/src/CoreEx/Data/QueryArgs.cs b/src/CoreEx/Data/QueryArgs.cs index 12615a97c..e9664934d 100644 --- a/src/CoreEx/Data/QueryArgs.cs +++ b/src/CoreEx/Data/QueryArgs.cs @@ -23,6 +23,13 @@ public class QueryArgs /// public string? OrderBy { get; set; } + /// + /// Gets or sets the free-text $search statement. + /// + /// The search text is opaque; i.e. no grammar (boolean operators, phrases, etc.) is defined or interpreted by CoreEx. The text is passed as-is to the configured + /// search capability (keyword, full-text, semantic/vector, hybrid, etc.) which is responsible for its interpretation; see CoreEx.Data.Querying.QuerySearchParser. + public string? Search { get; set; } + /// /// Gets or sets the list of included fields. /// @@ -45,6 +52,17 @@ public class QueryArgs /// public bool IsIncludeInactive { get; set; } + /// + /// Sets the free-text statement. + /// + /// The free-text $search statement. + /// The to support fluent-style method-chaining. + public QueryArgs WithSearch(string? search) + { + Search = search; + return this; + } + /// /// Adds the specified fields to the list. /// @@ -90,5 +108,5 @@ public QueryArgs IncludeInactive() } /// - public override string ToString() => $"Filter={Filter}, OrderBy={OrderBy}, IncludeFields={(IncludeFields is null ? string.Empty : string.Join(',', IncludeFields))}, ExcludeFields={(ExcludeFields is null ? string.Empty : string.Join(',', ExcludeFields))}, IsIncludeText={IsIncludeText}, IsIncludeInactive={IsIncludeInactive}"; + public override string ToString() => $"Filter={Filter}, OrderBy={OrderBy}, Search={Search}, IncludeFields={(IncludeFields is null ? string.Empty : string.Join(',', IncludeFields))}, ExcludeFields={(ExcludeFields is null ? string.Empty : string.Join(',', ExcludeFields))}, IsIncludeText={IsIncludeText}, IsIncludeInactive={IsIncludeInactive}"; } diff --git a/src/CoreEx/Extensions.HttpRequestMessage.cs b/src/CoreEx/Extensions.HttpRequestMessage.cs index 4bf01826c..1b835a74c 100644 --- a/src/CoreEx/Extensions.HttpRequestMessage.cs +++ b/src/CoreEx/Extensions.HttpRequestMessage.cs @@ -94,7 +94,25 @@ public static HttpRequestMessage WithPaging(this HttpRequestMessage request, lon /// The . /// The to support fluent-style method-chaining. public static HttpRequestMessage WithQuery(this HttpRequestMessage request, QueryArgs? query) - => WithQuery(request, query?.Filter, query?.OrderBy, query?.IncludeFields, query?.ExcludeFields); + { + WithQuery(request, query?.Filter, query?.OrderBy, query?.IncludeFields, query?.ExcludeFields); + if (query is null) + return request; + + return AddQuery(request, qb => qb + .AddQuery(HttpNames.QuerySearchQueryStringName, query.Search) + .AddQuery(HttpNames.IncludeTextQueryStringName, query.IsIncludeText ? "true" : null) + .AddQuery(HttpNames.IncludeInactiveQueryStringName, query.IsIncludeInactive ? "true" : null)); + } + + /// + /// Adds the free-text statement to the by adding to the . + /// + /// The . + /// The free-text $search statement; ignored where or white space. + /// The to support fluent-style method-chaining. + public static HttpRequestMessage WithSearch(this HttpRequestMessage request, string? search) + => AddQuery(request, qb => qb.AddQuery(HttpNames.QuerySearchQueryStringName, search)); /// /// Adds to the by adding to the . diff --git a/src/CoreEx/Http/HttpNames.cs b/src/CoreEx/Http/HttpNames.cs index 61835447e..418df027c 100644 --- a/src/CoreEx/Http/HttpNames.cs +++ b/src/CoreEx/Http/HttpNames.cs @@ -45,6 +45,11 @@ public static class HttpNames /// public static string QueryOrderByQueryStringName { get; set; } = "$orderby"; + /// + /// Gets or sets the name. + /// + public static string QuerySearchQueryStringName { get; set; } = "$search"; + /// /// Gets or sets the name. /// diff --git a/src/CoreEx/Http/HttpRequestBuilder.cs b/src/CoreEx/Http/HttpRequestBuilder.cs index ea147615a..fbec4dc7d 100644 --- a/src/CoreEx/Http/HttpRequestBuilder.cs +++ b/src/CoreEx/Http/HttpRequestBuilder.cs @@ -109,6 +109,13 @@ public HttpRequestBuilder Configure(Action configureRequest) /// The instance to support fluent chaining. public HttpRequestBuilder WithQuery(string? filter = null, string? orderBy = null, IEnumerable? include = null, IEnumerable? exclude = null) => Configure(r => r.WithQuery(filter, orderBy, include, exclude)); + /// + /// Adds the free-text statement to the request URI query string. + /// + /// The free-text $search statement; ignored where or white space. + /// The instance to support fluent chaining. + public HttpRequestBuilder WithSearch(string? search) => Configure(r => r.WithSearch(search)); + /// /// Adds to the request URI query string. /// diff --git a/tests/CoreEx.Data.Test.Unit/Querying/QuerySearchParserTests.cs b/tests/CoreEx.Data.Test.Unit/Querying/QuerySearchParserTests.cs new file mode 100644 index 000000000..e3e105657 --- /dev/null +++ b/tests/CoreEx.Data.Test.Unit/Querying/QuerySearchParserTests.cs @@ -0,0 +1,209 @@ +using CoreEx.Data; +using CoreEx.Data.Querying; +using CoreEx.Http; +using System.Linq.Dynamic.Core; + +namespace CoreEx.Data.Test.Unit.Querying; + +[TestFixture] +public class QuerySearchParserTests +{ + private record Person(string FirstName, string? LastName, string Code); + + private static IQueryable People => new[] + { + new Person("Bob", "Brown", "AB-100"), + new Person("Zoe", "Smith", "AB-200"), + new Person("Angela", "Smith", "XY-300"), + new Person("Brownie", null, "ZZ-400") + }.AsQueryable(); + + private static QueryArgsConfig CreateConfig(Action? configure = null) => QueryArgsConfig.Create() + .WithFilter(f => f.AddField("LastName")) + .WithOrderBy(o => o.AddField("FirstName", c => c.WithDefault())) + .WithSearch(s => + { + s.AddKeywordField("FirstName", c => c.AsUpperCase()) + .AddKeywordField("LastName", c => c.AsUpperCase()) + .AddKeywordField("Code", c => c.WithOperator(QueryFilterOperator.StartsWith).AsUpperCase()); + + configure?.Invoke(s); + }); + + [Test] + public void Parse_NullOrWhiteSpace_NoText() + { + var config = CreateConfig(); + config.SearchParser.Parse(null).HasText.Should().BeFalse(); + config.SearchParser.Parse(" ").HasText.Should().BeFalse(); + config.Parse(new QueryArgs { Search = " " }).SearchResult.Should().BeNull(); + } + + [Test] + public void Parse_Normalizes_And_Dedupes() + { + var result = CreateConfig().SearchParser.Parse(" smith \t Brown SMITH ").ThrowOnError(); + result.Text.Should().Be("smith Brown SMITH"); + result.Terms.Should().BeEquivalentTo(["smith", "Brown"], o => o.WithStrictOrdering()); + result.IsRanked.Should().BeFalse(); + } + + [Test] + public void Parse_Keyword_ToLinqString() + { + var result = CreateConfig().SearchParser.Parse("smith ab").ThrowOnError(); + result.ToLinqString(out var args).Should().Be( + "((FirstName != null && FirstName.ToUpper().Contains(@0)) || (LastName != null && LastName.ToUpper().Contains(@1)) || (Code != null && Code.ToUpper().StartsWith(@2)))" + + " && ((FirstName != null && FirstName.ToUpper().Contains(@3)) || (LastName != null && LastName.ToUpper().Contains(@4)) || (Code != null && Code.ToUpper().StartsWith(@5)))"); + + args.Should().BeEquivalentTo(["SMITH", "SMITH", "SMITH", "AB", "AB", "AB"], o => o.WithStrictOrdering()); + } + + [Test] + public void Parse_Keyword_Field_Options() + { + var config = QueryArgsConfig.Create().WithSearch(s => s + .WithDefaultModelPrefix("P") + .AddKeywordField("A", c => c.AsLowerCase().WithNoNullCheck().WithOperator(QueryFilterOperator.Equal)) + .AddKeywordField("B", c => c.WithNoModelPrefix().WithOperator(QueryFilterOperator.EndsWith))); + + config.SearchParser.Parse("Xy").ToLinqString(out var args).Should().Be("(P.A.ToLower() == @0 || (B != null && B.EndsWith(@1)))"); + args.Should().BeEquivalentTo(["xy", "Xy"], o => o.WithStrictOrdering()); + } + + [Test] + public void Parse_MaxLength_Error() + { + var result = CreateConfig(s => s.WithMaxLength(5)).SearchParser.Parse("abcdef"); + result.HasError.Should().BeTrue(); + result.Error!.Messages![0].Text.ToString().Should().Be("Statement must not exceed 5 characters in length."); + result.Error.Messages[0].Property.Should().Be(HttpNames.QuerySearchQueryStringName); + result.Error.Extensions.Should().ContainKey("schema"); + } + + [Test] + public void Parse_MaxTerms_Error() + { + var result = CreateConfig(s => s.WithMaxTerms(2)).SearchParser.Parse("a b c"); + result.HasError.Should().BeTrue(); + result.Error!.Messages![0].Text.ToString().Should().Be("Statement must not contain more than 2 distinct terms."); + + // Duplicates are not counted. + CreateConfig(s => s.WithMaxTerms(2)).SearchParser.Parse("a b A").HasError.Should().BeFalse(); + } + + [Test] + public void Parse_MaxTerms_NotEnforced_WhenRanked() + { + var result = QueryArgsConfig.Create().WithSearch(s => s.WithMode(QuerySearchMode.Semantic).WithMaxTerms(1)).SearchParser.Parse("cheap red bike"); + result.HasError.Should().BeFalse(); + result.IsRanked.Should().BeTrue(); + result.ToLinqString(out var args).Should().BeNull(); + args.Should().BeEmpty(); + } + + [Test] + public void Parse_Validator_Error() + { + var result = CreateConfig(s => s.WithValidator(r => { if (r.Terms.Any(t => t.Length < 2)) throw new QuerySearchParserException("Each term must be at least 2 characters."); })).SearchParser.Parse("ab c"); + result.HasError.Should().BeTrue(); + result.Error!.Messages![0].Text.ToString().Should().Be("Each term must be at least 2 characters."); + } + + [Test] + public void Parse_Keyword_NoFields_Throws() + { + var config = QueryArgsConfig.Create().WithSearch(_ => { }); + FluentActions.Invoking(() => config.SearchParser.Parse("x")).Should().Throw(); + } + + [Test] + public void Config_SearchNotSupported_Error() + { + var result = QueryArgsConfig.Create().Parse(new QueryArgs { Search = "x" }); + result.HasError.Should().BeTrue(); + result.Error.Should().BeOfType(); + ((ValidationException)result.Error!).Messages![0].Text.ToString().Should().Be("Search statement is not currently supported."); + } + + [Test] + public void Config_Keyword_WithOrderBy_Allowed() + { + var result = CreateConfig().Parse(new QueryArgs { Search = "smith", OrderBy = "firstname desc" }).ThrowOnError(); + result.SearchResult.Should().NotBeNull(); + result.IsRankedSearch.Should().BeFalse(); + result.OrderByResult!.ToLinqString().Should().Be("FirstName desc"); + } + + [Test] + public void Config_Ranked_WithOrderBy_Error() + { + var result = CreateConfig(s => s.WithMode(QuerySearchMode.Hybrid)).Parse(new QueryArgs { Search = "smith", OrderBy = "firstname" }); + result.HasError.Should().BeTrue(); + result.Error.Should().BeOfType(); + } + + [Test] + public void Config_Ranked_NoDefaultOrderBy() + { + var config = CreateConfig(s => s.WithMode(QuerySearchMode.Semantic)); + var result = config.Parse(new QueryArgs { Search = "smith" }).ThrowOnError(); + result.IsRankedSearch.Should().BeTrue(); + result.SearchResult!.Text.Should().Be("smith"); + result.OrderByResult.Should().BeNull(); + + // Without search text the default order-by is applied as per usual. + config.Parse(new QueryArgs()).OrderByResult!.ToLinqString().Should().Be("FirstName"); + } + + [Test] + public void Config_ToJsonSchema_IncludesSearch() + { + var json = CreateConfig(s => s.WithHelpText("Searches names.")).ToJsonSchema(); + var search = json.GetProperty("search"); + search.GetProperty("mode").GetString().Should().Be("keyword"); + search.GetProperty("ranked").GetBoolean().Should().BeFalse(); + search.GetProperty("maxLength").GetInt32().Should().Be(QuerySearchParser.DefaultMaxLength); + search.GetProperty("maxTerms").GetInt32().Should().Be(QuerySearchParser.DefaultMaxTerms); + search.GetProperty("description").GetString().Should().Be("Searches names."); + search.TryGetProperty("fields", out _).Should().BeFalse(); + } + + [Test] + public void QueryExtensions_Where_ParseResult_AppliesFilterAndSearch() + { + var config = CreateConfig(); + var parsed = config.Parse(new QueryArgs { Filter = "lastname eq 'Smith'", Search = "zo" }).ThrowOnError(); + People.Where(parsed).OrderBy(parsed).Select(p => p.FirstName).ToArray().Should().BeEquivalentTo(["Zoe"]); + } + + [Test] + public void QueryExtensions_Where_ParseResult_AllTermsMustMatch() + { + var config = CreateConfig(); + var parsed = config.Parse(new QueryArgs { Search = "brown ab" }).ThrowOnError(); + People.Where(parsed).OrderBy(parsed).Select(p => p.FirstName).ToArray().Should().BeEquivalentTo(["Bob"]); + + // Null field values are safely ignored (Brownie has no LastName). + parsed = config.Parse(new QueryArgs { Search = "brown" }).ThrowOnError(); + People.Where(parsed).OrderBy(parsed).Select(p => p.FirstName).ToArray().Should().BeEquivalentTo(["Bob", "Brownie"], o => o.WithStrictOrdering()); + } + + [Test] + public void QueryExtensions_Where_ParseResult_Ranked_NotApplied() + { + var config = CreateConfig(s => s.WithMode(QuerySearchMode.Semantic)); + var parsed = config.Parse(new QueryArgs { Search = "nothing-matches" }).ThrowOnError(); + People.Where(parsed).Count().Should().Be(4); + } + + [Test] + public void QueryExtensions_Where_QueryArgs_AppliesFilterAndSearch() + { + var config = CreateConfig(); + People.Where(config, new QueryArgs { Filter = "lastname eq 'Smith'", Search = "xy" }).Select(p => p.FirstName).ToArray().Should().BeEquivalentTo(["Angela"]); + People.WhereSearch(config, "ab").Count().Should().Be(2); + FluentActions.Invoking(() => People.WhereSearch(QueryArgsConfig.Create(), "ab")).Should().Throw(); + People.WhereSearch(QueryArgsConfig.Create(), null).Count().Should().Be(4); + } +} diff --git a/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs b/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs index 12f1be770..3edf2c0a7 100644 --- a/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs +++ b/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs @@ -407,6 +407,7 @@ await client.Send(HttpMethod.Patch, "/api/widgets/1") .WithFields("id", "name") .WithoutFields("internalNotes") .WithPaging(skip: 10, take: 20, count: true) + .WithSearch("red bike") .GetResponseAsync(); captured.Should().NotBeNull(); @@ -423,6 +424,7 @@ await client.Send(HttpMethod.Patch, "/api/widgets/1") query.Should().Contain("$skip=10"); query.Should().Contain("$take=20"); query.Should().Contain("$count=true"); + query.Should().Contain("$search=red%20bike"); } [Test] diff --git a/tests/CoreEx.Test.Unit/Http/HttpRequestMessageExtensionsTests.cs b/tests/CoreEx.Test.Unit/Http/HttpRequestMessageExtensionsTests.cs index 0695762b5..84eb6a7a2 100644 --- a/tests/CoreEx.Test.Unit/Http/HttpRequestMessageExtensionsTests.cs +++ b/tests/CoreEx.Test.Unit/Http/HttpRequestMessageExtensionsTests.cs @@ -34,6 +34,33 @@ public void WithQuery_EncodesSpacesAndSpecialCharacters_RoundTrips() pairs["$orderby"].Should().Be("name desc"); } + [Test] + public void WithQuery_QueryArgs_IncludesSearchAndFlags() + { + var request = new HttpRequestMessage(HttpMethod.Get, "https://example.com/api"); + request.WithQuery(new CoreEx.Data.QueryArgs { Filter = "a eq 1", OrderBy = "a", Search = "red & bike", IsIncludeText = true, IsIncludeInactive = true }); + + var pairs = request.RequestUri!.Query.TrimStart('?').Split('&').Select(p => p.Split('=', 2)).ToDictionary(p => p[0], p => Uri.UnescapeDataString(p[1])); + + pairs.Should().HaveCount(5); + pairs["$filter"].Should().Be("a eq 1"); + pairs["$orderby"].Should().Be("a"); + pairs["$search"].Should().Be("red & bike"); + pairs["$text"].Should().Be("true"); + pairs["$inactive"].Should().Be("true"); + } + + [Test] + public void WithSearch_BlankIsIgnored() + { + var request = new HttpRequestMessage(HttpMethod.Get, "https://example.com/api"); + request.WithSearch(" "); + request.RequestUri!.Query.Should().BeEmpty(); + + request.WithSearch("bike"); + request.RequestUri!.Query.Should().Be("?$search=bike"); + } + [Test] public void WithIdempotencyKey_AddsHeader() { From aa0ba101bea82d0a746c1c05a9add44cd008f5ed Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 13 Oct 2026 08:50:55 -0700 Subject: [PATCH 03/25] Document Products vector and hybrid search design Add a beginner-friendly living guide covering embeddings, pgvector, retrieval, indexing, evaluation and implementation status. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- samples/README.md | 1 + samples/docs/search.md | 566 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 567 insertions(+) create mode 100644 samples/docs/search.md diff --git a/samples/README.md b/samples/README.md index 6d3905a79..50346e06d 100644 --- a/samples/README.md +++ b/samples/README.md @@ -12,6 +12,7 @@ Each of Products and Shopping is an independently deployable unit with an API ho > |---|---| > | [Layers](docs/layers.md) | Business and host layer overview with dependency diagram | > | [Patterns](docs/patterns.md) | Catalog of architectural patterns implemented across the samples | +> | [Search: from keywords to vectors](docs/search.md) | Beginner-friendly explanation and staged design for Products search, PostgreSQL/pgvector, hybrid ranking, indexing, and MCP; distinguishes implemented behavior from planned work | > | [Tooling](docs/tooling.md) | Code generation (`*.CodeGen`) and database management (`*.Database`) | > | [Testing](docs/testing.md) | Unit, intra-domain, inter-domain, and E2E testing strategy | > | [Aspire & E2E](docs/aspire.md) | Local orchestration and cross-domain end-to-end validation | diff --git a/samples/docs/search.md b/samples/docs/search.md new file mode 100644 index 000000000..1c34f5a9a --- /dev/null +++ b/samples/docs/search.md @@ -0,0 +1,566 @@ +# Product search: from keywords to vectors + +This guide explains how product search works, why vectors can help, and how we intend to extend the Contoso Products sample using PostgreSQL and pgvector. It assumes no previous experience with embeddings, vector databases, or search engines. + +The goal is not merely to store some numbers and label the result "AI". It is to demonstrate an understandable search capability with correct filtering, reliable updates, and evidence of useful results. + +**Read this as both a learning guide and a living design.** The status below separates working sample behavior from proposed implementation. SQL examples illustrate the design; they are not migrations to run against the current database. + +## Implementation status + +| Capability | Status | What that means | +|---|---|---| +| CoreEx `QueryArgs.Search` and search parser configuration | Implemented | REST, HTTP clients, OpenAPI, and GraphQL-lite can carry search text. | +| Products keyword search | Implemented | Every distinct whitespace-separated term must match the product text or the beginning of its SKU. | +| MCP `search_products` forwarding to Products | Implemented | The tool calls the Products API with `$search`; it does not query the database. | +| Ranked-search mode plumbing | Implemented | CoreEx validates and carries ranked search intent; the repository must execute it. | +| Product search document and separate search projection | Proposed | No product embeddings are currently stored. | +| Embedding provider and pgvector semantic retrieval | Proposed | No embedding model is currently configured or called. | +| PostgreSQL full-text retrieval and hybrid ranking | Proposed | The current sample does not perform full-text or hybrid search. | +| Asynchronous search indexing and backfill | Proposed | The outbox and subscriber foundations exist; search-specific processing does not. | +| Real-model relevance evaluation and HNSW benchmark | Proposed | Existing keyword tests are not evidence of semantic-search quality or scale. | + +The proposed baseline is PostgreSQL as the search store, one embedding per product, a rebuildable search projection, and indexing outside product-write transactions. Model selection, vector dimensions, final schema, retrieval windows, and count semantics still need to be resolved during implementation. + +## Reading map + +- [Start with the three kinds of search](#three-kinds-of-search). +- [Learn what embeddings and distances mean](#embeddings-text-as-coordinates). +- [Understand PostgreSQL storage and retrieval](#postgresql-and-pgvector). +- [Follow the proposed sample architecture](#where-this-fits-in-the-sample). +- [See how indexing stays correct](#keeping-search-current). +- [Understand hybrid ranking](#hybrid-search-combining-two-result-lists). +- [Review the implementation stages and evidence](#implementation-stages). +- [Keep this guide aligned with the code](#keeping-this-guide-in-sync). + +## Three kinds of search + +Imagine a catalog containing "Muc-Off Nano Tech Bike Cleaner". + +### Keyword search: find these words + +A keyword search asks whether particular text occurs in selected fields. The implemented sample splits a request into whitespace-separated terms, requires every term to match, and allows each term to match either the product text or the SKU prefix. + +```text +GET /api/products?$search=rear%20derailleur +``` + +This finds products whose configured fields collectively match both words. It is not phrase matching: the words do not have to be adjacent. Repeated terms are deduplicated case-insensitively. + +Keyword search is simple, predictable, and useful. It is not inherently inferior to vectors. For an exact identifier, it may be the better tool. + +### Lexical full-text search: understand word forms and rank matches + +A full-text search engine processes words into searchable tokens and can rank matching documents. It can handle language features such as reducing "bikes" to a shared word form and ignoring common words. + +PostgreSQL provides full-text search without pgvector. This is more sophisticated than substring matching, but it still primarily matches words, not general meaning. + +### Semantic search: find related meaning + +Semantic search can connect: + +```text +User: "something to clean mud off my bike" +Product: "Muc-Off Nano Tech Bike Cleaner" +``` + +The words differ, but their meanings are related. An embedding model provides the numerical representation used to discover that relationship. + +### Hybrid search: use both signals + +Product catalogs contain both natural-language concepts and exact names or identifiers. Hybrid search combines lexical and semantic retrieval, so one approach can compensate for the other's weaknesses. + +It is a recommendation for this sample, not a universal requirement. Whether it improves results must be measured. + +## Search versus filtering + +Search and filtering answer different questions: + +| Capability | Question | Example | +|---|---|---| +| `$filter` | Which records satisfy explicit conditions? | `brand eq 'YETI'` | +| `$search` | Which records match this text or intent? | `trail bike` | +| `$orderby` | In what sequence should records appear? | `text desc` | +| Paging | Which portion of the result should be returned? | `$take=10` | + +For example: + +```text +GET /api/products?$search=trail%20bike&$filter=brand%20eq%20'YETI'&$take=10 +``` + +This should retrieve the most relevant eligible Yeti products, not search the entire catalog for ten products and then discard non-Yeti entries. + +Keyword mode allows ordinary ordering. Ranked modes own relevance ordering: CoreEx rejects an explicit `$orderby` and does not apply the configured default ordering. + +The server configuration selects the mode. There is currently no public `$searchMode` parameter. + +Price, availability, and other hard constraints belong in structured filtering, not in assumptions about what an embedding understands. The current Products filter configuration does not expose price or inventory filters; those would require explicit additions. + +See the [CoreEx querying guide](../../src/CoreEx.Data/Querying/README.md) for actual parser configuration and syntax. + +## Embeddings: text as coordinates + +An **embedding** is an array of numbers produced by a trained model. Texts with related meanings tend to receive nearby representations. + +An **embedding model** is the component that turns text into that array. It is not necessarily the chat model used by an AI assistant. + +Illustrative output might look like this: + +```text +"bike cleaner" -> [0.12, -0.48, 0.31, ...] +"remove bicycle mud" -> [0.15, -0.43, 0.29, ...] +``` + +These are made-up numbers to show the shape, not real embeddings. + +### What is a dimension? + +Each position in the array is a dimension. Real models often produce hundreds or thousands of dimensions. + +Do not picture a hand-written spreadsheet with "bicycle" in column one and "cleaning" in column two. Meaning is distributed across the coordinates learned by the model; individual dimensions usually do not have simple human-readable labels. + +The useful mental model is a map of relationships. Similar meanings tend to be near one another on that map. + +### Products and queries need compatible maps + +At indexing time, embed product text and store its vector. At query time, embed the user's text and compare it with those stored vectors. Do not regenerate every product embedding for each request. + +Both sides need compatible embedding profiles: + +- The same model and compatible version. +- The same output dimensions. +- The correct document/query instructions for that model. +- Consistent preprocessing and any required normalization. + +Some models require different prefixes or instructions for queries and documents. Follow the model's documented retrieval recipe rather than assuming identical inputs are always appropriate. + +Equal dimensions do not establish compatibility. An RGB color and a three-dimensional location both contain three numbers; comparing them would still be nonsense. + +### Embedding is not answer generation + +Generating an embedding does not write an answer or invent new product facts. Retrieval finds candidate records. An AI client may subsequently explain those records to a user. + +That explanation is a separate step, and the returned product data remains the authoritative evidence. + +## PostgreSQL and pgvector + +**pgvector** is a PostgreSQL extension that adds vector types, distance operators, and optional nearest-neighbor indexes. It does not call an embedding model. + +An **extension** is additional database functionality installed on the server and enabled in a database. `CREATE EXTENSION` cannot install server binaries that are absent. + +The current Compose setup uses a plain, unpinned `postgres` image. Implementation will need a compatible, pinned image containing pgvector, plus a database migration enabling the extension. Changing database images or major versions requires checking compatibility and preserving existing local data. + +### Proposed storage + +A **search projection** is derived data prepared for retrieval. It can be rebuilt from the authoritative products. + +The proposed one-to-one projection could have this shape: + +```sql +-- Illustrative only: names, dimensions, and constraints are not finalized. +CREATE EXTENSION IF NOT EXISTS vector; + +CREATE TABLE products.product_search +( + product_id text PRIMARY KEY, + document_text text NOT NULL, + embedding vector(768) NOT NULL, + embedding_profile text NOT NULL, + source_hash text NOT NULL, + embedded_at timestamptz NOT NULL DEFAULT now() +); +``` + +The `768` is an example, not a chosen model dimension. Choose the model first, then the storage type and index strategy. The final design must also define ownership/foreign keys, deletion handling, full-text storage, and profile migration. + +Why a separate table instead of another product column? + +- Ordinary CRUD need not load large vectors. +- Product contracts do not acquire provider-specific properties. +- Indexing need not change the business entity's ETag or changelog. +- Model/profile metadata and freshness are explicit. +- Search can be rebuilt without rewriting business records. + +Putting a vector on the product table is also legitimate for simpler applications. Separation here makes the derived-state lesson clearer. + +### Distance: how near are two vectors? + +**Cosine similarity** compares the direction of two vectors: + +```text +cosine similarity = dot product / (length of first vector * length of second vector) +cosine distance = 1 - cosine similarity +``` + +A dot product multiplies corresponding coordinates and adds them. Vector length measures the magnitude of the array as a point in that space. + +With cosine distance, smaller means closer. pgvector uses the `<=>` operator: + +```sql +SELECT product_id, + embedding <=> @query_vector AS distance +FROM products.product_search +ORDER BY embedding <=> @query_vector +LIMIT @take; +``` + +`@query_vector` and `@take` represent application-bound parameters, not SQL assembled by concatenating user text. This introductory example omits eligibility and profile checks; those are mandatory in the real repository. + +Other metrics include Euclidean distance and inner product. Use the embedding model's intended metric; cosine is a starting candidate, not an unconditional rule. + +For index-backed nearest-neighbor retrieval, preserve the supported distance expression in ascending `ORDER BY` with a limit. Ordering instead by a transformed similarity expression can prevent the intended index plan. + +### A score is not confidence + +A similarity of `0.82` does not mean an 82% probability of relevance. It is a geometric score, whose meaning depends on the model and dataset. + +Nearest-neighbor search still returns neighbors for an unrelated request such as "replacement submarine propeller". Nearest does not mean suitable. + +There is no universal relevance threshold. The sample needs unrelated-query examples and an evaluated policy for handling weak results rather than arbitrary confidence labels. + +## Choosing the searchable product document + +The **search document** is the text we deliberately prepare for the embedding model and lexical retrieval. + +Current product data includes short names, SKU, category/subcategory, brand, and tags. Many names contain little explanatory content. + +A canonical document might be: + +```text +Name: Yeti ASR C2 +Brand: Yeti +Category: Bikes +Subcategory: Cross-country +Tags: cross-country, full-suspension +``` + +Only include verified attributes. If the demonstration needs concepts such as weight, riding style, or component compatibility, add accurate descriptive data or select examples that the existing data supports. + +Do not generate invented product specifications merely to make semantic results impressive. + +### Keep rapidly changing facts relational + +Generally exclude price, current stock, and activity status from embeddings. Retrieve or filter these from live product data. + +Otherwise a price change can leave the stored embedding representing yesterday's price. Hard constraints must not depend on asynchronous re-embedding. + +### One product, one vector + +One embedding per product is the proposed baseline. + +**Chunking** means splitting a long document into smaller passages and embedding them separately. It is useful for manuals and lengthy articles, not automatically useful for a short catalog record. + +Searching product manuals could later introduce multiple passages per product, but that is a different retrieval shape. + +### Record how each vector was made + +An embedding profile should identify the model/version, output dimensions, and document-template version. A source hash identifies the exact searchable content. + +A **hash** is a compact fingerprint of content. It allows us to recognize unchanged documents without comparing or embedding them repeatedly. It does not establish which of two versions is newer. + +Changing models is an index migration. For a live system, build a compatible new projection alongside the old one, then switch coherently. Do not mix incompatible vectors just because they fit the same column. + +## Where this fits in the sample + +The Products domain owns search because it owns the catalog and its eligibility rules. MCP is a client-facing tool surface, not a second search engine. + +```mermaid +flowchart LR + Client["Website, API client, or MCP tool"] + API["Products API"] + Search["Products search implementation"] + Model["Embedding provider"] + DB["PostgreSQL + pgvector"] + Client -->|"Text and filters"| API + API --> Search + Search -->|"Embed query text"| Model + Model -->|"Compatible vector"| Search + Search -->|"Filtered ranked retrieval"| DB + DB -->|"Products in relevance order"| Search + Search --> API + API --> Client +``` + +This is the proposed ranked flow. Today the repository executes keyword search without an embedding provider. + +**CoreEx responsibilities:** carry intent through `QueryArgs`, validate configured constraints, apply keyword predicates, and distinguish ranked search from ordinary ordering. + +**Products implementation responsibilities:** prepare documents, generate embeddings, execute provider-specific queries, enforce eligibility, maintain derived state, and evaluate relevance/performance. + +The existing `ProductRepository.QueryAsync` joins product, subcategory, and inventory data. Ranked retrieval must preserve that behavior and all configured filters rather than quietly implementing a reduced API. + +Application orchestration should depend on application-owned interfaces. Embedding transport/SDK and pgvector access belong in Infrastructure, composed by the hosts. The exact interface names are not yet chosen. + +The .NET pgvector integration supports Npgsql and EF Core. Versions must match this repository's EF/Npgsql target matrix; external Npgsql data sources and EF options must both be configured where required. Database changes remain owned by DbEx migrations and regeneration, not hand-edited generated files. + +### What MCP adds + +MCP (Model Context Protocol) lets an AI client discover and invoke tools such as `search_products`. + +The tool sends search text to Products and returns product data. It does not generate its own embeddings, query PostgreSQL directly, or maintain its own index. Non-MCP clients use the same capability. + +After switching the sample to ranked search, update the tool description to describe the actual behavior instead of promising keyword matching. + +Search does not resolve authentication or authorization. The sample's existing authentication gaps remain separate concerns. + +## Keeping search current + +Product CRUD already commits product mutations and outgoing events together using the transactional outbox. + +An **outbox** stores an event in the same database transaction as its business change. A **relay** forwards committed events to the broker. A **subscriber** handles delivered events. + +The proposed indexing flow reuses that foundation: + +```mermaid +sequenceDiagram + participant API as Products API + participant DB as PostgreSQL + participant Relay as Outbox Relay + participant Bus as Service Bus + participant Worker as Search indexing subscriber + participant Model as Embedding provider + API->>DB: Commit product change and outbox event + Relay->>DB: Read committed outbox event + Relay->>Bus: Publish event + Bus->>Worker: Deliver event + Worker->>DB: Read current searchable source + Worker->>Model: Embed document outside transaction + Model-->>Worker: Return vector + Worker->>DB: Conditionally upsert search projection +``` + +This is **eventual consistency**: the product write can succeed before its search representation catches up. A freshly created product may temporarily be absent from semantic results; updated content may temporarily retrieve using its older representation. + +The API must not wait for external inference inside a product-write transaction. That would tie business writes to model availability and hold database resources during inference. + +The existing Products Subscribe host is a natural home for the worker. A dedicated subscription/receiver may isolate slower indexing from reservation commands without adding another host. Broker subscription rules, handler registration, concurrency, retries, and dead-letter behavior all need explicit wiring. + +### Duplicate delivery is normal + +At-least-once delivery means an event can arrive multiple times. The handler must be **idempotent**: retrying the same work must not corrupt state. + +The proposed handler should: + +1. Load current authoritative product data instead of blindly using an old event payload. +2. Construct the canonical document and source hash. +3. Skip work if the stored hash/profile already matches. +4. Generate the embedding outside a database transaction. +5. Recheck source freshness before writing. +6. Store only if the result still corresponds to current source state; otherwise schedule/retry fresh work. + +The final check/write requires a concurrency strategy, not just two unrelated reads. A short transaction with an appropriate lock/version check can prevent an older worker overwriting a newer projection. + +If reference-data descriptions contribute to the document, changes to those descriptions must also trigger or schedule reindexing of affected products. + +### Deactivation and deletion + +Live eligibility must exclude deleted or inactive products when appropriate, even while their old vectors remain stored. Deletion should also clean up the projection. + +Search eligibility must not rely solely on yesterday's indexed status. + +### Existing products and missed work + +Events do not automatically index pre-existing products. A **backfill** processes existing records; **reindexing** rebuilds records after a model/template change or repair. + +Provide an explicit resumable operation using the same builder and profile as event processing. It should batch work, skip current records, report failures, and support reconciliation after missed events or dead-lettered work. + +### Provider failures + +Embedding failures must be visible and retriable according to the operation's policy. Product writes should remain independent. + +A configured ranked request must not silently become keyword search or a successful empty result because inference failed. Any deliberate fallback needs explicit configuration, documentation, and observable behavior. + +## Exact and approximate nearest neighbors + +**Exact search** calculates distances over the eligible vectors and returns the actual nearest vectors. Start here for the small catalog. + +Exact vector neighbors are not necessarily perfect human matches. Exactness describes the distance calculation, not the model's understanding. + +**Approximate nearest-neighbor search (ANN)** uses an index to search promising regions instead of evaluating every vector. It trades some recall for speed. + +**Recall** here means how many true nearest neighbors the approximate search recovers relative to exact search. + +### HNSW + +HNSW (Hierarchical Navigable Small World) organizes vectors into a multilayer graph. Broad upper layers guide navigation; denser lower layers explore nearby candidates. + +An illustrative cosine index is: + +```sql +CREATE INDEX product_search_embedding_hnsw +ON products.product_search +USING hnsw (embedding vector_cosine_ops); +``` + +More graph exploration can improve recall at the cost of latency. Construction choices affect memory, build time, and update cost. + +IVFFlat is another pgvector index, grouping vectors into lists and searching selected lists. It has different build, memory, and recall tradeoffs; it is not necessary for the first implementation. + +For supported approximate indexes, dimensional limits depend on the type/operator class and pgvector version. Confirm model dimensions before choosing `vector`, `halfvec`, or another representation. + +An HNSW index over a few dozen rows is not proof of scalability. PostgreSQL may correctly choose a sequential scan. Use a separate larger corpus and `EXPLAIN (ANALYZE, BUFFERS)` to study actual plans and costs. + +### Filtering has two layers + +At the application level, put eligibility and client filters inside retrieval, before the candidate limit. Include authorization/tenant restrictions where applicable and embedding-profile compatibility. + +At the index level, an approximate scan may find candidates before PostgreSQL filters them. Restrictive filters can therefore return fewer results than requested even with correct SQL. + +Options include exact search over a selective subset, greater search effort, iterative scans, or suitable partial indexes/partitioning. Iterative scans explore more candidates but still have limits; they are not a guarantee of perfect recall. + +Settings that vary per request must not leak through pooled connections. Use transaction-local settings where appropriate. + +Do not glue retrieval to filtering by loading huge lists of eligible IDs into application memory. Prove database-side composition against the existing repository shape. + +## Hybrid search: combining two result lists + +PostgreSQL full-text search uses: + +| Term | Meaning | +|---|---| +| `tsvector` | Processed document words, often called lexemes, with optional positions/weights. | +| `tsquery` | Processed query conditions over those words. | +| `@@` | Tests whether a document matches a text query. | +| `ts_rank` / `ts_rank_cd` | PostgreSQL functions for ranking lexical matches. | +| GIN index | An index useful for finding documents containing lexemes. | + +A `tsvector` is not a neural embedding. PostgreSQL's built-in rank functions are not BM25 by default. + +`plainto_tsquery` is a straightforward candidate for our opaque text contract: it processes plain text and ANDs surviving words. Long conversational queries may produce few lexical matches. + +`websearch_to_tsquery` recognizes syntax such as quotes, OR, and negation. Choosing it would deliberately introduce provider-specific query interpretation that needs documenting. CoreEx does not currently promise that grammar. + +SKUs need explicit treatment because punctuation and language tokenization can behave differently from ordinary prose. Exact SKU matching is a useful separate signal, not something to leave to semantic similarity. + +### Why not just add the scores? + +Lexical rank and cosine similarity have different scales. Adding them without calibration can make one signal dominate for accidental numerical reasons. + +**Reciprocal Rank Fusion (RRF)** combines positions instead: + +```text +fused score(document) = sum over branches of weight / (k + rank) +``` + +Ranks start at one. A document missing from a branch contributes nothing from that branch. + +Suppose: + +```text +Lexical list: A, B, C +Semantic list: B, D, A +``` + +A and B benefit from appearing in both lists. The constant `k` smooths the effect of position; 60 is a common starting value, not a universal optimum. + +Retrieve a bounded candidate pool from each eligible branch, combine by product ID, fuse ranks, apply a deterministic tie-breaker, and return the requested window. + +The candidate pool is usually larger than the final page. Its size affects both quality and work; ten returned products does not imply ten candidates per branch are sufficient. + +RRF improves transparency: we can show lexical rank, semantic rank, and fused rank without introducing another inference model. A later reranker could assess query/document pairs more deeply, but it adds latency and complexity and is not part of the baseline. + +## Results, counts, and paging + +Keep vectors out of public product contracts. Initially, existing `ProductLite` responses in relevance order are sufficient. Diagnostic tooling can report component scores separately. + +Switching Products from keyword to semantic/hybrid is a behavior change, not merely internal optimization. Update schema, API examples, MCP descriptions, and tests together. + +Ranked search must define its result window and count semantics: + +- "All eligible products" is not "all relevant products". +- Nearest-neighbor retrieval has no inherent binary match count. +- A bounded hybrid pool is not the entire set of potential matches. +- Deep offset paging can require increasingly large candidate retrieval. +- Stable tie-breaking does not freeze rankings when content or indexes change. + +Before implementation is complete, record the chosen behavior for `$skip`, `$take`, `$count`, maximum ranked window, and empty/weak matches. Do not report candidate count as total match count without making that meaning explicit. + +## Model hosting and operational costs + +A locally runnable, pinned model is attractive for a self-contained learning sample, but "local" is not "free". Downloads, licensing, RAM, CPU/GPU needs, startup time, and query latency still matter. + +A hosted provider can simplify execution on developer machines, but introduces credentials, usage charges, network latency, availability, and data-transfer/privacy considerations. + +No provider or model is selected yet. Document the actual model, retrieval instructions, dimensions, resource requirements, and setup once chosen. + +Measure document and query embedding separately. Caching repeated query embeddings can help, but keys must include the profile and relevant preprocessing; user text may be sensitive, so retention/logging needs care. + +Trace embedding time, database time, indexing lag, retries, and failures. Avoid logging raw vectors, credentials, or full user queries by default. + +## Implementation stages + +| Stage | Learning objective | Evidence required | +|---|---|---| +| 1. Documents and embeddings | See how verified catalog content becomes vectors. | Selected profile, reproducible builder, provider configuration, provenance. | +| 2. Exact pgvector retrieval | Understand distance ordering with real filters. | Real database tests for ordering, eligibility, profile isolation, and paging policy. | +| 3. Full-text and RRF | Compare exact terminology with semantic intent. | Inspectable branch ranks and curated hybrid comparisons. | +| 4. Indexing and backfill | Maintain rebuildable derived state. | Duplicate/out-of-order handling, stale-write protection, updates/deletes, recovery. | +| 5. HNSW benchmark | Measure speed versus recall at meaningful scale. | Exact baseline, larger corpus, real query plans, latency and recall results. | +| 6. MCP demonstration | Reuse the domain capability from an AI client. | Same API behavior, accurate tool descriptions, cross-host demonstration. | + +These are teaching milestones, not instructions to ship ranked search without lifecycle support. Completion requires the corresponding operational behavior, not only the read query. + +### Two different kinds of testing + +**Deterministic engineering tests** use controlled embeddings/provider stubs to verify distance ordering, filters, failure handling, idempotency, and freshness against real PostgreSQL where appropriate. + +**Real-model evaluation** uses a pinned model and a curated set of queries with expected relevant products. Include exact SKUs, paraphrases, ambiguous intent, restrictive filters, and requests outside the catalog. + +A fake embedding generator proves plumbing, not semantic understanding. Real-model evaluation should be identifiable and reproducible rather than unexpectedly introducing a paid/network dependency into ordinary unit tests. + +For evaluation: + +| Measurement | What it tells us | +|---|---| +| Precision at K | How many of the first K results are relevant. | +| Recall at K | How many known relevant results appear in the first K. | +| nDCG at K | Whether more relevant results appear higher, using graded relevance judgments. | +| ANN recall | How closely approximate neighbors reproduce exact vector neighbors. | +| Latency distribution | Typical and slower-request times, including embedding and database work. | +| Indexing freshness | Delay between committed source changes and updated searchable state. | + +ANN recall and human relevance are different measurements. A fast index can faithfully reproduce poor embeddings. + +## Implementation map + +These existing files are the starting points, not claims that the proposed functionality already lives there: + +| Existing surface | Relevance | +|---|---| +| [Product query configuration](../src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs) | Current keyword fields and future server-selected ranked mode. | +| [Product repository](../src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs) | Existing filters, joins, ordering, mapping, and paging to preserve. | +| [Product read service](../src/Contoso.Products.Application/ProductReadService.cs) | Current query delegation through the Application boundary. | +| [Product service](../src/Contoso.Products.Application/ProductService.cs) | Atomic product mutations and outbox events. | +| [Product contract](../src/Contoso.Products.Contracts/ProductBase.cs) | Current authoritative catalog attributes. | +| [Database configuration](../src/Contoso.Products.Database/dbex.yaml) | DbEx-owned persistence generation and schema configuration. | +| [DbContext](../src/Contoso.Products.Infrastructure/Repositories/ProductsDbContext.cs) | Hand-authored EF/Npgsql composition; generated partials remain generator-owned. | +| [Subscribe host](../src/Contoso.Products.Subscribe/Program.cs) | Existing message receiving/composition foundations. | +| [MCP tools](../src/Contoso.Mcp/Tools/CommerceTools.cs) | API-only search forwarding and tool description. | +| [Compose infrastructure](../../docker-compose.yml) | PostgreSQL image and potential local inference resource. | +| [Product query tests](../tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs) | Current keyword API behavior and schema expectations. | + +See also [layer boundaries](layers.md), [Infrastructure](infrastructure-layer.md), [messaging hosts](hosts-layer.md), [tooling](tooling.md), [testing](testing.md), and [Aspire](aspire.md). + +## Keeping this guide in sync + +Treat documentation as part of each search implementation change, not a final clean-up exercise. + +When a stage lands: + +1. Update the status table only for behavior that is implemented and verified. +2. Replace illustrative decisions with the actual model/profile, schema, settings, and failure behavior; explain meaningful deviations. +3. Add links to the owning implementation and tests. Remove obsolete proposed details rather than keeping conflicting narratives. +4. Add runnable setup/backfill/evaluation commands when they exist. Put environment setup in [local-dev.md](local-dev.md) and cross-host wiring in [aspire.md](aspire.md), linking from here instead of duplicating them. +5. Update API examples, schema expectations, MCP descriptions, and observable behavior together. + +Keep unresolved choices explicitly unresolved. If the implementation remains keyword-only, this page must continue to say so. Never turn a proposed design into present-tense documentation just because the SQL looks convincing. + +## Primary references + +- [pgvector](https://github.com/pgvector/pgvector): vector types, metrics, exact/approximate retrieval, HNSW, filtering, and iterative scans. +- [pgvector for .NET](https://github.com/pgvector/pgvector-dotnet): Npgsql and EF Core integration and version compatibility. +- [PostgreSQL full-text search controls](https://www.postgresql.org/docs/current/textsearch-controls.html): lexemes, query parsing, weighting, and ranking. +- [CoreEx querying](../../src/CoreEx.Data/Querying/README.md): the implemented `$filter`, `$orderby`, and `$search` contract. + +Upstream documentation evolves. Pin implementation dependencies and verify their version-specific behavior instead of assuming the latest upstream README describes an older installed release. From e3312bf2ba8bc80f2b314de56396706325b1343c Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 13 Oct 2026 10:42:34 -0700 Subject: [PATCH 04/25] Add local Products embedding foundation and relevance diagnostics Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- CoreEx.slnx | 4 +- docker-compose.yml | 21 ++- samples/docs/local-dev.md | 35 +++++ samples/docs/search.md | 57 ++++++- samples/src/Contoso.Products.Api/Program.cs | 2 + .../appsettings.Development.json | 16 +- .../Embeddings/IProductEmbeddingAdapter.cs | 19 +++ .../GlobalUsing.cs | 4 +- .../Search/ProductSearchDocument.cs | 40 +++++ .../Embeddings/ProductEmbeddingAdapter.cs | 21 +++ .../Clients/Ollama/OllamaHttpClient.cs | 53 +++++++ .../GlobalUsing.cs | 3 + .../Clients/Ollama/OllamaHttpClientTests.cs | 141 ++++++++++++++++++ .../Contoso.Products.Test.Unit.csproj | 1 + .../Contoso.Products.Test.Unit/EntryPoint.cs | 4 +- .../Contoso.Products.Test.Unit/GlobalUsing.cs | 13 +- .../Search/ProductSearchDocumentTests.cs | 64 ++++++++ .../CoreExExtensions.ApplicationBuilder.cs | 2 +- tests/CoreEx.Test.Unit/GlobalUsing.cs | 3 + .../Http/HttpClientConfigurationTests.cs | 30 ++++ tools/test-product-embeddings.ps1 | 86 +++++++++++ 21 files changed, 604 insertions(+), 15 deletions(-) create mode 100644 samples/src/Contoso.Products.Application/Adapters/Embeddings/IProductEmbeddingAdapter.cs create mode 100644 samples/src/Contoso.Products.Application/Search/ProductSearchDocument.cs create mode 100644 samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductEmbeddingAdapter.cs create mode 100644 samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs create mode 100644 samples/tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs create mode 100644 samples/tests/Contoso.Products.Test.Unit/Search/ProductSearchDocumentTests.cs create mode 100644 tests/CoreEx.Test.Unit/GlobalUsing.cs create mode 100644 tests/CoreEx.Test.Unit/Http/HttpClientConfigurationTests.cs create mode 100644 tools/test-product-embeddings.ps1 diff --git a/CoreEx.slnx b/CoreEx.slnx index dbc3472b5..c0ed18e54 100644 --- a/CoreEx.slnx +++ b/CoreEx.slnx @@ -35,7 +35,7 @@ - + @@ -47,6 +47,7 @@ + @@ -104,6 +105,7 @@ + diff --git a/docker-compose.yml b/docker-compose.yml index 91d2f7946..8d7e56095 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,22 @@ services: ports: - "6379:6379" + ollama: + image: docker.io/ollama/ollama:0.13.5 + profiles: ["search"] + environment: + OLLAMA_NUM_PARALLEL: "1" + OLLAMA_MAX_LOADED_MODELS: "1" + ports: + - "127.0.0.1:11434:11434" + volumes: + - ollama-models:/root/.ollama + healthcheck: + test: ["CMD", "ollama", "list"] + interval: 10s + timeout: 5s + retries: 12 + aspire-dashboard: image: mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest environment: @@ -60,4 +76,7 @@ services: cosmos-emulator: image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-latest command: ["--protocol", "https", "--health-port", "8083"] - network_mode: host \ No newline at end of file + network_mode: host + +volumes: + ollama-models: diff --git a/samples/docs/local-dev.md b/samples/docs/local-dev.md index e20571bc2..63e6e7e9b 100644 --- a/samples/docs/local-dev.md +++ b/samples/docs/local-dev.md @@ -53,6 +53,41 @@ podman compose -f docker-compose.yml down | `dts-emulator` | `dts/dts-emulator:latest` | 8080, 8082 | Orders.Workflow.Worker, Orders.Api | Task hubs: `default`, `order` | | `aspire-dashboard` | `aspire-dashboard:latest` | 18888 (UI), 4317 (OTLP) | Optional — any host with OTLP configured | Usable standalone; no need to run the full Aspire AppHost just for traces | +### Optional local embedding model + +The first Products search milestone uses **Ollama 0.13.5** with **`nomic-embed-text:v1.5`**. +It is opt-in: normal Compose startup does not start the `search` profile or download a model. +From the repository root: + +```powershell +podman compose -f docker-compose.yml --profile search up -d ollama +podman compose -f docker-compose.yml exec ollama ollama pull nomic-embed-text:v1.5 +Invoke-RestMethod http://localhost:11434/api/version +.\tools\test-product-embeddings.ps1 +``` + +The runtime image is approximately **4 GB uncompressed** and the model file is approximately **274 MB**. +Allow additional room for downloaded layers, extraction, and container storage; the model size alone is not the disk budget. +This CPU-based setup requests no GPU and limits Ollama to one loaded model and one parallel inference request. + +The endpoint binds to host loopback at `localhost:11434` because Ollama has no authentication configured here. +The `ollama-models` named volume keeps downloads across container recreation. +Stopping the container does not delete that volume: + +```powershell +podman compose -f docker-compose.yml stop ollama +``` + +Nomic emits **768-dimensional** vectors with `search_document: ` and `search_query: ` task prefixes. +The client sends `truncate=false`, so an oversized input fails instead of silently changing the searchable document. +The smoke script checks the installed model manifest digest, validates vectors, and prints cosine rankings for seeded catalog examples. +It intentionally fails if the expected semantic ranking is not obtained; do not confuse a working embedding endpoint with good retrieval. + +Products API development configuration contains `Ollama:BaseAddress` and longer inference timeout settings. +Other environments must supply their own endpoint/timeouts before using the adapter. Keyword requests do not call Ollama. +This milestone does **not** enable pgvector or change the Products API's default keyword behavior. +See [the search guide](search.md) for implementation status and relevance findings. + ### Service Bus configuration The emulator is pre-configured by `servicebus/Config.json`. Key values the sample hosts depend on: diff --git a/samples/docs/search.md b/samples/docs/search.md index 1c34f5a9a..4dba2b0ce 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -14,13 +14,14 @@ The goal is not merely to store some numbers and label the result "AI". It is to | Products keyword search | Implemented | Every distinct whitespace-separated term must match the product text or the beginning of its SKU. | | MCP `search_products` forwarding to Products | Implemented | The tool calls the Products API with `$search`; it does not query the database. | | Ranked-search mode plumbing | Implemented | CoreEx validates and carries ranked search intent; the repository must execute it. | -| Product search document and separate search projection | Proposed | No product embeddings are currently stored. | -| Embedding provider and pgvector semantic retrieval | Proposed | No embedding model is currently configured or called. | +| Canonical product search document | Implemented | Versioned, whitespace-normalized documents and source hashes exclude price, status, inventory and business metadata. | +| Local embedding provider | Implemented | Ollama/Nomic document and query instructions, response validation, and an Application adapter boundary are available. | +| Separate search projection and pgvector semantic retrieval | Proposed | No product embeddings are currently stored or queried in PostgreSQL. | | PostgreSQL full-text retrieval and hybrid ranking | Proposed | The current sample does not perform full-text or hybrid search. | | Asynchronous search indexing and backfill | Proposed | The outbox and subscriber foundations exist; search-specific processing does not. | -| Real-model relevance evaluation and HNSW benchmark | Proposed | Existing keyword tests are not evidence of semantic-search quality or scale. | +| Real-model relevance evaluation and HNSW benchmark | Initial probes implemented; broader evaluation proposed | Two illustrative semantic expectations currently fail. Full-catalog relevance and index performance remain unproven. | -The proposed baseline is PostgreSQL as the search store, one embedding per product, a rebuildable search projection, and indexing outside product-write transactions. Model selection, vector dimensions, final schema, retrieval windows, and count semantics still need to be resolved during implementation. +The proposed baseline is PostgreSQL as the search store, one embedding per product, a rebuildable search projection, and indexing outside product-write transactions. The first milestone selects local Ollama with Nomic v1.5 and 768 dimensions. The final schema, retrieval windows, and count semantics still need to be resolved during implementation. ## Reading map @@ -482,7 +483,48 @@ A locally runnable, pinned model is attractive for a self-contained learning sam A hosted provider can simplify execution on developer machines, but introduces credentials, usage charges, network latency, availability, and data-transfer/privacy considerations. -No provider or model is selected yet. Document the actual model, retrieval instructions, dimensions, resource requirements, and setup once chosen. +The first milestone selects **Ollama 0.13.5** and **`nomic-embed-text:v1.5`**, using its native **768 dimensions** and separate `search_document: ` / `search_query: ` prefixes. The observed model manifest digest is `0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f`; the live smoke script requires that identity. The application client validates the response model name and vector shape, not the server's manifest digest; deployment must preserve the recorded model identity before a persisted profile is introduced. + +The container is CPU-based and opt-in through the `search` Compose profile. Its runtime image occupies about **4 GB uncompressed**, in addition to the **274 MB** model and temporary/container overhead. See [local embedding setup](local-dev.md#optional-local-embedding-model). + +The client requires exactly one finite, non-zero vector, disables silent truncation, and surfaces provider failures instead of substituting empty embeddings. A versioned canonical document builder creates a SHA-256 source fingerprint and deterministically orders/deduplicates tags. + +The document builder reads labels through the contract's `Brand`, `Category` and `SubCategory` reference-data properties. Call it within a CoreEx execution context with the reference-data orchestrator configured; callers do not pass duplicate label arguments. Its unit tests use the same seeded reference data through the unit-test host. Because `Text` supports localization, an indexer must use a consistent language context when producing documents. + +### First real-model finding + +The local embedding endpoint works, but the initial semantic ranking check does **not** yet pass. For "something to clean mud off my bike", Nomic ranked the Yeti bike above the bike cleaner when given current short names, category/brand labels and tags. It also ranked the Yeti bike above the Shimano derailleur for "a component for changing gears". Even the name-only probe did not reliably favor the cleaner. + +This is a useful counterexample to "embeddings work, therefore search is good". The failures are recorded rather than hidden by changing the queries or weakening the expected results. `tools/test-product-embeddings.ps1` evaluates both examples before reporting any failures. It calculates exact cosine distances in memory; it is not pgvector retrieval, a full-catalog evaluation, or evidence about index performance. + +Possible next steps are verified descriptive enrichment, a deliberately evaluated hybrid retrieval policy, or comparison with another model. These are explicit relevance decisions; none has been silently applied to make the example pass. + +### Controlled document-content experiment + +A follow-up experiment kept the same model digest, task prefixes, 768 dimensions, cosine calculation and three products. It compared four document recipes across eight queries: the two original failing queries, a whole-bike query, the short queries "bike cleaner" and "rear derailleur", and three additional paraphrases about washing a dirty cycle, shifting the chain on the rear cassette, and buying a whole bike rather than parts. + +| Document recipe | Expected product ranked first | +|---|---| +| Current canonical names, codes, labels and tags | 2 of 8 | +| Product title alone | 2 of 8 | +| Diagnostic functional description alone | 8 of 8 | +| Current canonical document plus diagnostic description | 8 of 8 | + +The diagnostic descriptions were deliberately simple: + +- Cleaner: "A bicycle cleaning product used to wash dirt and mud from a bicycle." +- Derailleur: "A rear derailleur is a bicycle drivetrain component that moves the chain between sprockets to change gears." +- Bike: "A complete cross-country mountain bicycle for riding on trails." + +These are **synthetic diagnostic inputs**, not verified manufacturer descriptions, new product facts, or changes to the seeded catalog. They test whether stating a product's function changes retrieval; they must not be quietly promoted into production data. + +For the original cleaning query, adding the description increased the cleaner's cosine similarity from **0.2934** to **0.7321**, ahead of the derailleur (**0.5374**) and bike (**0.5223**). For the gear-changing query, the enriched derailleur scored **0.6671**, ahead of the cleaner (**0.5584**) and bike (**0.5119**). Similarity is not a probability. + +Additional controls used an independent Python standard-library HTTP client and cosine calculation, reproducing the original PowerShell baseline rankings. Repeated query embeddings and single-versus-batched requests had cosine similarity **1.0** in this run. The cleaning query was closer to the functional cleaning description (**0.7803**) than to unrelated cake (**0.3661**) or database-backup (**0.4082**) descriptions. + +This is evidence that document content materially affects this configuration, and that the model can separate these functional descriptions. It is **not** proof that sparse text is the only cause, that the model is optimal, or that an eight-query/three-product sample predicts full-catalog quality. The whole-bike-versus-parts query had a particularly small winning margin, despite passing. There was no held-out evaluation set, and descriptions were designed with the task in mind. + +The next content decision is whether to introduce authoritative product descriptions or another curated source of functional text, followed by broader evaluation. The actual document builder and original smoke expectations remain unchanged by this experiment; pgvector cannot supply meaning absent from its input vectors. Measure document and query embedding separately. Caching repeated query embeddings can help, but keys must include the profile and relevant preprocessing; user text may be sensitive, so retention/logging needs care. @@ -528,6 +570,11 @@ These existing files are the starting points, not claims that the proposed funct | Existing surface | Relevance | |---|---| +| [Search document builder](../src/Contoso.Products.Application/Search/ProductSearchDocument.cs) | Versioned canonical text and source fingerprint. | +| [Embedding adapter interface](../src/Contoso.Products.Application/Adapters/Embeddings/IProductEmbeddingAdapter.cs) | Application-owned query/document embedding boundary. | +| [Ollama client](../src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs) | Task prefixes, model profile, explicit truncation policy, and response validation. | +| [Embedding client tests](../tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs) | Mocked success/error responses and exact request construction. | +| [Live embedding smoke check](../../tools/test-product-embeddings.ps1) | Explicit local-model probe and retained relevance expectations. | | [Product query configuration](../src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs) | Current keyword fields and future server-selected ranked mode. | | [Product repository](../src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs) | Existing filters, joins, ordering, mapping, and paging to preserve. | | [Product read service](../src/Contoso.Products.Application/ProductReadService.cs) | Current query delegation through the Application boundary. | diff --git a/samples/src/Contoso.Products.Api/Program.cs b/samples/src/Contoso.Products.Api/Program.cs index 923cb436e..54cff7212 100644 --- a/samples/src/Contoso.Products.Api/Program.cs +++ b/samples/src/Contoso.Products.Api/Program.cs @@ -29,6 +29,8 @@ private static void Main(string[] args) // Add all the dynamically registered services. builder.Services.AddDynamicServicesUsing(); + builder.AddTypedHttpClient("Ollama"); + // Add L1/L2 caching services. builder.Services.AddMemoryCache(); // Adds the in-memory cache - L1. builder.AddRedisDistributedCache("redis"); // Adds Redis as the distributed cache (using Aspire library) - L2. diff --git a/samples/src/Contoso.Products.Api/appsettings.Development.json b/samples/src/Contoso.Products.Api/appsettings.Development.json index 33507b598..b0e4ac49c 100644 --- a/samples/src/Contoso.Products.Api/appsettings.Development.json +++ b/samples/src/Contoso.Products.Api/appsettings.Development.json @@ -1,4 +1,18 @@ { + "Ollama": { + "BaseAddress": "http://localhost:11434/", + "Resilience": { + "AttemptTimeout": { + "Timeout": "00:01:00" + }, + "TotalRequestTimeout": { + "Timeout": "00:02:00" + }, + "CircuitBreaker": { + "SamplingDuration": "00:02:00" + } + } + }, "Logging": { "LogLevel": { "Default": "Information", @@ -23,4 +37,4 @@ } } } -} \ No newline at end of file +} diff --git a/samples/src/Contoso.Products.Application/Adapters/Embeddings/IProductEmbeddingAdapter.cs b/samples/src/Contoso.Products.Application/Adapters/Embeddings/IProductEmbeddingAdapter.cs new file mode 100644 index 000000000..606c0a3f7 --- /dev/null +++ b/samples/src/Contoso.Products.Application/Adapters/Embeddings/IProductEmbeddingAdapter.cs @@ -0,0 +1,19 @@ +namespace Contoso.Products.Application.Adapters.Embeddings; + +/// +/// Provides compatible document and query embeddings for product retrieval. +/// +public interface IProductEmbeddingAdapter +{ + /// Gets the provider/model profile that must match stored document embeddings. + string Profile { get; } + + /// Gets the required vector dimensions. + int Dimensions { get; } + + /// Embeds a canonical product search document. + Task> EmbedDocumentAsync(string text, CancellationToken ct = default); + + /// Embeds user search text using the model's retrieval-query instructions. + Task> EmbedQueryAsync(string text, CancellationToken ct = default); +} diff --git a/samples/src/Contoso.Products.Application/GlobalUsing.cs b/samples/src/Contoso.Products.Application/GlobalUsing.cs index 4aaebc93c..262c23287 100644 --- a/samples/src/Contoso.Products.Application/GlobalUsing.cs +++ b/samples/src/Contoso.Products.Application/GlobalUsing.cs @@ -12,4 +12,6 @@ global using CoreEx.RefData.Abstractions; global using CoreEx.Results; global using CoreEx.Validation; -global using System.Text.Json; \ No newline at end of file +global using System.Security.Cryptography; +global using System.Text; +global using System.Text.Json; diff --git a/samples/src/Contoso.Products.Application/Search/ProductSearchDocument.cs b/samples/src/Contoso.Products.Application/Search/ProductSearchDocument.cs new file mode 100644 index 000000000..e0985be65 --- /dev/null +++ b/samples/src/Contoso.Products.Application/Search/ProductSearchDocument.cs @@ -0,0 +1,40 @@ +namespace Contoso.Products.Application.Search; + +/// +/// Represents a canonical, model-independent product document and its content fingerprint. +/// +/// The document containing only authoritative searchable attributes. +/// The SHA-256 fingerprint of the template version and document. +public sealed record ProductSearchDocument(string Text, string SourceHash) +{ + /// Gets the template version; changes to the document recipe require a new version. + public const string TemplateVersion = "product-document-v1"; + + /// + /// Creates a document using the product's reference-data labels, falling back to codes where labels are unavailable. + /// + /// Price, inventory, activity, ETag and changelog are deliberately excluded; use live relational state for eligibility and display. + public static ProductSearchDocument Create(ProductBase product) + { + product.ThrowIfNull(); + var sku = Normalize(product.Sku).ThrowIfNullOrEmpty(nameof(product.Sku)); + var name = Normalize(product.Text).ThrowIfNullOrEmpty(nameof(product.Text)); + var tags = product.Tags is null ? [] : product.Tags.Select(Normalize).Where(t => t.Length > 0).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray(); + var text = string.Join('\n', + $"Name: {name}", + $"SKU: {sku}", + $"Brand: {LabelOrCode(product.Brand?.Text, product.BrandCode)}", + $"Category: {LabelOrCode(product.Category?.Text, product.CategoryCode)}", + $"Subcategory: {LabelOrCode(product.SubCategory?.Text, product.SubCategoryCode)}", + $"Tags: {string.Join(", ", tags)}"); + + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes($"{TemplateVersion}\n{text}"))); + return new ProductSearchDocument(text, hash); + } + + /// Normalizes whitespace without changing word order, repetition or casing. + private static string Normalize(string? value) => string.IsNullOrWhiteSpace(value) ? string.Empty : string.Join(' ', value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); + + /// Uses an available reference-data label, otherwise retaining its code. + private static string LabelOrCode(string? label, string? code) => string.IsNullOrWhiteSpace(label) ? Normalize(code) : Normalize(label); +} diff --git a/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductEmbeddingAdapter.cs b/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductEmbeddingAdapter.cs new file mode 100644 index 000000000..c8d755e35 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductEmbeddingAdapter.cs @@ -0,0 +1,21 @@ +namespace Contoso.Products.Infrastructure.Adapters.Embeddings; + +/// Isolates the Products application from Ollama's transport and retrieval instructions. +/// The validated embedding client. +[ScopedService] +public sealed class ProductEmbeddingAdapter(OllamaHttpClient client) : IProductEmbeddingAdapter +{ + private readonly OllamaHttpClient _client = client.ThrowIfNull(); + + /// + public string Profile => OllamaHttpClient.Profile; + + /// + public int Dimensions => OllamaHttpClient.Dimensions; + + /// + public Task> EmbedDocumentAsync(string text, CancellationToken ct = default) => _client.EmbedDocumentAsync(text, ct); + + /// + public Task> EmbedQueryAsync(string text, CancellationToken ct = default) => _client.EmbedQueryAsync(text, ct); +} diff --git a/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs b/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs new file mode 100644 index 000000000..bd9e84ab4 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs @@ -0,0 +1,53 @@ +namespace Contoso.Products.Infrastructure.Clients.Ollama; + +/// +/// Provides validated, full-dimensional Nomic retrieval embeddings through the local Ollama API. +/// +/// The typed client configured for the Ollama endpoint. +public sealed class OllamaHttpClient(HttpClient httpClient) +{ + /// Gets the versioned model tag used for this sample's embedding profile. + public const string ModelName = "nomic-embed-text:v1.5"; + + /// Gets the native embedding dimensions; reduced-dimensional embeddings are not used. + public const int Dimensions = 768; + + /// Gets the profile identifying model, dimensions and task-prefix recipe. + public const string Profile = "ollama:nomic-embed-text:v1.5:768:retrieval-prefix-v1"; + + // Ollama defaults to truncation when the false value is omitted; do not use CoreEx's omit-default serialization here. + private static readonly JsonSerializerOptions _requestOptions = new(JsonSerializerDefaults.Web); + + private static readonly Validator _validator = Validator.Create() + .HasProperty(r => r.Model, c => c.Mandatory().Error("The embedding response model does not match the requested profile.").WhenValue(v => v != ModelName)) + .HasProperty(r => r.Embeddings, c => c.Mandatory().Error("The embedding response must contain one finite, non-zero 768-dimensional vector.") + .WhenValue(v => v is not [{ Length: Dimensions } vector] || vector.Any(x => !float.IsFinite(x)) || !vector.Any(x => x != 0))); + + private readonly HttpClient _httpClient = httpClient.ThrowIfNull(); + + /// Embeds product text using Nomic's document instruction. + public Task> EmbedDocumentAsync(string text, CancellationToken ct = default) => EmbedAsync(text, "search_document: ", ct); + + /// Embeds search text using Nomic's query instruction. + public Task> EmbedQueryAsync(string text, CancellationToken ct = default) => EmbedAsync(text, "search_query: ", ct); + + /// Requests one embedding without silently truncating oversized input. + private Task> EmbedAsync(string text, string prefix, CancellationToken ct) + { + text = text.ThrowIfNull().Trim().ThrowIfNullOrEmpty(nameof(text)); + return _httpClient.SendValue(HttpMethod.Post, "api/embed", new { model = ModelName, input = prefix + text, truncate = false, keep_alive = "5m" }, jsonSerializerOptions: _requestOptions) + .WithValidator(_validator) + .ToResultAsync(ct) + .ThenAs(r => r.Embeddings is [{ Length: Dimensions } vector] ? vector : throw new HttpRequestException("The validated embedding response contains no vector.")); + } + + /// Represents only the fields consumed from Ollama's embedding response. + private sealed class EmbedResponse + { + /// Gets or sets the model that produced the vector. + public string? Model { get; set; } + + /// Gets or sets the embedding batch; this client requires exactly one vector. + public float[][]? Embeddings { get; set; } + } +} diff --git a/samples/src/Contoso.Products.Infrastructure/GlobalUsing.cs b/samples/src/Contoso.Products.Infrastructure/GlobalUsing.cs index 3ac6741e7..de18e5b6c 100644 --- a/samples/src/Contoso.Products.Infrastructure/GlobalUsing.cs +++ b/samples/src/Contoso.Products.Infrastructure/GlobalUsing.cs @@ -1,4 +1,6 @@ +global using Contoso.Products.Application.Adapters.Embeddings; global using Contoso.Products.Application.Repositories; +global using Contoso.Products.Infrastructure.Clients.Ollama; global using Contoso.Products.Infrastructure.Mapping; global using CoreEx; global using CoreEx.Data; @@ -15,6 +17,7 @@ global using CoreEx.Events.Publishing; global using CoreEx.Mapping; global using CoreEx.Results; +global using CoreEx.Validation; global using Microsoft.EntityFrameworkCore; global using Microsoft.Extensions.Logging; global using System.Text.Json; diff --git a/samples/tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs b/samples/tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs new file mode 100644 index 000000000..cca7a37d3 --- /dev/null +++ b/samples/tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs @@ -0,0 +1,141 @@ +namespace Contoso.Products.Test.Unit.Clients.Ollama; + +public class OllamaHttpClientTests : WithGenericTester +{ + private UnitTestEx.Mocking.MockHttpClientRequest _request = null!; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var factory = MockHttpClientFactory.Create(); + _request = factory.CreateClient("Ollama").Request(HttpMethod.Post, "api/embed"); + _request.WithAnyBody(); + Test.ReplaceHttpClientFactory(factory); + } + + [Test] + public void EmbedQuery_ValidResponse_ReturnsFullVector() => Test.Scoped(test => + { + var vector = Enumerable.Repeat(0.1f, OllamaHttpClient.Dimensions).ToArray(); + _request.Respond.WithJson(new { model = OllamaHttpClient.ModelName, embeddings = new[] { vector } }); + test.Run(async _ => + { + var result = await ExecutionContext.GetRequiredService().EmbedQueryAsync("bike cleaner").ConfigureAwait(false); + result.IsSuccess.Should().BeTrue(); + result.Value.Should().Equal(vector); + }).AssertSuccess(); + _request.Verify(); + }); + + [TestCase("missing")] + [TestCase("empty")] + [TestCase("wrong-dimensions")] + [TestCase("zero")] + [TestCase("multiple")] + [TestCase("wrong-model")] + [TestCase("missing-model")] + [TestCase("null-vector")] + public void EmbedDocument_InvalidSuccessfulResponse_ReturnsDependencyFailure(string scenario) => Test.Scoped(test => + { + var vector = Enumerable.Repeat(0.1f, OllamaHttpClient.Dimensions).ToArray(); + float[][]? embeddings = scenario switch + { + "missing" => null, + "empty" => [], + "wrong-dimensions" => [new float[3]], + "zero" => [new float[OllamaHttpClient.Dimensions]], + "multiple" => [vector, vector], + "null-vector" => [null!], // Exercises an invalid provider payload, not an application-produced vector. + _ => [vector] + }; + _request.Respond.WithJson(new { model = scenario == "missing-model" ? null : scenario == "wrong-model" ? "another-model" : OllamaHttpClient.ModelName, embeddings }); + test.Run(async _ => + { + var result = await ExecutionContext.GetRequiredService().EmbedDocumentAsync("Name: Cleaner").ConfigureAwait(false); + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType().Which.InnerException.Should().BeOfType(); + }).AssertSuccess(); + _request.Verify(); + }); + + [TestCase(HttpStatusCode.InternalServerError)] + [TestCase(HttpStatusCode.NotFound)] + [TestCase(HttpStatusCode.BadRequest)] + public void EmbedQuery_ProviderError_ReturnsFailure(HttpStatusCode status) => Test.Scoped(test => + { + _request.Respond.With(status); + test.Run(async _ => + { + var result = await ExecutionContext.GetRequiredService().EmbedQueryAsync("cleaner").ConfigureAwait(false); + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType(); + }).AssertSuccess(); + _request.Verify(); + }); + + [TestCase(true, "search_query: ")] + [TestCase(false, "search_document: ")] + public async Task Embed_UsesCorrectInstructionAndDisablesTruncation(bool query, string prefix) + { + using var handler = new InspectHandler(async (request, ct) => + { + request.RequestUri.Should().Be(new Uri("http://localhost:11434/api/embed")); + var json = JsonDocument.Parse(await request.Content!.ReadAsStringAsync(ct)); + using (json) + { + json.RootElement.GetProperty("model").GetString().Should().Be(OllamaHttpClient.ModelName); + json.RootElement.GetProperty("input").GetString().Should().Be(prefix + "cleaner"); + json.RootElement.GetProperty("truncate").GetBoolean().Should().BeFalse(); + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(new { model = OllamaHttpClient.ModelName, embeddings = new[] { Enumerable.Repeat(0.1f, 768).ToArray() } })) + }; + }); + using var http = new HttpClient(handler) { BaseAddress = new Uri("http://localhost:11434/") }; + var client = new OllamaHttpClient(http); + var result = query ? await client.EmbedQueryAsync(" cleaner ") : await client.EmbedDocumentAsync(" cleaner "); + result.IsSuccess.Should().BeTrue(); + } + + [TestCase(null)] + [TestCase("")] + [TestCase(" ")] + public void Embed_BlankInputDoesNotCallProvider(string? text) + { + using var http = new HttpClient(); + Action action = () => new OllamaHttpClient(http).EmbedQueryAsync(text!); + action.Should().Throw(); + } + + [Test] + public async Task Embed_NonFiniteCoordinate_ReturnsDependencyFailure() + { + var values = string.Join(',', Enumerable.Repeat("0.1", OllamaHttpClient.Dimensions - 1).Append("1e100")); + using var handler = new InspectHandler((_, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent($"{{\"model\":\"{OllamaHttpClient.ModelName}\",\"embeddings\":[[{values}]]}}") + })); + using var http = new HttpClient(handler) { BaseAddress = new Uri("http://localhost:11434/") }; + var result = await new OllamaHttpClient(http).EmbedQueryAsync("cleaner"); + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType().Which.InnerException.Should().BeOfType(); + } + + [Test] + public async Task Embed_CancellationIsPropagated() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + using var http = new HttpClient { BaseAddress = new Uri("http://localhost:11434/") }; + var client = new OllamaHttpClient(http); + Func action = () => client.EmbedQueryAsync("cleaner", cancellation.Token); + await action.Should().ThrowAsync(); + } + + private sealed class InspectHandler(Func> inspect) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => inspect(request, cancellationToken); + } +} diff --git a/samples/tests/Contoso.Products.Test.Unit/Contoso.Products.Test.Unit.csproj b/samples/tests/Contoso.Products.Test.Unit/Contoso.Products.Test.Unit.csproj index dbd414227..d09be4a99 100644 --- a/samples/tests/Contoso.Products.Test.Unit/Contoso.Products.Test.Unit.csproj +++ b/samples/tests/Contoso.Products.Test.Unit/Contoso.Products.Test.Unit.csproj @@ -14,6 +14,7 @@ + diff --git a/samples/tests/Contoso.Products.Test.Unit/EntryPoint.cs b/samples/tests/Contoso.Products.Test.Unit/EntryPoint.cs index 4660122fb..f94052716 100644 --- a/samples/tests/Contoso.Products.Test.Unit/EntryPoint.cs +++ b/samples/tests/Contoso.Products.Test.Unit/EntryPoint.cs @@ -9,6 +9,8 @@ public static void ConfigureApplication(IHostApplicationBuilder builder) builder.Services.AddMemoryCache(); builder.Services.AddReferenceDataOrchestrator(); + builder.AddTypedHttpClient("Ollama", client => client.BaseAddress = new Uri("http://localhost:11434/")); + // Reuse the "real" database configured reference data. var jdr = JsonDataReader.ParseYaml("ref-data.seed.yaml", JsonDataReaderOptions.CreateForReferenceData(JsonPropertyNamingConvention.SnakeCase)); builder.Services.AddSingleton(new ReferenceDataProviderDecorator(jdr)); @@ -18,7 +20,7 @@ public class ReferenceDataProviderDecorator(JsonDataReader jdr) : ReferenceDataP { public override Task GetAsync(Type type, CancellationToken cancellationToken = default) => type switch { - _ when type == typeof(Category) => Task.FromResult((IReferenceDataCollection)jdr.Deserialize("products.$^category")!.ExtendForTesting([new Category { Id = Runtime.NewId(), Code = "X", IsInactive = false }])), + _ when type == typeof(Category) => Task.FromResult((IReferenceDataCollection)jdr.Deserialize("products.$^category")!.ExtendForTesting(new Category[] { new() { Id = Runtime.NewId(), Code = "X", IsInactive = false } })), _ when type == typeof(SubCategory) => Task.FromResult((IReferenceDataCollection)jdr.Deserialize("products.$^sub_category")!), _ when type == typeof(UnitOfMeasure) => Task.FromResult((IReferenceDataCollection)jdr.Deserialize("products.$^unit_of_measure")!), _ when type == typeof(Brand) => Task.FromResult((IReferenceDataCollection)jdr.Deserialize("products.$^brand")!), diff --git a/samples/tests/Contoso.Products.Test.Unit/GlobalUsing.cs b/samples/tests/Contoso.Products.Test.Unit/GlobalUsing.cs index 4a2f441eb..2985ef222 100644 --- a/samples/tests/Contoso.Products.Test.Unit/GlobalUsing.cs +++ b/samples/tests/Contoso.Products.Test.Unit/GlobalUsing.cs @@ -1,16 +1,21 @@ -global using Contoso.Products.Contracts; +global using AwesomeAssertions; global using Contoso.Products.Application; -global using Contoso.Products.Application.Validators; global using Contoso.Products.Application.Repositories; +global using Contoso.Products.Application.Search; +global using Contoso.Products.Application.Validators; +global using Contoso.Products.Contracts; +global using Contoso.Products.Infrastructure.Clients.Ollama; global using CoreEx; +global using CoreEx.Data.Json; global using CoreEx.RefData; global using CoreEx.RefData.Abstractions; global using CoreEx.Results; global using CoreEx.UnitTesting; -global using CoreEx.Data.Json; global using CoreEx.Validation; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Hosting; global using Moq; +global using System.Net; +global using System.Text.Json; global using UnitTestEx; -global using ExecutionContext = CoreEx.ExecutionContext; \ No newline at end of file +global using ExecutionContext = CoreEx.ExecutionContext; diff --git a/samples/tests/Contoso.Products.Test.Unit/Search/ProductSearchDocumentTests.cs b/samples/tests/Contoso.Products.Test.Unit/Search/ProductSearchDocumentTests.cs new file mode 100644 index 000000000..ab0054d5c --- /dev/null +++ b/samples/tests/Contoso.Products.Test.Unit/Search/ProductSearchDocumentTests.cs @@ -0,0 +1,64 @@ +namespace Contoso.Products.Test.Unit.Search; + +public class ProductSearchDocumentTests : WithGenericTester +{ + [Test] + public void Create_NormalizesContentAndSortsTags() => Test.Scoped(test => test.Run(_ => + { + var product = new Product { Sku = " test-1 ", Text = " Bike \t cleaner ", BrandCode = "MUCOFF", CategoryCode = "A", SubCategoryCode = "CL", Tags = ["mud", " cleaning ", "mud", ""] }; + var document = ProductSearchDocument.Create(product); + + document.Text.Should().Be("Name: Bike cleaner\nSKU: TEST-1\nBrand: Muc-Off\nCategory: Accessories\nSubcategory: Cleaner\nTags: cleaning, mud"); + document.SourceHash.Should().HaveLength(64); + ProductSearchDocument.Create(product).Should().Be(document); + }).AssertSuccess()); + + [Test] + public void Create_ReorderedTagsAndNonSearchChangesDoNotChangeHash() => Test.Scoped(test => test.Run(_ => + { + var product = new Product { Sku = "test-1", Text = "Cleaner", Tags = ["mud", "cleaning"], Price = 10 }; + var before = ProductSearchDocument.Create(product); + product.Tags = ["cleaning", "mud", "mud"]; + product.Price = 99; + product.IsInactive = true; + product.ETag = "updated"; + ProductSearchDocument.Create(product).Should().Be(before); + }).AssertSuccess()); + + [Test] + public void Create_ChangedTextOrReferenceDataChangesHash() => Test.Scoped(test => test.Run(_ => + { + var product = new Product { Sku = "test-1", Text = "Cleaner", BrandCode = "MUCOFF" }; + var before = ProductSearchDocument.Create(product); + product.BrandCode = "SHIMANO"; + ProductSearchDocument.Create(product).SourceHash.Should().NotBe(before.SourceHash); + product.BrandCode = "MUCOFF"; + product.Text = "Lubricant"; + ProductSearchDocument.Create(product).SourceHash.Should().NotBe(before.SourceHash); + }).AssertSuccess()); + + [TestCase(null)] + [TestCase("")] + [TestCase(" ")] + public void Create_MissingReferenceDataDoesNotInventLabels(string? code) => Test.Scoped(test => test.Run(_ => + { + var product = new Product { Sku = "test-1", Text = "Cleaner", BrandCode = code, CategoryCode = code, SubCategoryCode = code }; + ProductSearchDocument.Create(product).Text.Should().Contain("Brand: \nCategory: \nSubcategory: \n"); + }).AssertSuccess()); + + [Test] + public void Create_UnknownReferenceDataFallsBackToCodes() => Test.Scoped(test => test.Run(_ => + { + var product = new Product { Sku = "test-1", Text = "Cleaner", BrandCode = "UNKNOWN", CategoryCode = "UNKNOWN", SubCategoryCode = "UNKNOWN" }; + ProductSearchDocument.Create(product).Text.Should().Contain("Brand: UNKNOWN\nCategory: UNKNOWN\nSubcategory: UNKNOWN\n"); + }).AssertSuccess()); + + [TestCase(null)] + [TestCase("")] + [TestCase(" ")] + public void Create_MissingProductNameIsRejected(string? text) + { + Action action = () => ProductSearchDocument.Create(new Product { Sku = "test-1", Text = text }); + action.Should().Throw(); + } +} diff --git a/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs b/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs index 53ffd5ebd..9b9ccd8b1 100644 --- a/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs +++ b/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs @@ -89,7 +89,7 @@ public static IHttpClientBuilder AddNamedHttpClient(this IHostApplicationBuilder cb.AddServiceDiscovery(); if (config.GetSection("Resilience").Exists()) - cb.AddStandardResilienceHandler(config); + cb.AddStandardResilienceHandler(config.GetSection("Resilience")); else cb.AddStandardResilienceHandler(); diff --git a/tests/CoreEx.Test.Unit/GlobalUsing.cs b/tests/CoreEx.Test.Unit/GlobalUsing.cs new file mode 100644 index 000000000..63a474677 --- /dev/null +++ b/tests/CoreEx.Test.Unit/GlobalUsing.cs @@ -0,0 +1,3 @@ +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Hosting; diff --git a/tests/CoreEx.Test.Unit/Http/HttpClientConfigurationTests.cs b/tests/CoreEx.Test.Unit/Http/HttpClientConfigurationTests.cs new file mode 100644 index 000000000..1a61d9523 --- /dev/null +++ b/tests/CoreEx.Test.Unit/Http/HttpClientConfigurationTests.cs @@ -0,0 +1,30 @@ +namespace CoreEx.Test.Unit.Http; + +public class HttpClientConfigurationTests +{ + [TestCase(false)] + [TestCase(true)] + public async Task NamedClient_BindsOnlyResilienceSection(bool configureResilience) + { + var builder = Host.CreateApplicationBuilder(); + var settings = new Dictionary + { + ["Embedding:BaseAddress"] = "http://localhost:11434/" + }; + if (configureResilience) + { + settings["Embedding:Resilience:AttemptTimeout:Timeout"] = "00:01:00"; + settings["Embedding:Resilience:TotalRequestTimeout:Timeout"] = "00:02:00"; + settings["Embedding:Resilience:CircuitBreaker:SamplingDuration"] = "00:02:00"; + } + + builder.Configuration.AddInMemoryCollection(settings); + builder.AddNamedHttpClient("Embedding"); + using var host = builder.Build(); + + await host.StartAsync().ConfigureAwait(false); + using var client = host.Services.GetRequiredService().CreateClient("Embedding"); + client.BaseAddress.Should().Be(new Uri("http://localhost:11434/")); + await host.StopAsync().ConfigureAwait(false); + } +} diff --git a/tools/test-product-embeddings.ps1 b/tools/test-product-embeddings.ps1 new file mode 100644 index 000000000..cfd6e2bf7 --- /dev/null +++ b/tools/test-product-embeddings.ps1 @@ -0,0 +1,86 @@ +[CmdletBinding()] +param( + [uri] $BaseAddress = 'http://localhost:11434/' +) + +$ErrorActionPreference = 'Stop' +$model = 'nomic-embed-text:v1.5' +$expectedDigest = '0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f' +$dimensions = 768 + +$models = Invoke-RestMethod -Uri ([uri]::new($BaseAddress, 'api/tags')) -TimeoutSec 30 +$installed = @($models.models | Where-Object name -EQ $model) +if ($installed.Count -ne 1 -or $installed[0].digest -ne $expectedDigest) { + throw "Expected $model with digest $expectedDigest. Pull the documented model and verify its identity before evaluating." +} + +function Get-Embedding([string] $Text) { + $body = @{ model = $model; input = $Text; truncate = $false; keep_alive = '5m' } | ConvertTo-Json + $response = Invoke-RestMethod -Uri ([uri]::new($BaseAddress, 'api/embed')) -Method Post -ContentType 'application/json' -Body $body -TimeoutSec 120 + if ($response.model -ne $model -or $response.embeddings.Count -ne 1 -or $response.embeddings[0].Count -ne $dimensions) { + throw 'Ollama returned an incompatible embedding response.' + } + + $vector = [double[]] $response.embeddings[0] + $norm = 0.0 + foreach ($value in $vector) { + if ([double]::IsNaN($value) -or [double]::IsInfinity($value)) { + throw 'Ollama returned a non-finite vector.' + } + $norm += $value * $value + } + if ($norm -eq 0) { + throw 'Ollama returned a zero vector.' + } + return ,$vector +} + +function Get-CosineDistance([double[]] $Query, [double[]] $Document) { + $dot = 0.0 + $queryNorm = 0.0 + $documentNorm = 0.0 + for ($i = 0; $i -lt $dimensions; $i++) { + $dot += $Query[$i] * $Document[$i] + $queryNorm += $Query[$i] * $Query[$i] + $documentNorm += $Document[$i] * $Document[$i] + } + return 1 - $dot / [math]::Sqrt($queryNorm * $documentNorm) +} + +# These names, labels and tags come from the existing product and reference-data seeds. +$products = @( + @{ Sku = 'MUCOFF-NANO-CLEANER'; Text = 'Muc-Off Nano Tech Bike Cleaner'; Brand = 'Muc-Off'; Category = 'Accessories'; Subcategory = 'Cleaner'; Tags = '' }, + @{ Sku = 'SHIMANO-XT-M8100-RD'; Text = 'Shimano Deore XT M8100 Rear Derailleur'; Brand = 'Shimano'; Category = 'Parts'; Subcategory = 'Derailleur'; Tags = '' }, + @{ Sku = 'YETI-ASR-C2-2025'; Text = 'Yeti ASR C2'; Brand = 'Yeti Cycles'; Category = 'Bikes'; Subcategory = 'Cross country'; Tags = 'cross-country, full-suspension' } +) +foreach ($product in $products) { + $document = "Name: $($product.Text)`nSKU: $($product.Sku)`nBrand: $($product.Brand)`nCategory: $($product.Category)`nSubcategory: $($product.Subcategory)`nTags: $($product.Tags)" + $product.Vector = Get-Embedding "search_document: $document" +} + +$queries = @( + @{ Text = 'something to clean mud off my bike'; Expected = 'MUCOFF-NANO-CLEANER' }, + @{ Text = 'a component for changing gears'; Expected = 'SHIMANO-XT-M8100-RD' } +) +$failures = [Collections.Generic.List[string]]::new() +foreach ($query in $queries) { + $vector = Get-Embedding "search_query: $($query.Text)" + $ranked = @($products | ForEach-Object { + [pscustomobject]@{ + Sku = $_.Sku + Distance = Get-CosineDistance $vector $_.Vector + } + } | Sort-Object Distance) + Write-Host "`nQuery: $($query.Text)" + $ranked | Format-Table Sku, @{Label = 'Cosine distance (lower is closer)'; Expression = { '{0:F4}' -f $_.Distance }} + if ($ranked[0].Sku -ne $query.Expected) { + $failures.Add("Query '$($query.Text)' expected $($query.Expected) first but retrieved $($ranked[0].Sku).") + } +} + +if ($failures.Count -gt 0) { + throw ($failures -join "`n") +} + +Write-Host "`nVerified $dimensions-dimensional real embeddings and two illustrative rankings." +Write-Host 'This is an in-memory learning smoke check, not PostgreSQL retrieval or a comprehensive relevance evaluation.' From 88e7c57b53727042d6d57192f5d2f9014bca0558 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 13 Oct 2026 11:10:17 -0700 Subject: [PATCH 05/25] Add product descriptions and price filtering Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- samples/docs/search.md | 77 ++++++++++++++++--- .../src/Contoso.Mcp/Tools/CommerceTools.cs | 4 +- .../Search/ProductSearchDocument.cs | 3 +- .../Validators/ProductValidator.cs | 3 +- .../Contoso.Products.Contracts/ProductBase.cs | 5 +- ...61013-174408-add-product-description.pgsql | 4 + .../Mapping/ProductMapper.cs | 4 +- .../Persistence/Product.g.cs | 5 +- .../Repositories/ProductQueryArgsConfig.cs | 1 + .../Repositories/ProductRepository.cs | 1 + .../Repositories/ProductsDbContext.g.cs | 1 + .../ProductMutateTests.Create.cs | 3 + .../ProductMutateTests.Patch.cs | 34 +++++++- .../ProductMutateTests.Update.cs | 29 ++++++- .../ReadTests.ProductGraphQLLite.cs | 27 +++++++ .../ReadTests.ProductQuery.cs | 39 ++++++++++ .../Create_Success.res.json | 3 +- .../ReadTests/Product_Get_Found.res.json | 3 +- .../ReadTests/Product_Query_Schema.res.json | 15 +++- .../Data/mutate-data.seed.yaml | 66 ++++++++-------- .../Data/read-data.seed.yaml | 66 ++++++++-------- .../Search/ProductSearchDocumentTests.cs | 16 +++- .../Validators/ProductValidatorTests.cs | 24 +++++- tools/test-product-embeddings.ps1 | 8 +- 24 files changed, 346 insertions(+), 95 deletions(-) create mode 100644 samples/src/Contoso.Products.Database/Migrations/20261013-174408-add-product-description.pgsql diff --git a/samples/docs/search.md b/samples/docs/search.md index 4dba2b0ce..6c4a4585d 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -15,11 +15,12 @@ The goal is not merely to store some numbers and label the result "AI". It is to | MCP `search_products` forwarding to Products | Implemented | The tool calls the Products API with `$search`; it does not query the database. | | Ranked-search mode plumbing | Implemented | CoreEx validates and carries ranked search intent; the repository must execute it. | | Canonical product search document | Implemented | Versioned, whitespace-normalized documents and source hashes exclude price, status, inventory and business metadata. | +| Product descriptions and price filtering | Implemented | Optional descriptive content supports CRUD; decimal price constraints use existing REST and GraphQL-lite query configuration. | | Local embedding provider | Implemented | Ollama/Nomic document and query instructions, response validation, and an Application adapter boundary are available. | | Separate search projection and pgvector semantic retrieval | Proposed | No product embeddings are currently stored or queried in PostgreSQL. | | PostgreSQL full-text retrieval and hybrid ranking | Proposed | The current sample does not perform full-text or hybrid search. | | Asynchronous search indexing and backfill | Proposed | The outbox and subscriber foundations exist; search-specific processing does not. | -| Real-model relevance evaluation and HNSW benchmark | Initial probes implemented; broader evaluation proposed | Two illustrative semantic expectations currently fail. Full-catalog relevance and index performance remain unproven. | +| Real-model relevance evaluation and HNSW benchmark | Initial probes implemented; broader evaluation proposed | Two illustrative semantic expectations pass after descriptive enrichment. Full-catalog relevance and index performance remain unproven. | The proposed baseline is PostgreSQL as the search store, one embedding per product, a rebuildable search projection, and indexing outside product-write transactions. The first milestone selects local Ollama with Nomic v1.5 and 768 dimensions. The final schema, retrieval windows, and count semantics still need to be resolved during implementation. @@ -96,7 +97,14 @@ Keyword mode allows ordinary ordering. Ranked modes own relevance ordering: Core The server configuration selects the mode. There is currently no public `$searchMode` parameter. -Price, availability, and other hard constraints belong in structured filtering, not in assumptions about what an embedding understands. The current Products filter configuration does not expose price or inventory filters; those would require explicit additions. +Price, availability, and other hard constraints belong in structured filtering, not in assumptions about what an embedding understands. Products exposes a decimal `price` filter with `eq`, `ne`, `in`, `gt`, `ge`, `lt` and `le`; inventory filters are not currently exposed. + +```text +GET /api/products?$search=cleaner&$filter=price%20lt%2020 +GET /api/products?$filter=price%20ge%2099%20and%20price%20lt%20130 +``` + +GraphQL-lite uses the same configuration, for example `products(where: {price: {ge: 99, lt: 130}})`. No generic `QueryArgs` changes or client-supplied vectors are needed. See the [CoreEx querying guide](../../src/CoreEx.Data/Querying/README.md) for actual parser configuration and syntax. @@ -224,15 +232,17 @@ There is no universal relevance threshold. The sample needs unrelated-query exam The **search document** is the text we deliberately prepare for the embedding model and lexical retrieval. -Current product data includes short names, SKU, category/subcategory, brand, and tags. Many names contain little explanatory content. +Current product data includes short names, an optional `Description`, SKU, category/subcategory, brand, and tags. Many names contain little explanatory content, so descriptions explain product function and intended use. A canonical document might be: ```text Name: Yeti ASR C2 -Brand: Yeti +SKU: YETI-ASR-C2-2025 +Description: ... +Brand: Yeti Cycles Category: Bikes -Subcategory: Cross-country +Subcategory: Cross country Tags: cross-country, full-suspension ``` @@ -240,6 +250,53 @@ Only include verified attributes. If the demonstration needs concepts such as we Do not generate invented product specifications merely to make semantic results impressive. +`Description` is optional plain text, limited to 2,000 characters in validation and PostgreSQL. It is returned in full-product and `ProductLite` responses and editable through existing POST, PUT and merge-PATCH endpoints; PATCH can explicitly clear it with `{"description":null}`. Updates retain the existing ETag, changelog and transactional-outbox behavior. The search document recipe is now `product-document-v2`, and description edits change its source hash; price changes do not. + +Descriptions are included in embedding documents. The default keyword `$search` still matches the product name (`Text`) or SKU prefix, not the new description field; this preserves the existing keyword behavior until a deliberate search-mode change. + +### Where the seeded descriptions come from + +Both Products read and mutation seed files contain the same original, short descriptions for all 32 entries. These are paraphrases of product function and intended use, not copied marketing text. The sources below support those statements; they do not validate the sample prices, every build's components, or every model-year SKU. + +Several sources describe a model family rather than the exact seeded build. The descriptions deliberately avoid travel numbers, model-year changes and precise component specifications. In particular, Specialized sells both a **Chisel Hardtail Comp** and a full-suspension **Chisel Comp**. The sample places its Chisel in the hardtail category, but its description says only "aluminium cross-country mountain bike" rather than guessing the suspension from the shared name. + +| Seeded product | Reference | +|---|---| +| Yeti ASR C2 | [Yeti ASR](https://yeticycles.com/en-us/bikes/asr) | +| Yeti SB120 C2 | [Yeti SB120](https://yeticycles.com/en-us/bikes/sb120) | +| Yeti SB140 LR C2 | [Yeti SB140](https://yeticycles.com/en-us/bikes/sb140) | +| Yeti SB165 C3 | [Yeti SB165 specifications and FAQ](https://support.yeticycles.com/hc/en-us/articles/56816935325211-SB165-specs-and-FAQ) | +| Yeti MTe C2 eMTB | [Yeti MTe](https://yeticycles.com/en-us/bikes/mte) | +| Canyon Lux Trail CF | [Canyon Lux Trail announcement](https://media-centre.canyon.com/en-INT/231333-the-brand-new-lux-trail-canyon-s-ultra-capable-xc-bike/) | +| Canyon Spectral CF | [Canyon Spectral CF announcement](https://media-centre.canyon.com/en-INT/234670-canyon-launch-all-new-trail-taming-spectral-cf/) | +| Canyon Neuron CF | [Canyon Neuron](https://www.canyon.com/en-us/mountain-bikes/trail-bikes/neuron/) | +| Canyon Torque Mullet CF | [MTB Database: Torque Mullet CF 8](https://mtbdatabase.com/bikes/2024/canyon/torque/2024-canyon-torque-mullet-cf-8/) (third-party model-family reference) | +| Canyon Sender CFR | [Canyon Sender](https://www.canyon.com/en-gb/mountain-bikes/downhill-bikes/sender/) | +| Specialized Stumpjumper 15 Comp | [Stumpjumper 15 Comp](https://www.specialized.com/us/en/stumpjumper-15-comp-sram-s-1000-axs-fox-performance/p/4221395) | +| Specialized Enduro Pro | [Enduro Pro](https://www.specialized.com/us/en/enduro-pro/p/4276010) | +| Specialized Epic 8 Pro | [Epic 8 Pro](https://www.specialized.com/us/en/epic-8-pro-sram-x0-axs-rockshox-ultimate/p/220874) | +| Specialized Chisel Comp | [Chisel Hardtail Comp](https://www.specialized.com/us/en/chisel-hardtail-comp/p/4221772) and [Chisel Comp](https://www.specialized.com/us/en/chisel-comp/p/4221778); variant ambiguity described above | +| Specialized Status 140 | [Status 140](https://www.specialized.com/us/en/status-140/p/199765); exact generation not asserted | +| Giant Trance X Advanced Pro 29 | [Trance X Advanced family](https://www.giant-bicycles.com/global/showcase/trance-x-advanced); exact build not asserted | +| Giant Anthem Advanced 29 1 | [Anthem Advanced 29 family](https://www.giant-bicycles.com/global/showcase/anthem-advanced-29); exact build not asserted | +| Giant Talon 2 | [Talon family](https://www.giant-bicycles.com/us/talon); exact build not asserted | +| Shimano Deore XT M8100 Rear Derailleur | [Shimano RD-M8100-SGS](https://bike.shimano.com/en-NA/products/components/pdp.P-RD-M8100-SGS.html) | +| Shimano SLX M7100 12-speed Cassette | [Shimano CS-M7100-12](https://bike.shimano.com/products/components/pdp.P-CS-M7100-12.html); exact gear range not asserted | +| Shimano XTR M9100 Crankset | [Shimano FC-M9100-1](https://bike.shimano.com/products/components/pdp.P-FC-M9100-1.html) | +| SRAM GX Eagle Rear Derailleur | [Mechanical GX Eagle](https://www.sram.com/en/sram/models/rd-gx-1-b2) | +| SRAM X0 Transmission Rear Derailleur | [X0 Eagle Transmission](https://www.sram.com/en/sram/models/rd-x0-e-b1) | +| SRAM XX1 Eagle Cassette | [SRAM XG-1299](https://www.sram.com/en/sram/models/cs-xg-1299-a2); exact gear range not asserted | +| OneUp Dropper Post V3 180mm | [Dropper Post V3](https://www.oneupcomponents.com/products/dropper-post-v3) | +| OneUp Carbon Handlebar | [V2 Carbon Handlebar](https://www.oneupcomponents.com/products/v2-carbon-handlebar); description avoids version-specific measurements | +| OneUp Composite Pedals | [Composite Pedals](https://www.oneupcomponents.com/products/comp-pedal) | +| OneUp EDC Tool V2 | [EDC V2 Tool](https://www.oneupcomponents.com/products/edc-v2-tool) | +| Muc-Off Nano Tech Bike Cleaner | [Muc-Off Bike Cleaner](https://muc-off.com/products/bike-cleaner) | +| Muc-Off MTB Tubeless Sealant | [Muc-Off MTB Tubeless Sealant](https://muc-off.com/products/mtb-tubeless-sealant) | +| Muc-Off Drivetrain Cleaner | [Drivetrain Cleaner retailer listing](https://www.bikefreeus.com/products/muc-off-drivetrain-cleaner) | +| Labour | Authored sample description of hourly workshop maintenance/repair; not a manufacturer product | + +Keep these sources and descriptions aligned when changing the fixtures. A product-family page is not a licence to invent build-specific claims, and manufacturer pages can change over time. + ### Keep rapidly changing facts relational Generally exclude price, current stock, and activity status from embeddings. Retrieve or filter these from live product data. @@ -493,11 +550,11 @@ The document builder reads labels through the contract's `Brand`, `Category` and ### First real-model finding -The local embedding endpoint works, but the initial semantic ranking check does **not** yet pass. For "something to clean mud off my bike", Nomic ranked the Yeti bike above the bike cleaner when given current short names, category/brand labels and tags. It also ranked the Yeti bike above the Shimano derailleur for "a component for changing gears". Even the name-only probe did not reliably favor the cleaner. +Before descriptive content was added, the local embedding endpoint worked but the initial semantic ranking check did **not** pass. For "something to clean mud off my bike", Nomic ranked the Yeti bike above the bike cleaner when given short names, category/brand labels and tags. It also ranked the Yeti bike above the Shimano derailleur for "a component for changing gears". Even the name-only probe did not reliably favor the cleaner. -This is a useful counterexample to "embeddings work, therefore search is good". The failures are recorded rather than hidden by changing the queries or weakening the expected results. `tools/test-product-embeddings.ps1` evaluates both examples before reporting any failures. It calculates exact cosine distances in memory; it is not pgvector retrieval, a full-catalog evaluation, or evidence about index performance. +This is a useful counterexample to "embeddings work, therefore search is good". The failures are recorded rather than hidden by changing the queries or weakening the expected results. `tools/test-product-embeddings.ps1` retains the original queries and expected products, evaluates both examples before reporting failures, and now includes the seeded descriptions. It calculates exact cosine distances in memory; it is not pgvector retrieval, a full-catalog evaluation, or evidence about index performance. -Possible next steps are verified descriptive enrichment, a deliberately evaluated hybrid retrieval policy, or comparison with another model. These are explicit relevance decisions; none has been silently applied to make the example pass. +Possible remedies included verified descriptive enrichment, a deliberately evaluated hybrid retrieval policy, or comparison with another model. The subsequent implementation selects descriptive enrichment, retaining the model and original expectations. ### Controlled document-content experiment @@ -524,7 +581,9 @@ Additional controls used an independent Python standard-library HTTP client and This is evidence that document content materially affects this configuration, and that the model can separate these functional descriptions. It is **not** proof that sparse text is the only cause, that the model is optimal, or that an eight-query/three-product sample predicts full-catalog quality. The whole-bike-versus-parts query had a particularly small winning margin, despite passing. There was no held-out evaluation set, and descriptions were designed with the task in mind. -The next content decision is whether to introduce authoritative product descriptions or another curated source of functional text, followed by broader evaluation. The actual document builder and original smoke expectations remain unchanged by this experiment; pgvector cannot supply meaning absent from its input vectors. +The experiment did not modify catalog data or change the original smoke expectations. The subsequent implementation adds researched product descriptions and includes them in `product-document-v2`; broader evaluation still matters. pgvector cannot supply meaning absent from its input vectors. + +With the researched descriptions, the original smoke examples now pass: the cleaner ranks first for the cleaning query (cosine distance **0.2958**) and the Shimano derailleur ranks first for the gear-changing query (**0.3442**). This is still a three-product illustration, not proof of full-catalog relevance. Measure document and query embedding separately. Caching repeated query embeddings can help, but keys must include the profile and relevant preprocessing; user text may be sensitive, so retention/logging needs care. diff --git a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs index ab5e816a5..4e53a82cd 100644 --- a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs +++ b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs @@ -4,10 +4,10 @@ namespace Contoso.Mcp.Tools; public static class CommerceTools { /// Searches products by free-text keywords. - [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products using free-text keywords; each word must appear in the product description or be the beginning of the product SKU. Returns up to the requested number of matching product summaries.")] + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products using free-text keywords; each word must appear in the product name or be the beginning of the product SKU. Returns up to the requested number of matching product summaries.")] public static Task SearchProductsAsync( [FromServices] IHttpClientFactory clientFactory, - [Description("One or more keywords to find in product descriptions, or the beginning of a product SKU.")] string searchTerm, + [Description("One or more keywords to find in product names, or the beginning of a product SKU.")] string searchTerm, [Description("Maximum number of products to return (1-25).")] int take = 10, CancellationToken cancellationToken = default) { diff --git a/samples/src/Contoso.Products.Application/Search/ProductSearchDocument.cs b/samples/src/Contoso.Products.Application/Search/ProductSearchDocument.cs index e0985be65..e7c350f69 100644 --- a/samples/src/Contoso.Products.Application/Search/ProductSearchDocument.cs +++ b/samples/src/Contoso.Products.Application/Search/ProductSearchDocument.cs @@ -8,7 +8,7 @@ namespace Contoso.Products.Application.Search; public sealed record ProductSearchDocument(string Text, string SourceHash) { /// Gets the template version; changes to the document recipe require a new version. - public const string TemplateVersion = "product-document-v1"; + public const string TemplateVersion = "product-document-v2"; /// /// Creates a document using the product's reference-data labels, falling back to codes where labels are unavailable. @@ -23,6 +23,7 @@ public static ProductSearchDocument Create(ProductBase product) var text = string.Join('\n', $"Name: {name}", $"SKU: {sku}", + $"Description: {Normalize(product.Description)}", $"Brand: {LabelOrCode(product.Brand?.Text, product.BrandCode)}", $"Category: {LabelOrCode(product.Category?.Text, product.CategoryCode)}", $"Subcategory: {LabelOrCode(product.SubCategory?.Text, product.SubCategoryCode)}", diff --git a/samples/src/Contoso.Products.Application/Validators/ProductValidator.cs b/samples/src/Contoso.Products.Application/Validators/ProductValidator.cs index 903282da5..8ff83b206 100644 --- a/samples/src/Contoso.Products.Application/Validators/ProductValidator.cs +++ b/samples/src/Contoso.Products.Application/Validators/ProductValidator.cs @@ -6,8 +6,9 @@ public ProductValidator() { Property(p => p.Sku).Mandatory().MaximumLength(50); Property(p => p.Text).Mandatory().MaximumLength(250); + Property(p => p.Description).MaximumLength(2000); Property(p => p.SubCategory).Mandatory().IsValid(); Property(p => p.UnitOfMeasure).Mandatory().IsValid(); Property(p => p.Price).PrecisionScale(null, 2).GreaterThanOrEqualTo(0, _ => "zero"); } -} \ No newline at end of file +} diff --git a/samples/src/Contoso.Products.Contracts/ProductBase.cs b/samples/src/Contoso.Products.Contracts/ProductBase.cs index 6f5e50665..4c8ae8fd9 100644 --- a/samples/src/Contoso.Products.Contracts/ProductBase.cs +++ b/samples/src/Contoso.Products.Contracts/ProductBase.cs @@ -12,6 +12,9 @@ public abstract partial class ProductBase : IIdentifier [NonNullable] public string? Text { get; set; } + /// Gets or sets the descriptive product information and intended use. + public string? Description { get; set; } + [ReadOnly(true)] [ReferenceData] public partial string? CategoryCode { get; set; } @@ -37,4 +40,4 @@ public abstract partial class ProductBase : IIdentifier [ReadOnly(true)] public bool IsInactive { get; set; } -} \ No newline at end of file +} diff --git a/samples/src/Contoso.Products.Database/Migrations/20261013-174408-add-product-description.pgsql b/samples/src/Contoso.Products.Database/Migrations/20261013-174408-add-product-description.pgsql new file mode 100644 index 000000000..a18dd1e56 --- /dev/null +++ b/samples/src/Contoso.Products.Database/Migrations/20261013-174408-add-product-description.pgsql @@ -0,0 +1,4 @@ +-- Adds optional descriptive product text without changing existing rows. + +ALTER TABLE "products"."product" + ADD "description" VARCHAR(2000) NULL; diff --git a/samples/src/Contoso.Products.Infrastructure/Mapping/ProductMapper.cs b/samples/src/Contoso.Products.Infrastructure/Mapping/ProductMapper.cs index 23fc2ff90..cfb66a495 100644 --- a/samples/src/Contoso.Products.Infrastructure/Mapping/ProductMapper.cs +++ b/samples/src/Contoso.Products.Infrastructure/Mapping/ProductMapper.cs @@ -7,6 +7,7 @@ public class ProductMapper : BiDirectionMapper p.CategoryCode = p.SubCategory?.CategoryCode); -} \ No newline at end of file +} diff --git a/samples/src/Contoso.Products.Infrastructure/Persistence/Product.g.cs b/samples/src/Contoso.Products.Infrastructure/Persistence/Product.g.cs index 0b66e7e75..8a10626d7 100644 --- a/samples/src/Contoso.Products.Infrastructure/Persistence/Product.g.cs +++ b/samples/src/Contoso.Products.Infrastructure/Persistence/Product.g.cs @@ -39,8 +39,11 @@ public partial class Product : ModelBase, ILogicallyDeleted /// Gets or sets the value of the 'tags_json' column (type 'JSONB NULL'). public List? Tags { get; set; } + /// Gets or sets the value of the 'description' column (type 'CHARACTER VARYING(2000) NULL'). + public string? Description { get; set; } + /// Gets or sets the value of the 'is_deleted' column (type 'BOOLEAN'); see . public bool IsDeleted { get; set; } } -#nullable restore \ No newline at end of file +#nullable restore diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs index 39e5e7b3d..c3f734bce 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs @@ -13,6 +13,7 @@ public ProductQueryArgsConfig() .WithDefaultModelPrefix("Product") .AddField(nameof(Contracts.ProductBase.Sku), c => c.WithOperators(QueryFilterOperator.EqualityOperators | QueryFilterOperator.StartsWith).AsUpperCase()) .AddField(nameof(Contracts.ProductBase.Text), c => c.WithOperators(QueryFilterOperator.StringFunctions).AsUpperCase()) + .AddField(nameof(Contracts.ProductBase.Price), c => c.WithOperators(QueryFilterOperator.ComparisonOperators)) .AddReferenceDataField(nameof(Contracts.ProductBase.Category), "CategoryCode", c => c.WithNoModelPrefix()) .AddReferenceDataField(nameof(Contracts.ProductBase.SubCategory), "SubCategoryCode") .AddReferenceDataField(nameof(Contracts.ProductBase.Brand), "BrandCode")); diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs index 8f4a164a7..4bcc7b21d 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs @@ -45,6 +45,7 @@ from i in ig.DefaultIfEmpty() Id = x.Product.Id, Sku = x.Product.Sku, Text = x.Product.Text, + Description = x.Product.Description, CategoryCode = x.CategoryCode, SubCategoryCode = x.SubCategoryCode, BrandCode = x.Product.BrandCode, diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductsDbContext.g.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductsDbContext.g.cs index 67c215ae0..4d2e47de6 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductsDbContext.g.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductsDbContext.g.cs @@ -169,6 +169,7 @@ partial void AddGeneratedModels(Microsoft.EntityFrameworkCore.ModelBuilder model e.Property(p => p.IsInactive).HasColumnName("is_inactive").HasColumnType("BOOLEAN"); e.Property(p => p.IsNonStocked).HasColumnName("is_non_stocked").HasColumnType("BOOLEAN"); e.Property(p => p.Tags).HasColumnName("tags_json").HasColumnType("JSONB").HasConversion(TypeToJsonStringEfConverter?>.Default, TypeToJsonStringEfComparer?>.Default); + e.Property(p => p.Description).HasColumnName("description").HasColumnType("CHARACTER VARYING(2000)"); e.Property(p => p.CreatedBy).HasColumnName("created_by").HasColumnType("CHARACTER VARYING(250)").ValueGeneratedOnUpdate(); e.Property(p => p.CreatedOn).HasColumnName("created_on").HasColumnType("TIMESTAMP WITH TIME ZONE").ValueGeneratedOnUpdate(); e.Property(p => p.UpdatedBy).HasColumnName("updated_by").HasColumnType("CHARACTER VARYING(250)").ValueGeneratedOnAdd(); diff --git a/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Create.cs b/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Create.cs index fece6aa52..6737eacbf 100644 --- a/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Create.cs +++ b/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Create.cs @@ -70,6 +70,7 @@ public void Create_Success() { Sku = "New-SKU-123", Text = "New Product", + Description = "A carbon cross-country mountain bicycle for trail riding.", Price = 1000M, SubCategoryCode = "XC", UnitOfMeasureCode = "EA", @@ -88,6 +89,8 @@ public void Create_Success() .AssertLocationHeader(r => new Uri($"/api/products/{r!.Id}", UriKind.Relative)) .Value!; + r.Description.Should().Be(p.Description); + // Assert. Test.Http() .Run(HttpMethod.Get, $"/api/products/{r.Id}") diff --git a/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Patch.cs b/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Patch.cs index be5fc40aa..d5adc2849 100644 --- a/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Patch.cs +++ b/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Patch.cs @@ -56,16 +56,18 @@ public void Patch_Success() .Value!; p.Text += " Patched"; + p.Description = "A full-suspension carbon mountain bicycle for technical trail riding."; // Act/Assert. var u = Test.Http() .ExpectPostgresOutboxEvents(e => e.AssertWithValue("contoso", "contoso.products.product.updated.v1")) - .Run(HttpMethod.Patch, $"/api/products/{p.Id}", new { text = p.Text }, requestModifier: r => r.WithIfMatch(p.ETag).WithMergePatchJsonContentType()) + .Run(HttpMethod.Patch, $"/api/products/{p.Id}", new { text = p.Text, description = p.Description }, requestModifier: r => r.WithIfMatch(p.ETag).WithMergePatchJsonContentType()) .AssertOK() .AssertValue(p, "etag", "changelog") .Value!; u.Text.Should().Be(p.Text); + u.Description.Should().Be(p.Description); u.ETag.Should().NotBe(p.ETag); // Assert. @@ -75,6 +77,34 @@ public void Patch_Success() .AssertValue(u, "etag", "changelog"); } + [Test] + public void Patch_ClearDescription() + { + var product = Test.Http() + .Run(HttpMethod.Get, $"/api/products/{30.ToGuid()}") + .AssertOK() + .Value!; + product.Description.Should().NotBeNullOrEmpty(); + + var updated = Test.Http() + .ExpectPostgresOutboxEvents(e => e.AssertWithValue("contoso", "contoso.products.product.updated.v1")) + .Run(HttpMethod.Patch, $"/api/products/{product.Id}", requestModifier: r => + { + r.Content = new StringContent("{\"description\":null}", System.Text.Encoding.UTF8, "application/merge-patch+json"); + r.WithIfMatch(product.ETag); + }) + .AssertOK() + .Value!; + + updated.Description.Should().BeNull(); + updated.Text.Should().Be(product.Text); + updated.ETag.Should().NotBe(product.ETag); + Test.Http() + .Run(HttpMethod.Get, $"/api/products/{product.Id}") + .AssertOK() + .AssertValue(updated); + } + [Test] public void Patch_NoChanges() { @@ -104,4 +134,4 @@ public void Patch_NoChanges() .AssertOK() .AssertValue(u, "etag", "changelog"); } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Update.cs b/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Update.cs index e8512616a..e334f816f 100644 --- a/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Update.cs +++ b/samples/tests/Contoso.Products.Test.Api/ProductMutateTests.Update.cs @@ -44,6 +44,7 @@ public void Update_Success() .Value!; p.Text += " Updated"; + p.Description = "A full-suspension carbon mountain bicycle for cross-country trails."; // Act/Assert. var u = Test.Http() @@ -57,6 +58,7 @@ public void Update_Success() .Value!; u.Text.Should().Be(p.Text); + u.Description.Should().Be(p.Description); u.ETag.Should().NotBe(p.ETag); // Assert. @@ -66,6 +68,31 @@ public void Update_Success() .AssertValue(u); } + [Test] + public void Update_DescriptionOnly() + { + var product = Test.Http() + .Run(HttpMethod.Get, $"/api/products/{9.ToGuid()}") + .AssertOK() + .Value!; + var etag = product.ETag; + product.Description = "A carbon mixed-wheel mountain bicycle for bike-park jumps and steep descents."; + + var updated = Test.Http() + .ExpectChangeLogUpdated() + .ExpectPostgresOutboxEvents(e => e.AssertWithValue("contoso", "contoso.products.product.updated.v1")) + .Run(HttpMethod.Put, $"/api/products/{product.Id}", product) + .AssertOK() + .Value!; + + updated.Description.Should().Be(product.Description); + updated.ETag.Should().NotBe(etag); + Test.Http() + .Run(HttpMethod.Get, $"/api/products/{product.Id}") + .AssertOK() + .AssertValue(updated); + } + [Test] public void Update_NoChanges() { @@ -94,4 +121,4 @@ public void Update_NoChanges() .AssertOK() .AssertValue(u); } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductGraphQLLite.cs b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductGraphQLLite.cs index 203176c8c..286fac21d 100644 --- a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductGraphQLLite.cs +++ b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductGraphQLLite.cs @@ -59,6 +59,33 @@ public void GraphQLLite_Product_Get_Success() product.GetProperty("text").GetString().Should().Be("Yeti ASR C2"); } + [Test] + public void GraphQLLite_Products_PriceFilterAndDescription_MatchRest() + { + var graphql = Test.Http() + .Run(HttpMethod.Post, "/api/query", new + { + query = "{ products(where: {price: {ge: 99, lt: 130}}) { edges { node { sku price description } } } }" + }) + .AssertOK() + .Value; + graphql.TryGetProperty("errors", out _).Should().BeFalse(); + + var edges = graphql.GetProperty("data").GetProperty("products").GetProperty("edges"); + var rest = Test.Http() + .Run(HttpMethod.Get, "/api/products?$filter=price ge 99 and price lt 130") + .AssertOK() + .Value!; + edges.GetArrayLength().Should().Be(rest.Length).And.Be(2); + for (var i = 0; i < rest.Length; i++) + { + var node = edges[i].GetProperty("node"); + node.GetProperty("sku").GetString().Should().Be(rest[i].Sku); + node.GetProperty("price").GetDecimal().Should().Be(rest[i].Price); + node.GetProperty("description").GetString().Should().Be(rest[i].Description); + } + } + [Test] public void GraphQLLite_Product_Get_NotFound() { diff --git a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs index 4ce25c479..e48c12777 100644 --- a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs +++ b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs @@ -138,6 +138,45 @@ public void Product_Query_Search_WithFilterAndOrderBy() .And.BeInDescendingOrder(x => x.Text); } + [Test] + public void Product_Query_FilterByPriceRange() + { + var result = Test.Http() + .Run(HttpMethod.Get, "/api/products?$filter=price ge 99 and price lt 130&$count=true") + .AssertOK(); + + result.Value.Should().NotBeNull().And.HaveCount(2) + .And.OnlyContain(p => p.Price >= 99 && p.Price < 130); + result.Value!.Select(p => p.Sku).Should().BeEquivalentTo(["SHIMANO-SLX-M7100-CASSETTE", "SHIMANO-XT-M8100-RD"]); + result.Response.Headers.Should().ContainKey("X-Paging-Total-Count").WhoseValue.Should().ContainSingle().Which.Should().Be("2"); + } + + [TestCase("price eq 16.99")] + [TestCase("price in (16.99, 17.50)")] + [TestCase("price gt 16.98 and price le 16.99")] + [TestCase("brand eq 'MUCOFF' and price ne 8 and price ne 25")] + public void Product_Query_FilterByDecimalPrice(string filter) + { + var products = Test.Http() + .Run(HttpMethod.Get, $"/api/products?$filter={Uri.EscapeDataString(filter)}") + .AssertOK() + .Value!; + + products.Should().ContainSingle().Which.Sku.Should().Be("MUCOFF-NANO-CLEANER"); + } + + [Test] + public void Product_Query_Search_WithPriceFilter() + { + var products = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=cleaner&$filter=price lt 20") + .AssertOK() + .Value!; + + products.Should().ContainSingle().Which.Sku.Should().Be("MUCOFF-NANO-CLEANER"); + products[0].Description.Should().NotBeNullOrEmpty(); + } + [Test] public void Product_Query_Search_TooManyTerms() { diff --git a/samples/tests/Contoso.Products.Test.Api/Resources/ProductMutateTests/Create_Success.res.json b/samples/tests/Contoso.Products.Test.Api/Resources/ProductMutateTests/Create_Success.res.json index 3c638de0c..915fbe2b2 100644 --- a/samples/tests/Contoso.Products.Test.Api/Resources/ProductMutateTests/Create_Success.res.json +++ b/samples/tests/Contoso.Products.Test.Api/Resources/ProductMutateTests/Create_Success.res.json @@ -2,6 +2,7 @@ "id": "140d1f58-7927-47e4-a775-618b6516e2ab", "sku": "NEW-SKU-123", "text": "New Product", + "description": "A carbon cross-country mountain bicycle for trail riding.", "category": "B", "subCategory": "XC", "unitOfMeasure": "EA", @@ -13,4 +14,4 @@ "createdBy": "METACORTEX\\thomas.anderson", "createdOn": "2026-01-18T14:10:09.3076098\u002B00:00" } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Get_Found.res.json b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Get_Found.res.json index 25054d0fb..49cec43bf 100644 --- a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Get_Found.res.json +++ b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Get_Found.res.json @@ -2,6 +2,7 @@ "id": "00000001-0000-0000-0000-000000000000", "sku": "YETI-ASR-C2-2025", "text": "Yeti ASR C2", + "description": "A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack.", "category": "B", "subCategory": "XC", "unitOfMeasure": "EA", @@ -13,4 +14,4 @@ "createdBy": "METACORTEX\\thomas.anderson", "createdOn": "2026-01-18T00:40:12.697994+00:00" } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json index f308c3f91..9de99dca1 100644 --- a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json +++ b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json @@ -18,6 +18,19 @@ "endswith" ] }, + "price": { + "type": "number", + "format": "decimal", + "operators": [ + "eq", + "ne", + "lt", + "le", + "ge", + "gt", + "in" + ] + }, "category": { "type": "string", "operators": [ @@ -73,4 +86,4 @@ "maxLength": 200, "maxTerms": 8 } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.Products.Test.Common/Data/mutate-data.seed.yaml b/samples/tests/Contoso.Products.Test.Common/Data/mutate-data.seed.yaml index 8d3ac5825..0edc5a1da 100644 --- a/samples/tests/Contoso.Products.Test.Common/Data/mutate-data.seed.yaml +++ b/samples/tests/Contoso.Products.Test.Common/Data/mutate-data.seed.yaml @@ -1,37 +1,37 @@ products: - product: - - { product_id: ^1, sku: YETI-ASR-C2-2025, text: Yeti ASR C2, sub_category_code: XC, unit_of_measure_code: EA, price: 5800, brand_code: YETI, tags_json: '["cross-country","full-suspension"]' } - - { product_id: ^2, sku: YETI-SB120-C2, text: Yeti SB120 C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: YETI } - - { product_id: ^3, sku: YETI-SB140-LR-C2, text: Yeti SB140 LR C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6600, brand_code: YETI } - - { product_id: ^4, sku: YETI-SB165-C3, text: Yeti SB165 C3, sub_category_code: EN, unit_of_measure_code: EA, price: 7200, brand_code: YETI } - - { product_id: ^5, sku: YETI-MTE-C2, text: Yeti MTe C2 eMTB, sub_category_code: EM, unit_of_measure_code: EA, price: 9850, brand_code: YETI } - - { product_id: ^6, sku: CANYON-LUXTRAIL-CF6, text: Canyon Lux Trail CF 6, sub_category_code: XC, unit_of_measure_code: EA, price: 3899, brand_code: CANYON } - - { product_id: ^7, sku: CANYON-SPECTRAL-CF8, text: Canyon Spectral CF 8, sub_category_code: TR, unit_of_measure_code: EA, price: 4999, brand_code: CANYON } - - { product_id: ^8, sku: CANYON-NEURON-CF7, text: Canyon Neuron CF 7, sub_category_code: TR, unit_of_measure_code: EA, price: 3499, brand_code: CANYON } - - { product_id: ^9, sku: CANYON-TORQUE-MULLET-CF8, text: Canyon Torque Mullet CF 8, sub_category_code: EN, unit_of_measure_code: EA, price: 3599, brand_code: CANYON } - - { product_id: ^10, sku: CANYON-SENDER-CFR, text: Canyon Sender CFR, sub_category_code: DH, unit_of_measure_code: EA, price: 7799, brand_code: CANYON } - - { product_id: ^11, sku: SPEC-STUMPJUMPER-15-COMP, text: Specialized Stumpjumper 15 Comp, sub_category_code: TR, unit_of_measure_code: EA, price: 5500, brand_code: SPEC } - - { product_id: ^12, sku: SPEC-ENDURO-PRO, text: Specialized Enduro Pro, sub_category_code: EN, unit_of_measure_code: EA, price: 9500, brand_code: SPEC } - - { product_id: ^13, sku: SPEC-EPIC-8-PRO, text: Specialized Epic 8 Pro, sub_category_code: XC, unit_of_measure_code: EA, price: 9000, is_inactive: true, brand_code: SPEC } - - { product_id: ^14, sku: SPEC-CHISEL-COMP, text: Specialized Chisel Comp, sub_category_code: HT, unit_of_measure_code: EA, price: 3400, brand_code: SPEC } - - { product_id: ^15, sku: SPEC-STATUS-140, text: Specialized Status 140, sub_category_code: FR, unit_of_measure_code: EA, price: 2499, brand_code: SPEC } - - { product_id: ^16, sku: GIANT-TRANCE-X-ADVPRO29, text: Giant Trance X Advanced Pro 29, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: GIANT } - - { product_id: ^17, sku: GIANT-ANTHEM-ADV29-1, text: Giant Anthem Advanced 29 1, sub_category_code: XC, unit_of_measure_code: EA, price: 7800, brand_code: GIANT } - - { product_id: ^18, sku: GIANT-TALON-2, text: Giant Talon 2, sub_category_code: HT, unit_of_measure_code: EA, price: 800, is_deleted: true, brand_code: GIANT } - - { product_id: ^19, sku: SHIMANO-XT-M8100-RD, text: Shimano Deore XT M8100 Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 129, brand_code: SHIMANO } - - { product_id: ^20, sku: SHIMANO-SLX-M7100-CASSETTE, text: Shimano SLX M7100 12-speed Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 99, brand_code: SHIMANO } - - { product_id: ^21, sku: SHIMANO-XTR-M9100-CRANK, text: Shimano XTR M9100 Crankset, sub_category_code: CR, unit_of_measure_code: EA, price: 499, brand_code: SHIMANO } - - { product_id: ^22, sku: SRAM-GX-EAGLE-RD, text: SRAM GX Eagle Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 135, brand_code: SRAM } - - { product_id: ^23, sku: SRAM-X0-TRANSMISSION-RD, text: SRAM X0 Transmission Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 400, brand_code: SRAM } - - { product_id: ^24, sku: SRAM-XX1-EAGLE-CASSETTE, text: SRAM XX1 Eagle Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 449, brand_code: SRAM } - - { product_id: ^25, sku: ONEUP-V3-DROPPER-180, text: OneUp Dropper Post V3 180mm, sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP } - - { product_id: ^26, sku: ONEUP-CARBON-HANDLEBAR, text: OneUp Carbon Handlebar, sub_category_code: HB, unit_of_measure_code: EA, price: 159, brand_code: ONEUP } - - { product_id: ^27, sku: ONEUP-COMPOSITE-PEDALS, text: OneUp Composite Pedals, sub_category_code: PE, unit_of_measure_code: PR, price: 49, is_inactive: true, brand_code: ONEUP } - - { product_id: ^28, sku: ONEUP-EDC-TOOL-V2, text: OneUp EDC Tool V2, sub_category_code: TO, unit_of_measure_code: EA, price: 59, brand_code: ONEUP } - - { product_id: ^29, sku: MUCOFF-NANO-CLEANER, text: Muc-Off Nano Tech Bike Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 16.99, brand_code: MUCOFF } - - { product_id: ^30, sku: MUCOFF-TUBELESS-SEALANT, text: Muc-Off MTB Tubeless Sealant, sub_category_code: SE, unit_of_measure_code: EA, price: 8.00, brand_code: MUCOFF } - - { product_id: ^31, sku: MUCOFF-DRIVETRAIN-CLEANER, text: Muc-Off Drivetrain Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 25.00, brand_code: MUCOFF } - - { product_id: ^32, sku: LABOR, text: Labor, sub_category_code: LA, unit_of_measure_code: HR, price: 80.00, brand_code: NONE, is_non_stocked: true } + - { product_id: ^1, sku: YETI-ASR-C2-2025, text: Yeti ASR C2, sub_category_code: XC, unit_of_measure_code: EA, price: 5800, brand_code: YETI, tags_json: '["cross-country","full-suspension"]', description: 'A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack.' } + - { product_id: ^2, sku: YETI-SB120-C2, text: Yeti SB120 C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: YETI, description: 'A carbon full-suspension trail mountain bike designed for efficient climbs, responsive handling and flowing singletrack descents.' } + - { product_id: ^3, sku: YETI-SB140-LR-C2, text: Yeti SB140 LR C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6600, brand_code: YETI, description: 'A carbon full-suspension trail mountain bike in the Lunch Ride configuration, built for technical climbs and rough, fast descents.' } + - { product_id: ^4, sku: YETI-SB165-C3, text: Yeti SB165 C3, sub_category_code: EN, unit_of_measure_code: EA, price: 7200, brand_code: YETI, description: 'A carbon full-suspension mountain bike for gravity-focused riding, steep descents and demanding bike-park terrain.' } + - { product_id: ^5, sku: YETI-MTE-C2, text: Yeti MTe C2 eMTB, sub_category_code: EM, unit_of_measure_code: EA, price: 9850, brand_code: YETI, description: 'An electric-assist mountain bike with a carbon full-suspension frame, designed for technical trail riding and assisted climbs.' } + - { product_id: ^6, sku: CANYON-LUXTRAIL-CF6, text: Canyon Lux Trail CF 6, sub_category_code: XC, unit_of_measure_code: EA, price: 3899, brand_code: CANYON, description: 'A carbon full-suspension mountain bike combining cross-country pedalling efficiency with trail capability for long rides and technical singletrack.' } + - { product_id: ^7, sku: CANYON-SPECTRAL-CF8, text: Canyon Spectral CF 8, sub_category_code: TR, unit_of_measure_code: EA, price: 4999, brand_code: CANYON, description: 'A carbon full-suspension trail mountain bike for varied singletrack, balancing climbing capability with control on rough, technical descents.' } + - { product_id: ^8, sku: CANYON-NEURON-CF7, text: Canyon Neuron CF 7, sub_category_code: TR, unit_of_measure_code: EA, price: 3499, brand_code: CANYON, description: 'A carbon full-suspension trail mountain bike for long rides over varied terrain, combining efficient climbing with stable descending.' } + - { product_id: ^9, sku: CANYON-TORQUE-MULLET-CF8, text: Canyon Torque Mullet CF 8, sub_category_code: EN, unit_of_measure_code: EA, price: 3599, brand_code: CANYON, description: 'A carbon mixed-wheel mountain bike designed for freeride and bike-park use, including jumps, drops and steep descents.' } + - { product_id: ^10, sku: CANYON-SENDER-CFR, text: Canyon Sender CFR, sub_category_code: DH, unit_of_measure_code: EA, price: 7799, brand_code: CANYON, description: 'A carbon downhill mountain bike designed for racing and fast descending on steep, demanding tracks and bike-park terrain.' } + - { product_id: ^11, sku: SPEC-STUMPJUMPER-15-COMP, text: Specialized Stumpjumper 15 Comp, sub_category_code: TR, unit_of_measure_code: EA, price: 5500, brand_code: SPEC, description: 'A full-suspension trail mountain bike with adjustable geometry, designed to balance climbing efficiency and control on technical descents.' } + - { product_id: ^12, sku: SPEC-ENDURO-PRO, text: Specialized Enduro Pro, sub_category_code: EN, unit_of_measure_code: EA, price: 9500, brand_code: SPEC, description: 'A carbon full-suspension enduro mountain bike built for steep, rough descents and enduro racing while retaining the ability to pedal uphill.' } + - { product_id: ^13, sku: SPEC-EPIC-8-PRO, text: Specialized Epic 8 Pro, sub_category_code: XC, unit_of_measure_code: EA, price: 9000, is_inactive: true, brand_code: SPEC, description: 'A carbon full-suspension cross-country mountain bike designed for efficient pedalling, competitive racing and technical singletrack.' } + - { product_id: ^14, sku: SPEC-CHISEL-COMP, text: Specialized Chisel Comp, sub_category_code: HT, unit_of_measure_code: EA, price: 3400, brand_code: SPEC, description: 'An aluminium cross-country mountain bike designed for efficient pedalling, racing and fast singletrack riding.' } + - { product_id: ^15, sku: SPEC-STATUS-140, text: Specialized Status 140, sub_category_code: FR, unit_of_measure_code: EA, price: 2499, brand_code: SPEC, description: 'An aluminium mixed-wheel full-suspension mountain bike designed for playful trail riding, jumps and technical descents.' } + - { product_id: ^16, sku: GIANT-TRANCE-X-ADVPRO29, text: Giant Trance X Advanced Pro 29, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: GIANT, description: 'A carbon full-suspension trail mountain bike with 29-inch wheels and adjustable geometry, designed for climbing and technical descending.' } + - { product_id: ^17, sku: GIANT-ANTHEM-ADV29-1, text: Giant Anthem Advanced 29 1, sub_category_code: XC, unit_of_measure_code: EA, price: 7800, brand_code: GIANT, description: 'A carbon full-suspension cross-country mountain bike with 29-inch wheels, designed for efficient pedalling and fast riding on demanding race courses.' } + - { product_id: ^18, sku: GIANT-TALON-2, text: Giant Talon 2, sub_category_code: HT, unit_of_measure_code: EA, price: 800, is_deleted: true, brand_code: GIANT, description: 'An aluminium hardtail mountain bike with a suspension fork, designed as an accessible introduction to off-road trails.' } + - { product_id: ^19, sku: SHIMANO-XT-M8100-RD, text: Shimano Deore XT M8100 Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 129, brand_code: SHIMANO, description: 'A 12-speed mountain bike rear derailleur that moves the chain between cassette sprockets to change gears. Its clutch helps control chain movement on rough trails.' } + - { product_id: ^20, sku: SHIMANO-SLX-M7100-CASSETTE, text: Shimano SLX M7100 12-speed Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 99, brand_code: SHIMANO, description: 'A 12-speed mountain bike cassette that provides a wide range of gears for climbing and descending. It uses Shimano Micro Spline mounting and Hyperglide Plus shifting.' } + - { product_id: ^21, sku: SHIMANO-XTR-M9100-CRANK, text: Shimano XTR M9100 Crankset, sub_category_code: CR, unit_of_measure_code: EA, price: 499, brand_code: SHIMANO, description: 'A mountain bike crankset that transfers pedalling power to the chain through a direct-mount chainring, designed for lightweight cross-country and trail use.' } + - { product_id: ^22, sku: SRAM-GX-EAGLE-RD, text: SRAM GX Eagle Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 135, brand_code: SRAM, description: 'A cable-operated 12-speed mountain bike rear derailleur for SRAM Eagle drivetrains. It shifts the chain between rear sprockets and uses a clutch to control chain movement.' } + - { product_id: ^23, sku: SRAM-X0-TRANSMISSION-RD, text: SRAM X0 Transmission Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 400, brand_code: SRAM, description: 'A wireless electronic mountain bike rear derailleur for SRAM Eagle Transmission drivetrains. Its hangerless Full Mount design attaches around the rear axle on compatible frames.' } + - { product_id: ^24, sku: SRAM-XX1-EAGLE-CASSETTE, text: SRAM XX1 Eagle Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 449, brand_code: SRAM, description: 'A lightweight 12-speed mountain bike cassette with a wide gear range for climbing and descending, designed for SRAM Eagle drivetrains and XD freehub drivers.' } + - { product_id: ^25, sku: ONEUP-V3-DROPPER-180, text: OneUp Dropper Post V3 180mm, sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP, description: 'A mountain bike dropper seatpost with 180 mm of travel. Riders can lower the saddle for descending and raise it for pedalling using a compatible remote.' } + - { product_id: ^26, sku: ONEUP-CARBON-HANDLEBAR, text: OneUp Carbon Handlebar, sub_category_code: HB, unit_of_measure_code: EA, price: 159, brand_code: ONEUP, description: 'A carbon mountain bike handlebar with a shaped profile designed to absorb vibration and reduce hand fatigue while maintaining steering control.' } + - { product_id: ^27, sku: ONEUP-COMPOSITE-PEDALS, text: OneUp Composite Pedals, sub_category_code: PE, unit_of_measure_code: PR, price: 49, is_inactive: true, brand_code: ONEUP, description: 'Flat mountain bike pedals with composite bodies, broad platforms and replaceable traction pins, designed to grip ordinary flat-soled riding shoes.' } + - { product_id: ^28, sku: ONEUP-EDC-TOOL-V2, text: OneUp EDC Tool V2, sub_category_code: TO, unit_of_measure_code: EA, price: 59, brand_code: ONEUP, description: 'A compact trailside repair multi-tool with hex keys, a Torx key, a chain breaker and a tyre lever, designed for storage in compatible OneUp steerer or pump systems.' } + - { product_id: ^29, sku: MUCOFF-NANO-CLEANER, text: Muc-Off Nano Tech Bike Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 16.99, brand_code: MUCOFF, description: 'A spray-on bicycle cleaner that breaks down dirt and grime on frames and components before rinsing. Suitable for routine washing after muddy rides.' } + - { product_id: ^30, sku: MUCOFF-TUBELESS-SEALANT, text: Muc-Off MTB Tubeless Sealant, sub_category_code: SE, unit_of_measure_code: EA, price: 8.00, brand_code: MUCOFF, description: 'A tubeless mountain bike tyre sealant that helps seal punctures and maintain an airtight tyre system. Its UV-reactive dye helps locate leaks.' } + - { product_id: ^31, sku: MUCOFF-DRIVETRAIN-CLEANER, text: Muc-Off Drivetrain Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 25.00, brand_code: MUCOFF, description: 'A bicycle drivetrain cleaner that removes grease, oil and grime from chains, cassettes, chainrings and derailleurs. It can be sprayed on or used with a chain-cleaning device.' } + - { product_id: ^32, sku: LABOR, text: Labor, sub_category_code: LA, unit_of_measure_code: HR, price: 80.00, brand_code: NONE, is_non_stocked: true, description: 'Workshop labour charged by the hour for bicycle maintenance and repair. This is a service rather than a stocked physical product.' } - inventory: - { inventory_id: ^1, qty_on_hand: 3 } - { inventory_id: ^16, qty_on_hand: 1 } @@ -42,4 +42,4 @@ products: - { movement_id: ^2, product_id: ^6, quantity: -1, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - { movement_id: ^3, product_id: ^31, quantity: -3, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - { movement_id: ^4, product_id: ^24, quantity: -2, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - - { movement_id: ^5, product_id: ^28, quantity: -1, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } \ No newline at end of file + - { movement_id: ^5, product_id: ^28, quantity: -1, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } diff --git a/samples/tests/Contoso.Products.Test.Common/Data/read-data.seed.yaml b/samples/tests/Contoso.Products.Test.Common/Data/read-data.seed.yaml index 8d3ac5825..0edc5a1da 100644 --- a/samples/tests/Contoso.Products.Test.Common/Data/read-data.seed.yaml +++ b/samples/tests/Contoso.Products.Test.Common/Data/read-data.seed.yaml @@ -1,37 +1,37 @@ products: - product: - - { product_id: ^1, sku: YETI-ASR-C2-2025, text: Yeti ASR C2, sub_category_code: XC, unit_of_measure_code: EA, price: 5800, brand_code: YETI, tags_json: '["cross-country","full-suspension"]' } - - { product_id: ^2, sku: YETI-SB120-C2, text: Yeti SB120 C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: YETI } - - { product_id: ^3, sku: YETI-SB140-LR-C2, text: Yeti SB140 LR C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6600, brand_code: YETI } - - { product_id: ^4, sku: YETI-SB165-C3, text: Yeti SB165 C3, sub_category_code: EN, unit_of_measure_code: EA, price: 7200, brand_code: YETI } - - { product_id: ^5, sku: YETI-MTE-C2, text: Yeti MTe C2 eMTB, sub_category_code: EM, unit_of_measure_code: EA, price: 9850, brand_code: YETI } - - { product_id: ^6, sku: CANYON-LUXTRAIL-CF6, text: Canyon Lux Trail CF 6, sub_category_code: XC, unit_of_measure_code: EA, price: 3899, brand_code: CANYON } - - { product_id: ^7, sku: CANYON-SPECTRAL-CF8, text: Canyon Spectral CF 8, sub_category_code: TR, unit_of_measure_code: EA, price: 4999, brand_code: CANYON } - - { product_id: ^8, sku: CANYON-NEURON-CF7, text: Canyon Neuron CF 7, sub_category_code: TR, unit_of_measure_code: EA, price: 3499, brand_code: CANYON } - - { product_id: ^9, sku: CANYON-TORQUE-MULLET-CF8, text: Canyon Torque Mullet CF 8, sub_category_code: EN, unit_of_measure_code: EA, price: 3599, brand_code: CANYON } - - { product_id: ^10, sku: CANYON-SENDER-CFR, text: Canyon Sender CFR, sub_category_code: DH, unit_of_measure_code: EA, price: 7799, brand_code: CANYON } - - { product_id: ^11, sku: SPEC-STUMPJUMPER-15-COMP, text: Specialized Stumpjumper 15 Comp, sub_category_code: TR, unit_of_measure_code: EA, price: 5500, brand_code: SPEC } - - { product_id: ^12, sku: SPEC-ENDURO-PRO, text: Specialized Enduro Pro, sub_category_code: EN, unit_of_measure_code: EA, price: 9500, brand_code: SPEC } - - { product_id: ^13, sku: SPEC-EPIC-8-PRO, text: Specialized Epic 8 Pro, sub_category_code: XC, unit_of_measure_code: EA, price: 9000, is_inactive: true, brand_code: SPEC } - - { product_id: ^14, sku: SPEC-CHISEL-COMP, text: Specialized Chisel Comp, sub_category_code: HT, unit_of_measure_code: EA, price: 3400, brand_code: SPEC } - - { product_id: ^15, sku: SPEC-STATUS-140, text: Specialized Status 140, sub_category_code: FR, unit_of_measure_code: EA, price: 2499, brand_code: SPEC } - - { product_id: ^16, sku: GIANT-TRANCE-X-ADVPRO29, text: Giant Trance X Advanced Pro 29, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: GIANT } - - { product_id: ^17, sku: GIANT-ANTHEM-ADV29-1, text: Giant Anthem Advanced 29 1, sub_category_code: XC, unit_of_measure_code: EA, price: 7800, brand_code: GIANT } - - { product_id: ^18, sku: GIANT-TALON-2, text: Giant Talon 2, sub_category_code: HT, unit_of_measure_code: EA, price: 800, is_deleted: true, brand_code: GIANT } - - { product_id: ^19, sku: SHIMANO-XT-M8100-RD, text: Shimano Deore XT M8100 Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 129, brand_code: SHIMANO } - - { product_id: ^20, sku: SHIMANO-SLX-M7100-CASSETTE, text: Shimano SLX M7100 12-speed Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 99, brand_code: SHIMANO } - - { product_id: ^21, sku: SHIMANO-XTR-M9100-CRANK, text: Shimano XTR M9100 Crankset, sub_category_code: CR, unit_of_measure_code: EA, price: 499, brand_code: SHIMANO } - - { product_id: ^22, sku: SRAM-GX-EAGLE-RD, text: SRAM GX Eagle Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 135, brand_code: SRAM } - - { product_id: ^23, sku: SRAM-X0-TRANSMISSION-RD, text: SRAM X0 Transmission Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 400, brand_code: SRAM } - - { product_id: ^24, sku: SRAM-XX1-EAGLE-CASSETTE, text: SRAM XX1 Eagle Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 449, brand_code: SRAM } - - { product_id: ^25, sku: ONEUP-V3-DROPPER-180, text: OneUp Dropper Post V3 180mm, sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP } - - { product_id: ^26, sku: ONEUP-CARBON-HANDLEBAR, text: OneUp Carbon Handlebar, sub_category_code: HB, unit_of_measure_code: EA, price: 159, brand_code: ONEUP } - - { product_id: ^27, sku: ONEUP-COMPOSITE-PEDALS, text: OneUp Composite Pedals, sub_category_code: PE, unit_of_measure_code: PR, price: 49, is_inactive: true, brand_code: ONEUP } - - { product_id: ^28, sku: ONEUP-EDC-TOOL-V2, text: OneUp EDC Tool V2, sub_category_code: TO, unit_of_measure_code: EA, price: 59, brand_code: ONEUP } - - { product_id: ^29, sku: MUCOFF-NANO-CLEANER, text: Muc-Off Nano Tech Bike Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 16.99, brand_code: MUCOFF } - - { product_id: ^30, sku: MUCOFF-TUBELESS-SEALANT, text: Muc-Off MTB Tubeless Sealant, sub_category_code: SE, unit_of_measure_code: EA, price: 8.00, brand_code: MUCOFF } - - { product_id: ^31, sku: MUCOFF-DRIVETRAIN-CLEANER, text: Muc-Off Drivetrain Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 25.00, brand_code: MUCOFF } - - { product_id: ^32, sku: LABOR, text: Labor, sub_category_code: LA, unit_of_measure_code: HR, price: 80.00, brand_code: NONE, is_non_stocked: true } + - { product_id: ^1, sku: YETI-ASR-C2-2025, text: Yeti ASR C2, sub_category_code: XC, unit_of_measure_code: EA, price: 5800, brand_code: YETI, tags_json: '["cross-country","full-suspension"]', description: 'A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack.' } + - { product_id: ^2, sku: YETI-SB120-C2, text: Yeti SB120 C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: YETI, description: 'A carbon full-suspension trail mountain bike designed for efficient climbs, responsive handling and flowing singletrack descents.' } + - { product_id: ^3, sku: YETI-SB140-LR-C2, text: Yeti SB140 LR C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6600, brand_code: YETI, description: 'A carbon full-suspension trail mountain bike in the Lunch Ride configuration, built for technical climbs and rough, fast descents.' } + - { product_id: ^4, sku: YETI-SB165-C3, text: Yeti SB165 C3, sub_category_code: EN, unit_of_measure_code: EA, price: 7200, brand_code: YETI, description: 'A carbon full-suspension mountain bike for gravity-focused riding, steep descents and demanding bike-park terrain.' } + - { product_id: ^5, sku: YETI-MTE-C2, text: Yeti MTe C2 eMTB, sub_category_code: EM, unit_of_measure_code: EA, price: 9850, brand_code: YETI, description: 'An electric-assist mountain bike with a carbon full-suspension frame, designed for technical trail riding and assisted climbs.' } + - { product_id: ^6, sku: CANYON-LUXTRAIL-CF6, text: Canyon Lux Trail CF 6, sub_category_code: XC, unit_of_measure_code: EA, price: 3899, brand_code: CANYON, description: 'A carbon full-suspension mountain bike combining cross-country pedalling efficiency with trail capability for long rides and technical singletrack.' } + - { product_id: ^7, sku: CANYON-SPECTRAL-CF8, text: Canyon Spectral CF 8, sub_category_code: TR, unit_of_measure_code: EA, price: 4999, brand_code: CANYON, description: 'A carbon full-suspension trail mountain bike for varied singletrack, balancing climbing capability with control on rough, technical descents.' } + - { product_id: ^8, sku: CANYON-NEURON-CF7, text: Canyon Neuron CF 7, sub_category_code: TR, unit_of_measure_code: EA, price: 3499, brand_code: CANYON, description: 'A carbon full-suspension trail mountain bike for long rides over varied terrain, combining efficient climbing with stable descending.' } + - { product_id: ^9, sku: CANYON-TORQUE-MULLET-CF8, text: Canyon Torque Mullet CF 8, sub_category_code: EN, unit_of_measure_code: EA, price: 3599, brand_code: CANYON, description: 'A carbon mixed-wheel mountain bike designed for freeride and bike-park use, including jumps, drops and steep descents.' } + - { product_id: ^10, sku: CANYON-SENDER-CFR, text: Canyon Sender CFR, sub_category_code: DH, unit_of_measure_code: EA, price: 7799, brand_code: CANYON, description: 'A carbon downhill mountain bike designed for racing and fast descending on steep, demanding tracks and bike-park terrain.' } + - { product_id: ^11, sku: SPEC-STUMPJUMPER-15-COMP, text: Specialized Stumpjumper 15 Comp, sub_category_code: TR, unit_of_measure_code: EA, price: 5500, brand_code: SPEC, description: 'A full-suspension trail mountain bike with adjustable geometry, designed to balance climbing efficiency and control on technical descents.' } + - { product_id: ^12, sku: SPEC-ENDURO-PRO, text: Specialized Enduro Pro, sub_category_code: EN, unit_of_measure_code: EA, price: 9500, brand_code: SPEC, description: 'A carbon full-suspension enduro mountain bike built for steep, rough descents and enduro racing while retaining the ability to pedal uphill.' } + - { product_id: ^13, sku: SPEC-EPIC-8-PRO, text: Specialized Epic 8 Pro, sub_category_code: XC, unit_of_measure_code: EA, price: 9000, is_inactive: true, brand_code: SPEC, description: 'A carbon full-suspension cross-country mountain bike designed for efficient pedalling, competitive racing and technical singletrack.' } + - { product_id: ^14, sku: SPEC-CHISEL-COMP, text: Specialized Chisel Comp, sub_category_code: HT, unit_of_measure_code: EA, price: 3400, brand_code: SPEC, description: 'An aluminium cross-country mountain bike designed for efficient pedalling, racing and fast singletrack riding.' } + - { product_id: ^15, sku: SPEC-STATUS-140, text: Specialized Status 140, sub_category_code: FR, unit_of_measure_code: EA, price: 2499, brand_code: SPEC, description: 'An aluminium mixed-wheel full-suspension mountain bike designed for playful trail riding, jumps and technical descents.' } + - { product_id: ^16, sku: GIANT-TRANCE-X-ADVPRO29, text: Giant Trance X Advanced Pro 29, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: GIANT, description: 'A carbon full-suspension trail mountain bike with 29-inch wheels and adjustable geometry, designed for climbing and technical descending.' } + - { product_id: ^17, sku: GIANT-ANTHEM-ADV29-1, text: Giant Anthem Advanced 29 1, sub_category_code: XC, unit_of_measure_code: EA, price: 7800, brand_code: GIANT, description: 'A carbon full-suspension cross-country mountain bike with 29-inch wheels, designed for efficient pedalling and fast riding on demanding race courses.' } + - { product_id: ^18, sku: GIANT-TALON-2, text: Giant Talon 2, sub_category_code: HT, unit_of_measure_code: EA, price: 800, is_deleted: true, brand_code: GIANT, description: 'An aluminium hardtail mountain bike with a suspension fork, designed as an accessible introduction to off-road trails.' } + - { product_id: ^19, sku: SHIMANO-XT-M8100-RD, text: Shimano Deore XT M8100 Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 129, brand_code: SHIMANO, description: 'A 12-speed mountain bike rear derailleur that moves the chain between cassette sprockets to change gears. Its clutch helps control chain movement on rough trails.' } + - { product_id: ^20, sku: SHIMANO-SLX-M7100-CASSETTE, text: Shimano SLX M7100 12-speed Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 99, brand_code: SHIMANO, description: 'A 12-speed mountain bike cassette that provides a wide range of gears for climbing and descending. It uses Shimano Micro Spline mounting and Hyperglide Plus shifting.' } + - { product_id: ^21, sku: SHIMANO-XTR-M9100-CRANK, text: Shimano XTR M9100 Crankset, sub_category_code: CR, unit_of_measure_code: EA, price: 499, brand_code: SHIMANO, description: 'A mountain bike crankset that transfers pedalling power to the chain through a direct-mount chainring, designed for lightweight cross-country and trail use.' } + - { product_id: ^22, sku: SRAM-GX-EAGLE-RD, text: SRAM GX Eagle Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 135, brand_code: SRAM, description: 'A cable-operated 12-speed mountain bike rear derailleur for SRAM Eagle drivetrains. It shifts the chain between rear sprockets and uses a clutch to control chain movement.' } + - { product_id: ^23, sku: SRAM-X0-TRANSMISSION-RD, text: SRAM X0 Transmission Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 400, brand_code: SRAM, description: 'A wireless electronic mountain bike rear derailleur for SRAM Eagle Transmission drivetrains. Its hangerless Full Mount design attaches around the rear axle on compatible frames.' } + - { product_id: ^24, sku: SRAM-XX1-EAGLE-CASSETTE, text: SRAM XX1 Eagle Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 449, brand_code: SRAM, description: 'A lightweight 12-speed mountain bike cassette with a wide gear range for climbing and descending, designed for SRAM Eagle drivetrains and XD freehub drivers.' } + - { product_id: ^25, sku: ONEUP-V3-DROPPER-180, text: OneUp Dropper Post V3 180mm, sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP, description: 'A mountain bike dropper seatpost with 180 mm of travel. Riders can lower the saddle for descending and raise it for pedalling using a compatible remote.' } + - { product_id: ^26, sku: ONEUP-CARBON-HANDLEBAR, text: OneUp Carbon Handlebar, sub_category_code: HB, unit_of_measure_code: EA, price: 159, brand_code: ONEUP, description: 'A carbon mountain bike handlebar with a shaped profile designed to absorb vibration and reduce hand fatigue while maintaining steering control.' } + - { product_id: ^27, sku: ONEUP-COMPOSITE-PEDALS, text: OneUp Composite Pedals, sub_category_code: PE, unit_of_measure_code: PR, price: 49, is_inactive: true, brand_code: ONEUP, description: 'Flat mountain bike pedals with composite bodies, broad platforms and replaceable traction pins, designed to grip ordinary flat-soled riding shoes.' } + - { product_id: ^28, sku: ONEUP-EDC-TOOL-V2, text: OneUp EDC Tool V2, sub_category_code: TO, unit_of_measure_code: EA, price: 59, brand_code: ONEUP, description: 'A compact trailside repair multi-tool with hex keys, a Torx key, a chain breaker and a tyre lever, designed for storage in compatible OneUp steerer or pump systems.' } + - { product_id: ^29, sku: MUCOFF-NANO-CLEANER, text: Muc-Off Nano Tech Bike Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 16.99, brand_code: MUCOFF, description: 'A spray-on bicycle cleaner that breaks down dirt and grime on frames and components before rinsing. Suitable for routine washing after muddy rides.' } + - { product_id: ^30, sku: MUCOFF-TUBELESS-SEALANT, text: Muc-Off MTB Tubeless Sealant, sub_category_code: SE, unit_of_measure_code: EA, price: 8.00, brand_code: MUCOFF, description: 'A tubeless mountain bike tyre sealant that helps seal punctures and maintain an airtight tyre system. Its UV-reactive dye helps locate leaks.' } + - { product_id: ^31, sku: MUCOFF-DRIVETRAIN-CLEANER, text: Muc-Off Drivetrain Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 25.00, brand_code: MUCOFF, description: 'A bicycle drivetrain cleaner that removes grease, oil and grime from chains, cassettes, chainrings and derailleurs. It can be sprayed on or used with a chain-cleaning device.' } + - { product_id: ^32, sku: LABOR, text: Labor, sub_category_code: LA, unit_of_measure_code: HR, price: 80.00, brand_code: NONE, is_non_stocked: true, description: 'Workshop labour charged by the hour for bicycle maintenance and repair. This is a service rather than a stocked physical product.' } - inventory: - { inventory_id: ^1, qty_on_hand: 3 } - { inventory_id: ^16, qty_on_hand: 1 } @@ -42,4 +42,4 @@ products: - { movement_id: ^2, product_id: ^6, quantity: -1, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - { movement_id: ^3, product_id: ^31, quantity: -3, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - { movement_id: ^4, product_id: ^24, quantity: -2, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - - { movement_id: ^5, product_id: ^28, quantity: -1, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } \ No newline at end of file + - { movement_id: ^5, product_id: ^28, quantity: -1, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } diff --git a/samples/tests/Contoso.Products.Test.Unit/Search/ProductSearchDocumentTests.cs b/samples/tests/Contoso.Products.Test.Unit/Search/ProductSearchDocumentTests.cs index ab0054d5c..dcdd11b3d 100644 --- a/samples/tests/Contoso.Products.Test.Unit/Search/ProductSearchDocumentTests.cs +++ b/samples/tests/Contoso.Products.Test.Unit/Search/ProductSearchDocumentTests.cs @@ -5,14 +5,26 @@ public class ProductSearchDocumentTests : WithGenericTester [Test] public void Create_NormalizesContentAndSortsTags() => Test.Scoped(test => test.Run(_ => { - var product = new Product { Sku = " test-1 ", Text = " Bike \t cleaner ", BrandCode = "MUCOFF", CategoryCode = "A", SubCategoryCode = "CL", Tags = ["mud", " cleaning ", "mud", ""] }; + var product = new Product { Sku = " test-1 ", Text = " Bike \t cleaner ", Description = " Removes \n dirt and mud. ", BrandCode = "MUCOFF", CategoryCode = "A", SubCategoryCode = "CL", Tags = ["mud", " cleaning ", "mud", ""] }; var document = ProductSearchDocument.Create(product); - document.Text.Should().Be("Name: Bike cleaner\nSKU: TEST-1\nBrand: Muc-Off\nCategory: Accessories\nSubcategory: Cleaner\nTags: cleaning, mud"); + document.Text.Should().Be("Name: Bike cleaner\nSKU: TEST-1\nDescription: Removes dirt and mud.\nBrand: Muc-Off\nCategory: Accessories\nSubcategory: Cleaner\nTags: cleaning, mud"); document.SourceHash.Should().HaveLength(64); ProductSearchDocument.Create(product).Should().Be(document); }).AssertSuccess()); + [Test] + public void Create_DescriptionChangeChangesHash() => Test.Scoped(test => test.Run(_ => + { + var product = new Product { Sku = "test-1", Text = "Cleaner" }; + var before = ProductSearchDocument.Create(product); + product.Description = "A cleaner for removing mud from bicycles."; + ProductSearchDocument.Create(product).SourceHash.Should().NotBe(before.SourceHash); + var described = ProductSearchDocument.Create(product); + product.Description = " A cleaner for removing \n mud from bicycles. "; + ProductSearchDocument.Create(product).Should().Be(described); + }).AssertSuccess()); + [Test] public void Create_ReorderedTagsAndNonSearchChangesDoNotChangeHash() => Test.Scoped(test => test.Run(_ => { diff --git a/samples/tests/Contoso.Products.Test.Unit/Validators/ProductValidatorTests.cs b/samples/tests/Contoso.Products.Test.Unit/Validators/ProductValidatorTests.cs index 87ae38753..33c38786d 100644 --- a/samples/tests/Contoso.Products.Test.Unit/Validators/ProductValidatorTests.cs +++ b/samples/tests/Contoso.Products.Test.Unit/Validators/ProductValidatorTests.cs @@ -29,4 +29,26 @@ public void Success() => Test.Scoped(test => var p = new Product { Sku = "X", Text = "Test", SubCategoryCode = "XC", UnitOfMeasureCode = "EA", Price = 9.99m }; ProductValidator.Default.AssertSuccess(p); }); -} \ No newline at end of file + + [Test] + public void Description_TooLong() => Test.Scoped(test => + { + var p = new Product { Sku = "X", Text = "Test", SubCategoryCode = "XC", UnitOfMeasureCode = "EA", Description = new string('x', 2001) }; + ProductValidator.Default.AssertErrors(p, ("description", "Description must not exceed 2000 character(s) in length.")); + }); + + [Test] + public void Description_MaximumLength() => Test.Scoped(test => + { + var p = new Product { Sku = "X", Text = "Test", SubCategoryCode = "XC", UnitOfMeasureCode = "EA", Description = new string('x', 2000) }; + ProductValidator.Default.AssertSuccess(p); + }); + + [TestCase(null)] + [TestCase("A cross-country mountain bicycle.")] + public void Description_Optional(string? description) => Test.Scoped(test => + { + var p = new Product { Sku = "X", Text = "Test", SubCategoryCode = "XC", UnitOfMeasureCode = "EA", Description = description }; + ProductValidator.Default.AssertSuccess(p); + }); +} diff --git a/tools/test-product-embeddings.ps1 b/tools/test-product-embeddings.ps1 index cfd6e2bf7..2909540ac 100644 --- a/tools/test-product-embeddings.ps1 +++ b/tools/test-product-embeddings.ps1 @@ -49,12 +49,12 @@ function Get-CosineDistance([double[]] $Query, [double[]] $Document) { # These names, labels and tags come from the existing product and reference-data seeds. $products = @( - @{ Sku = 'MUCOFF-NANO-CLEANER'; Text = 'Muc-Off Nano Tech Bike Cleaner'; Brand = 'Muc-Off'; Category = 'Accessories'; Subcategory = 'Cleaner'; Tags = '' }, - @{ Sku = 'SHIMANO-XT-M8100-RD'; Text = 'Shimano Deore XT M8100 Rear Derailleur'; Brand = 'Shimano'; Category = 'Parts'; Subcategory = 'Derailleur'; Tags = '' }, - @{ Sku = 'YETI-ASR-C2-2025'; Text = 'Yeti ASR C2'; Brand = 'Yeti Cycles'; Category = 'Bikes'; Subcategory = 'Cross country'; Tags = 'cross-country, full-suspension' } + @{ Sku = 'MUCOFF-NANO-CLEANER'; Text = 'Muc-Off Nano Tech Bike Cleaner'; Description = 'A spray-on bicycle cleaner that breaks down dirt and grime on frames and components before rinsing. Suitable for routine washing after muddy rides.'; Brand = 'Muc-Off'; Category = 'Accessories'; Subcategory = 'Cleaner'; Tags = '' }, + @{ Sku = 'SHIMANO-XT-M8100-RD'; Text = 'Shimano Deore XT M8100 Rear Derailleur'; Description = 'A 12-speed mountain bike rear derailleur that moves the chain between cassette sprockets to change gears. Its clutch helps control chain movement on rough trails.'; Brand = 'Shimano'; Category = 'Parts'; Subcategory = 'Derailleur'; Tags = '' }, + @{ Sku = 'YETI-ASR-C2-2025'; Text = 'Yeti ASR C2'; Description = 'A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack.'; Brand = 'Yeti Cycles'; Category = 'Bikes'; Subcategory = 'Cross country'; Tags = 'cross-country, full-suspension' } ) foreach ($product in $products) { - $document = "Name: $($product.Text)`nSKU: $($product.Sku)`nBrand: $($product.Brand)`nCategory: $($product.Category)`nSubcategory: $($product.Subcategory)`nTags: $($product.Tags)" + $document = "Name: $($product.Text)`nSKU: $($product.Sku)`nDescription: $($product.Description)`nBrand: $($product.Brand)`nCategory: $($product.Category)`nSubcategory: $($product.Subcategory)`nTags: $($product.Tags)" $product.Vector = Get-Embedding "search_document: $document" } From 78a18263aec0983c001e8a9e722ea7a12940d47b Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 13 Oct 2026 17:47:14 -0700 Subject: [PATCH 06/25] Add exact pgvector product search and Aspire Ollama integration Manage search snapshots through provider unit-of-work isolation, add typed reference-data prefetch, and prepare search projections explicitly in test fixtures. Reuse CoreEx HTTP configuration for SDK-owned client names. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .github/workflows/CI.yml | 7 +- Directory.Packages.props | 1 + docker-compose.yml | 2 +- samples/aspire/Contoso.Aspire/AppHost.cs | 10 +- samples/aspire/Contoso.Aspire/GlobalUsing.cs | 2 + .../aspire/Contoso.Aspire/appsettings.json | 16 +- .../Contoso.Test.Aspire.csproj | 1 + .../aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs | 8 +- samples/aspire/Contoso.Test.Aspire/E2ETest.cs | 5 +- .../aspire/Contoso.Test.Aspire/GlobalUsing.cs | 4 + .../Contoso.Test.Aspire/ProductSearchSetup.cs | 45 ++++ samples/docs/local-dev.md | 37 ++- samples/docs/search.md | 147 ++++++++--- .../src/Contoso.Mcp/Tools/CommerceTools.cs | 6 +- .../src/Contoso.Products.Api/GlobalUsing.cs | 1 + samples/src/Contoso.Products.Api/Program.cs | 8 +- ...create-products-product_search-table.pgsql | 13 + .../Clients/Ollama/OllamaHostingExtensions.cs | 23 ++ .../Clients/Ollama/OllamaHttpClient.cs | 61 +++-- .../Contoso.Products.Infrastructure.csproj | 3 +- .../GlobalUsing.cs | 11 + .../Repositories/ProductQueryArgsConfig.cs | 7 +- .../Repositories/ProductRepository.cs | 69 +++-- .../Repositories/ProductSearchIndex.cs | 130 +++++++++ .../Contoso.Products.Subscribe/GlobalUsing.cs | 3 +- .../src/Contoso.Products.Subscribe/Program.cs | 1 + .../appsettings.Development.json | 16 +- .../Contoso.Products.Test.Api/GlobalUsing.cs | 8 + .../ReadTests.ProductQuery.cs | 24 +- .../ReadTests.ProductSemanticSearch.cs | 249 ++++++++++++++++++ .../Contoso.Products.Test.Api/ReadTests.cs | 8 +- .../ReadTests/Product_Query_Schema.res.json | 7 +- .../TestEmbeddingAdapter.cs | 45 ++++ .../Clients/Ollama/OllamaHttpClientTests.cs | 65 ++++- .../Contoso.Products.Test.Unit/EntryPoint.cs | 3 +- .../Contoso.Products.Test.Unit/GlobalUsing.cs | 1 + .../Extended/PostgresUnitOfWorkInvoker.cs | 4 +- .../Extended/SqlServerUnitOfWorkInvoker.cs | 4 +- .../Abstractions/DatabaseArgs.cs | 3 +- .../Abstractions/DatabaseInvoker.cs | 23 +- src/CoreEx.Database/README.md | 19 +- .../CoreExExtensions.ApplicationBuilder.cs | 17 +- src/CoreEx/README.md | 2 +- src/CoreEx/RefData/README.md | 18 +- .../RefData/ReferenceDataOrchestrator.cs | 31 ++- .../EntityFrameworkUowTests.cs | 56 +++- .../GlobalUsing.cs | 2 + .../EntityFrameworkUowTests.cs | 56 +++- .../GlobalUsing.cs | 2 + .../ReferenceDataOrchestratorTests.cs | 60 +++++ .../Http/HttpClientConfigurationTests.cs | 26 +- tools/test-product-search.ps1 | 46 ++++ 52 files changed, 1267 insertions(+), 149 deletions(-) create mode 100644 samples/aspire/Contoso.Test.Aspire/ProductSearchSetup.cs create mode 100644 samples/src/Contoso.Products.Database/Migrations/20261013-182033-create-products-product_search-table.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHostingExtensions.cs create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs create mode 100644 samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs create mode 100644 samples/tests/Contoso.Products.Test.Api/TestEmbeddingAdapter.cs create mode 100644 tests/CoreEx.Database.Postgres.Test.Unit/GlobalUsing.cs create mode 100644 tests/CoreEx.Database.SqlServer.Test.Unit/GlobalUsing.cs create mode 100644 tools/test-product-search.ps1 diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index bc2cccbde..e11656b85 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -48,7 +48,7 @@ jobs: - uses: actions/checkout@v4 - name: Start containers - run: docker compose up -d + run: docker compose --profile search up -d - name: Setup .NET uses: actions/setup-dotnet@v4 @@ -148,9 +148,14 @@ jobs: rm aspnetcore-dev.pem # The Aspire end-to-end tests orchestrate every sample host together, so they run last (after all the intra-domain sample tests) and on .NET 10 only (the AppHost is single-TFM). + - name: Prepare semantic search model + timeout-minutes: 10 + run: docker compose exec -T ollama ollama pull nomic-embed-text:v1.5 + - name: Test samples (Aspire end-to-end, net10.0 only) timeout-minutes: 20 run: dotnet test CoreEx.Samples.Test.E2E.slnf --no-build --configuration Release --logger trx --results-directory coverage/samples-e2e -m:1 -p:TestTfmsInParallel=false -p:BuildInParallel=false + - name: Upload samples test results if: always() uses: actions/upload-artifact@v4 diff --git a/Directory.Packages.props b/Directory.Packages.props index 54b94b35e..daf9f950f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,6 +12,7 @@ + diff --git a/docker-compose.yml b/docker-compose.yml index 8d7e56095..3289d76f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - "1433:1433" db-postgres: - image: postgres + image: docker.io/pgvector/pgvector:0.8.2-pg18-trixie environment: POSTGRES_PASSWORD: "yourStrong#!Password" ports: diff --git a/samples/aspire/Contoso.Aspire/AppHost.cs b/samples/aspire/Contoso.Aspire/AppHost.cs index 754d9c3ca..3588f8ce0 100644 --- a/samples/aspire/Contoso.Aspire/AppHost.cs +++ b/samples/aspire/Contoso.Aspire/AppHost.cs @@ -16,6 +16,7 @@ var redis = builder.AddExternalConnectionString("redis").WithIconName("Database"); var serviceBus = builder.AddExternalConnectionString("ServiceBus").WithIconName("MailMultiple"); var cosmos = builder.AddExternalConnectionString("Cosmos", endpointKey: "AccountEndpoint").WithIconName("DatabaseMultiple"); +var ollama = builder.AddExternalService("ollama", builder.Configuration["Ollama:BaseAddress"].ThrowIfNullOrEmpty()).WithHttpHealthCheck("/api/tags").WithIconName("BrainCircuitRegular"); // Customers domain. var customersApi = builder.AddProject("customers-api").WithReference(cosmos).WithReference(redis).AddEndpoints("/health/ready/detailed"); @@ -23,8 +24,15 @@ // Products domain. var productsApi = builder.AddProject("products-api").WithReference(postgres).WithReference(redis).AddEndpoints("/health/ready/detailed"); +productsApi.WithReference(ollama).WithEnvironment("Ollama__BaseAddress", "http://ollama").WaitFor(ollama); builder.AddProject("products-relay").WithReference(postgres).WithReference(serviceBus).AddEndpoints("/health/ready/detailed").AddHostedServiceSupport(); -builder.AddProject("products-subscribe").WithReference(postgres).WithReference(redis).WithReference(serviceBus).AddEndpoints("/health/ready/detailed").AddHostedServiceSupport(); +var productsSubscribe = builder.AddProject("products-subscribe").WithReference(postgres).WithReference(redis).WithReference(serviceBus).AddEndpoints("/health/ready/detailed").AddHostedServiceSupport(); +productsSubscribe.WithReference(ollama).WithEnvironment("Ollama__BaseAddress", "http://ollama"); +foreach (var setting in builder.Configuration.GetSection("Ollama:Resilience").AsEnumerable().Where(x => x.Value is not null)) +{ + productsApi.WithEnvironment(setting.Key.Replace(":", "__"), setting.Value.ThrowIfNull()); + productsSubscribe.WithEnvironment(setting.Key.Replace(":", "__"), setting.Value.ThrowIfNull()); +} // Shopping domain. // Note: shopping-api and shopping-subscribe call Products and Customers synchronously (see ProductsHttpClient and CustomersHttpClient). The appsettings default is the standalone diff --git a/samples/aspire/Contoso.Aspire/GlobalUsing.cs b/samples/aspire/Contoso.Aspire/GlobalUsing.cs index 717d1b02b..d9e1ce9ae 100644 --- a/samples/aspire/Contoso.Aspire/GlobalUsing.cs +++ b/samples/aspire/Contoso.Aspire/GlobalUsing.cs @@ -1,2 +1,4 @@ +global using CoreEx; +global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.Hosting; global using System.Net; diff --git a/samples/aspire/Contoso.Aspire/appsettings.json b/samples/aspire/Contoso.Aspire/appsettings.json index 888f884e2..d596deac6 100644 --- a/samples/aspire/Contoso.Aspire/appsettings.json +++ b/samples/aspire/Contoso.Aspire/appsettings.json @@ -1,4 +1,18 @@ { + "Ollama": { + "BaseAddress": "http://localhost:11434/", + "Resilience": { + "AttemptTimeout": { + "Timeout": "00:01:00" + }, + "TotalRequestTimeout": { + "Timeout": "00:02:00" + }, + "CircuitBreaker": { + "SamplingDuration": "00:02:00" + } + } + }, "Logging": { "LogLevel": { "Default": "Information", @@ -6,4 +20,4 @@ "Aspire.Hosting.Dcp": "Warning" } } -} \ No newline at end of file +} diff --git a/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj b/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj index 4832a2dd6..1c24ba83a 100644 --- a/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj +++ b/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj @@ -31,6 +31,7 @@ + diff --git a/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs b/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs index 9caf80c9e..6ba539dfe 100644 --- a/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs +++ b/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs @@ -4,7 +4,7 @@ public partial class E2ETest { private string _mcpEndpoint = string.Empty; - [Test] + [Test, Order(0)] public async Task Contoso_Mcp_FullBasketLifecycle() { Test.Checkpoint("Increase inventory for the seeded, active Product used by the MCP basket scenario."); @@ -33,11 +33,11 @@ public async Task Contoso_Mcp_FullBasketLifecycle() tools.Select(x => x.Name).Should().Contain(expectedToolNames); tools.Should().OnlyContain(x => x.ProtocolTool.OutputSchema.HasValue); - Test.Checkpoint("Search for and read a Product through MCP."); + Test.Checkpoint("Read a Product and search the prepared real-model projection through MCP."); var product = await CallToolAsync(client, "get_product", new Dictionary { ["productId"] = productId }).ConfigureAwait(false); product.GetProperty("id").GetString().Should().Be(productId); - var productSearch = await CallToolAsync(client, "search_products", new Dictionary { ["searchTerm"] = "spec", ["take"] = 25 }).ConfigureAwait(false); - productSearch.EnumerateArray().Should().Contain(x => x.GetProperty("sku").GetString()!.StartsWith("SPEC", StringComparison.Ordinal)); + var productSearch = await CallToolAsync(client, "search_products", new Dictionary { ["searchTerm"] = "something to clean mud off my bike", ["take"] = 1 }).ConfigureAwait(false); + productSearch.EnumerateArray().Should().ContainSingle().Which.GetProperty("sku").GetString().Should().Be("MUCOFF-NANO-CLEANER"); Test.Checkpoint("Create a Basket for the seeded MCP demo customer, then read and list it through MCP."); var customerId = 16.ToGuid().ToString(); diff --git a/samples/aspire/Contoso.Test.Aspire/E2ETest.cs b/samples/aspire/Contoso.Test.Aspire/E2ETest.cs index 7e6a03f00..9f9c581d6 100644 --- a/samples/aspire/Contoso.Test.Aspire/E2ETest.cs +++ b/samples/aspire/Contoso.Test.Aspire/E2ETest.cs @@ -20,6 +20,9 @@ protected override async Task OnBeforeStartAsync(DistributedApplication app) // Clear the Redis cache. await app.ClearRedisCacheAsync("redis"); + // Prepare the real-model derived projection explicitly; API startup never rebuilds it. + await ProductSearchSetup.RebuildAsync(app).ConfigureAwait(false); + // Reset the Azure Service Bus queues and topics/subscriptions to an initial state. await app.ResetAzureServiceBusAsync("servicebus", ServiceBus.GetQueues(), ServiceBus.GetTopicsAndSubscriptions()); } @@ -35,7 +38,7 @@ protected override async Task OnAfterStartAsync(DistributedApplication app) _mcpEndpoint = app.GetEndpoint("contoso-mcp", "http").ToString().TrimEnd('/'); } - [Test] + [Test, Order(1)] public async Task CreateOrderAndConfirmAsync() { Test.Checkpoint("Create and activate a new Product; this should sync to the Shopping domain via the Products domain's outbox and Service Bus."); diff --git a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs index 5c07dcbae..fbea82256 100644 --- a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs +++ b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs @@ -5,9 +5,13 @@ global using Azure.Messaging.ServiceBus; global using Azure.Messaging.ServiceBus.Administration; global using Contoso.Products.Contracts; +global using Contoso.Products.Infrastructure.Clients.Ollama; +global using Contoso.Products.Infrastructure.Repositories; global using Contoso.Shopping.Contracts; global using CoreEx; global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Hosting; global using ModelContextProtocol.Client; global using ModelContextProtocol.Protocol; global using NUnit.Framework; diff --git a/samples/aspire/Contoso.Test.Aspire/ProductSearchSetup.cs b/samples/aspire/Contoso.Test.Aspire/ProductSearchSetup.cs new file mode 100644 index 000000000..6849fa282 --- /dev/null +++ b/samples/aspire/Contoso.Test.Aspire/ProductSearchSetup.cs @@ -0,0 +1,45 @@ +namespace Contoso.Test.Aspire; + +internal static class ProductSearchSetup +{ + public static async Task RebuildAsync(DistributedApplication app, CancellationToken cancellationToken = default) + { + var connectionString = await AspireTesterBase.GetConnectionStringAsync(app, "Postgres").ConfigureAwait(false) + ?? throw new InvalidOperationException("The Postgres resource has no connection string."); + await RebuildAsync(app.Services.GetRequiredService(), connectionString, cancellationToken).ConfigureAwait(false); + } + + internal static async Task RebuildAsync(IConfiguration configuration, string connectionString, CancellationToken cancellationToken = default) + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.AddConfiguration(configuration); + builder.Configuration["ConnectionStrings:Postgres"] = connectionString; + builder.Configuration.GetRequiredSection("Ollama"); + builder.AddHostSettings(solutionName: "Contoso", domainName: "Products"); + builder.AddAzureNpgsqlDataSource("Postgres"); + builder.AddProductEmbeddings(); + builder.Services + .AddPrecisionTimeProvider() + .AddExecutionContext() + .AddMemoryCache() + .AddMemoryOnlyHybridCache() + .AddReferenceDataOrchestrator() + .AddDynamicServicesUsing() + .AddPostgresDatabase() + .AddPostgresUnitOfWork() + .AddDbContext() + .AddEfDb(); + + using var host = builder.Build(); + await using var scope = host.Services.CreateAsyncScope(); + scope.ServiceProvider.GetRequiredService(); + try + { + return await scope.ServiceProvider.GetRequiredService().RebuildAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + CoreEx.ExecutionContext.Reset(); + } + } +} diff --git a/samples/docs/local-dev.md b/samples/docs/local-dev.md index 63e6e7e9b..9814fa031 100644 --- a/samples/docs/local-dev.md +++ b/samples/docs/local-dev.md @@ -47,7 +47,7 @@ podman compose -f docker-compose.yml down | Service name | Image | Port(s) | Used by | Notes | |---|---|---|---|---| | `db-sql-server` | `mssql/server:2022-latest` | 1433 | Shopping domain; Service Bus emulator | SA password: `yourStrong(!)Password` | -| `db-postgres` | `postgres` | 5432 | Products domain | Password: `yourStrong#!Password` | +| `db-postgres` | `pgvector/pgvector:0.8.2-pg18-trixie` | 5432 | Products domain | PostgreSQL 18 with pgvector 0.8.2; password: `yourStrong#!Password` | | `redis-cache` | `redis:latest` | 6379 | All domains (FusionCache backplane) | No auth by default | | `servicebus-emulator` | `azure-messaging/servicebus-emulator:latest` | 5672 (AMQP), 5300 (mgmt) | Products.Subscribe, Shopping.Subscribe, all Relay hosts | Depends on `db-sql-server`; config mounted from `servicebus/Config.json` | | `dts-emulator` | `dts/dts-emulator:latest` | 8080, 8082 | Orders.Workflow.Worker, Orders.Api | Task hubs: `default`, `order` | @@ -55,8 +55,8 @@ podman compose -f docker-compose.yml down ### Optional local embedding model -The first Products search milestone uses **Ollama 0.13.5** with **`nomic-embed-text:v1.5`**. -It is opt-in: normal Compose startup does not start the `search` profile or download a model. +Products semantic search uses **Ollama 0.13.5** with **`nomic-embed-text:v1.5`**. +Normal Compose startup does not start the `search` profile or download a model. Requests without `$search` do not need it; semantic requests do. From the repository root: ```powershell @@ -83,9 +83,34 @@ The client sends `truncate=false`, so an oversized input fails instead of silent The smoke script checks the installed model manifest digest, validates vectors, and prints cosine rankings for seeded catalog examples. It intentionally fails if the expected semantic ranking is not obtained; do not confuse a working embedding endpoint with good retrieval. -Products API development configuration contains `Ollama:BaseAddress` and longer inference timeout settings. -Other environments must supply their own endpoint/timeouts before using the adapter. Keyword requests do not call Ollama. -This milestone does **not** enable pgvector or change the Products API's default keyword behavior. +Products API and Subscribe development configuration contain `Ollama:BaseAddress` and longer inference timeout settings. +Other environments must supply their own endpoint/timeouts before searching. PostgreSQL now includes pgvector and `$search` uses exact semantic ranking by default. + +Both hosts use the Aspire OllamaSharp client integration via `AddProductEmbeddings`, with CoreEx's named-client +registration providing service discovery, idempotency handling and HTTP resilience. The same registration is used by real-model test setup. This is client wiring only: Compose +still owns the server and model provisioning, and the Application layer still depends on `IProductEmbeddingAdapter`. + +The API has no rebuild command or startup rebuild. Products read-test `OneTimeSetUp` explicitly prepares controlled +embeddings; Test.Aspire prepares real embeddings after seeding and clearing the cache, before starting hosts. +Aspire exposes the Compose-owned Ollama endpoint as an external resource and uses the AppHost's `Ollama` +configuration for both fixture preparation and API calls. Start the `search` Compose profile and install the model +above before running Aspire E2E tests; CI does this explicitly too. + +`ProductSearchIndex.RebuildAsync` remains callable through a configured Infrastructure service scope. There is +currently no standalone indexing console or HTTP maintenance endpoint. Starting Aspire manually does not seed or +rebuild the catalog. After prepared-fixture execution, `.\tools\test-product-search.ps1` can exercise a normally +running API (default `http://localhost:5207/`). Searchable CRUD changes require another explicit rebuild until +incremental indexing is implemented. Missing/stale eligible embeddings produce HTTP 503 rather than an incomplete +successful search. Ordinary product reads and structured filters remain available. + +To prepare and verify a real-model catalog without subsequently running the E2E test that adds an unindexed product, +run the MCP lifecycle on its own. This is test setup: it reseeds all three domain databases and resets broker state, +so use it only against disposable local infrastructure. + +```powershell +dotnet test samples\aspire\Contoso.Test.Aspire --filter "FullyQualifiedName~E2ETest.Contoso_Mcp_FullBasketLifecycle" +``` + See [the search guide](search.md) for implementation status and relevance findings. ### Service Bus configuration diff --git a/samples/docs/search.md b/samples/docs/search.md index 6c4a4585d..e9e460f2b 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -1,6 +1,6 @@ # Product search: from keywords to vectors -This guide explains how product search works, why vectors can help, and how we intend to extend the Contoso Products sample using PostgreSQL and pgvector. It assumes no previous experience with embeddings, vector databases, or search engines. +This guide explains how product search works, why vectors can help, and how the Contoso Products sample uses PostgreSQL and pgvector. It assumes no previous experience with embeddings, vector databases, or search engines. The goal is not merely to store some numbers and label the result "AI". It is to demonstrate an understandable search capability with correct filtering, reliable updates, and evidence of useful results. @@ -11,25 +11,25 @@ The goal is not merely to store some numbers and label the result "AI". It is to | Capability | Status | What that means | |---|---|---| | CoreEx `QueryArgs.Search` and search parser configuration | Implemented | REST, HTTP clients, OpenAPI, and GraphQL-lite can carry search text. | -| Products keyword search | Implemented | Every distinct whitespace-separated term must match the product text or the beginning of its SKU. | +| Products semantic search | Implemented; default for `$search` | Exact cosine-distance ranking replaces the earlier keyword matching. Requests without `$search` retain ordinary filtering and ordering. | | MCP `search_products` forwarding to Products | Implemented | The tool calls the Products API with `$search`; it does not query the database. | | Ranked-search mode plumbing | Implemented | CoreEx validates and carries ranked search intent; the repository must execute it. | | Canonical product search document | Implemented | Versioned, whitespace-normalized documents and source hashes exclude price, status, inventory and business metadata. | | Product descriptions and price filtering | Implemented | Optional descriptive content supports CRUD; decimal price constraints use existing REST and GraphQL-lite query configuration. | | Local embedding provider | Implemented | Ollama/Nomic document and query instructions, response validation, and an Application adapter boundary are available. | -| Separate search projection and pgvector semantic retrieval | Proposed | No product embeddings are currently stored or queried in PostgreSQL. | +| Separate search projection and pgvector semantic retrieval | Implemented | PostgreSQL stores 768-dimensional embeddings and computes exact, filtered cosine-distance rankings. | | PostgreSQL full-text retrieval and hybrid ranking | Proposed | The current sample does not perform full-text or hybrid search. | -| Asynchronous search indexing and backfill | Proposed | The outbox and subscriber foundations exist; search-specific processing does not. | +| Explicit rebuild / asynchronous search indexing | Rebuild implemented; asynchronous processing proposed | Test setup invokes reusable Infrastructure logic to rebuild the catalog atomically. Missing/stale eligible documents return HTTP 503 until rebuilt. | | Real-model relevance evaluation and HNSW benchmark | Initial probes implemented; broader evaluation proposed | Two illustrative semantic expectations pass after descriptive enrichment. Full-catalog relevance and index performance remain unproven. | -The proposed baseline is PostgreSQL as the search store, one embedding per product, a rebuildable search projection, and indexing outside product-write transactions. The first milestone selects local Ollama with Nomic v1.5 and 768 dimensions. The final schema, retrieval windows, and count semantics still need to be resolved during implementation. +The implemented baseline is PostgreSQL as the search store, one embedding per product/profile/recipe, a rebuildable search projection, and indexing outside product-write transactions. It uses local Ollama with Nomic v1.5 and 768 dimensions. Exact retrieval ranks every eligible product; count means the number of eligible products, not the number judged relevant. Full-text/hybrid retrieval windows and asynchronous maintenance remain future work. ## Reading map - [Start with the three kinds of search](#three-kinds-of-search). - [Learn what embeddings and distances mean](#embeddings-text-as-coordinates). - [Understand PostgreSQL storage and retrieval](#postgresql-and-pgvector). -- [Follow the proposed sample architecture](#where-this-fits-in-the-sample). +- [Follow the sample architecture](#where-this-fits-in-the-sample). - [See how indexing stays correct](#keeping-search-current). - [Understand hybrid ranking](#hybrid-search-combining-two-result-lists). - [Review the implementation stages and evidence](#implementation-stages). @@ -41,13 +41,13 @@ Imagine a catalog containing "Muc-Off Nano Tech Bike Cleaner". ### Keyword search: find these words -A keyword search asks whether particular text occurs in selected fields. The implemented sample splits a request into whitespace-separated terms, requires every term to match, and allows each term to match either the product text or the SKU prefix. +A keyword search asks whether particular text occurs in selected fields. The sample's earlier keyword implementation split a request into whitespace-separated terms, required every term to match, and allowed each term to match either the product text or the SKU prefix. CoreEx still supports that mode; Products has now deliberately switched to semantic mode. ```text GET /api/products?$search=rear%20derailleur ``` -This finds products whose configured fields collectively match both words. It is not phrase matching: the words do not have to be adjacent. Repeated terms are deduplicated case-insensitively. +In keyword mode this finds products whose configured fields collectively match both words. It is not phrase matching: the words do not have to be adjacent. Repeated terms are deduplicated case-insensitively. In the current Products semantic mode, this same request instead returns nearby meanings in relevance order. Keyword search is simple, predictable, and useful. It is not inherently inferior to vectors. For an exact identifier, it may be the better tool. @@ -95,7 +95,7 @@ This should retrieve the most relevant eligible Yeti products, not search the en Keyword mode allows ordinary ordering. Ranked modes own relevance ordering: CoreEx rejects an explicit `$orderby` and does not apply the configured default ordering. -The server configuration selects the mode. There is currently no public `$searchMode` parameter. +The server configuration selects the mode; Products now selects semantic mode. There is no public `$searchMode` parameter. Semantic text is limited to 200 normalized characters, but the keyword-only eight-term limit no longer applies. Price, availability, and other hard constraints belong in structured filtering, not in assumptions about what an embedding understands. Products exposes a decimal `price` filter with `eq`, `ne`, `in`, `gt`, `ge`, `lt` and `le`; inventory filters are not currently exposed. @@ -158,30 +158,28 @@ That explanation is a separate step, and the returned product data remains the a An **extension** is additional database functionality installed on the server and enabled in a database. `CREATE EXTENSION` cannot install server binaries that are absent. -The current Compose setup uses a plain, unpinned `postgres` image. Implementation will need a compatible, pinned image containing pgvector, plus a database migration enabling the extension. Changing database images or major versions requires checking compatibility and preserving existing local data. +Compose uses `docker.io/pgvector/pgvector:0.8.2-pg18-trixie`: pgvector 0.8.2 on PostgreSQL 18. The Products migration enables `vector` and creates the search projection. Changing database images or major versions requires checking compatibility; do not reuse a data directory across incompatible PostgreSQL major versions. The local switch retained the existing PostgreSQL 18 data volume. -### Proposed storage +### Implemented storage A **search projection** is derived data prepared for retrieval. It can be rebuilt from the authoritative products. -The proposed one-to-one projection could have this shape: +The current projection has this shape (see the migration for complete constraints): ```sql --- Illustrative only: names, dimensions, and constraints are not finalized. -CREATE EXTENSION IF NOT EXISTS vector; - -CREATE TABLE products.product_search -( - product_id text PRIMARY KEY, - document_text text NOT NULL, - embedding vector(768) NOT NULL, - embedding_profile text NOT NULL, - source_hash text NOT NULL, - embedded_at timestamptz NOT NULL DEFAULT now() +CREATE TABLE products.product_search ( + product_id varchar(50) NOT NULL REFERENCES products.product(product_id) ON DELETE CASCADE, + profile varchar(250) NOT NULL, + recipe varchar(50) NOT NULL, + source_hash char(64) NOT NULL, + document text NOT NULL, + embedding vector(768) NOT NULL CHECK (vector_norm(embedding) > 0), + indexed_on timestamptz NOT NULL, + PRIMARY KEY (product_id, profile, recipe) ); ``` -The `768` is an example, not a chosen model dimension. Choose the model first, then the storage type and index strategy. The final design must also define ownership/foreign keys, deletion handling, full-text storage, and profile migration. +The `768` is Nomic's selected native dimensionality. The composite primary key permits separate profiles/recipes without mixing them in retrieval. There is no HNSW or IVFFlat index: PostgreSQL evaluates exact distances. Full-text storage and coherent profile rollout remain later milestones. Why a separate table instead of another product column? @@ -252,7 +250,7 @@ Do not generate invented product specifications merely to make semantic results `Description` is optional plain text, limited to 2,000 characters in validation and PostgreSQL. It is returned in full-product and `ProductLite` responses and editable through existing POST, PUT and merge-PATCH endpoints; PATCH can explicitly clear it with `{"description":null}`. Updates retain the existing ETag, changelog and transactional-outbox behavior. The search document recipe is now `product-document-v2`, and description edits change its source hash; price changes do not. -Descriptions are included in embedding documents. The default keyword `$search` still matches the product name (`Text`) or SKU prefix, not the new description field; this preserves the existing keyword behavior until a deliberate search-mode change. +Descriptions are included in embedding documents. The current semantic `$search` uses their meaning alongside names, SKU and classifications; it does not require literal keyword matches. For guaranteed SKU matching, use `$filter=sku eq '...'`, not semantic similarity. ### Where the seeded descriptions come from @@ -305,7 +303,7 @@ Otherwise a price change can leave the stored embedding representing yesterday's ### One product, one vector -One embedding per product is the proposed baseline. +One embedding per product/profile/recipe is the implemented baseline. **Chunking** means splitting a long document into smaller passages and embedding them separately. It is useful for manuals and lengthy articles, not automatically useful for a short catalog record. @@ -340,7 +338,7 @@ flowchart LR API --> Client ``` -This is the proposed ranked flow. Today the repository executes keyword search without an embedding provider. +This is the implemented semantic flow. Requests without `$search` do not call the embedding provider or require a populated projection. **CoreEx responsibilities:** carry intent through `QueryArgs`, validate configured constraints, apply keyword predicates, and distinguish ranked search from ordinary ordering. @@ -348,9 +346,9 @@ This is the proposed ranked flow. Today the repository executes keyword search w The existing `ProductRepository.QueryAsync` joins product, subcategory, and inventory data. Ranked retrieval must preserve that behavior and all configured filters rather than quietly implementing a reduced API. -Application orchestration should depend on application-owned interfaces. Embedding transport/SDK and pgvector access belong in Infrastructure, composed by the hosts. The exact interface names are not yet chosen. +Application orchestration depends on application-owned interfaces. `IProductEmbeddingAdapter` owns the model boundary, its Ollama implementation lives in Infrastructure, and `ProductSearchIndex` handles derived storage and rebuilds there. -The .NET pgvector integration supports Npgsql and EF Core. Versions must match this repository's EF/Npgsql target matrix; external Npgsql data sources and EF options must both be configured where required. Database changes remain owned by DbEx migrations and regeneration, not hand-edited generated files. +The sample deliberately uses bound vector-text parameters with an explicit PostgreSQL `vector(768)` cast, rather than adding pgvector EF/Npgsql packages. EF Core's unmapped `SqlQuery` exposes only product IDs, fingerprints and distances; the repository composes it with its existing relational joins and filters. Vectors never enter CRUD contracts. This SQL-only derived table is intentionally outside `dbex.yaml`'s generated CRUD models; its DDL is still owned by a DbEx migration. ### What MCP adds @@ -358,12 +356,72 @@ MCP (Model Context Protocol) lets an AI client discover and invoke tools such as The tool sends search text to Products and returns product data. It does not generate its own embeddings, query PostgreSQL directly, or maintain its own index. Non-MCP clients use the same capability. -After switching the sample to ranked search, update the tool description to describe the actual behavior instead of promising keyword matching. +The tool description now promises semantic candidate ordering, not literal keyword or exact-SKU matching. Price/brand filters remain available in REST/GraphQL-lite but are not yet arguments on the MCP tool. Search does not resolve authentication or authorization. The sample's existing authentication gaps remain separate concerns. ## Keeping search current +### Current milestone: an explicit, atomic rebuild + +Rebuilding is reusable Infrastructure logic, **not an API startup responsibility**. The earlier API +`--rebuild-search` maintenance mode has been removed. There is no HTTP rebuild endpoint or standalone indexing +console at this milestone. + +Products read-test `OneTimeSetUp` seeds the catalog and invokes `ProductSearchIndex.RebuildAsync` with controlled +embeddings. Aspire E2E setup seeds the catalog, clears Redis, then creates a small test-owned service scope using +the actual PostgreSQL resource connection string and AppHost Ollama configuration. It invokes the same rebuild +logic with the real model before starting the hosts. The scope uses fresh in-memory reference-data caching over +the real database; the serving API uses its normal Redis-backed cache, cleared by setup. The AppHost exposes +Ollama as a Compose-owned external resource. CI explicitly starts the search profile and pulls the selected model. +Neither API nor AppHost startup seeds or rebuilds anything during ordinary manual execution. + +The API, Subscribe host and real-model test fixture register `CommunityToolkit.Aspire.OllamaSharp` through the +shared `AddProductEmbeddings` composition helper. Aspire registers `IOllamaApiClient`; CoreEx's `AddNamedHttpClient` +augments the SDK's named client using the separate `Ollama` configuration section, adding idempotency handling, +service discovery and standard HTTP resilience from `Ollama:Resilience`, +with `HttpClient.Timeout` disabled so the resilience timeouts control request duration. The container remains +Compose-owned; the client integration does not launch it. + +The SDK handles request serialization, transport and response disposal. `OllamaHttpClient` is now a validated SDK +wrapper, not a hand-written HTTP transport. It preserves Nomic's prefixes, `truncate=false`, the pinned model and +768-dimensional finite, non-zero vector checks. OllamaSharp's response type omits the returned model name, so a +small read-only response converter retains that field for validation. Provider/protocol errors return failed +CoreEx results; cancellation still propagates. No provider types leak through `IProductEmbeddingAdapter`. +The SDK's direct embedding API is used instead of `IEmbeddingGenerator` to retain the provider-specific truncation +policy and response model metadata; portability remains at the application-owned adapter boundary. + +The Products Subscribe host also registers the Ollama client because its shared repositories and dynamically +registered services include the embedding adapter. Ordinary reservation handlers do not call the model; this +registration satisfies their service graph, not an indexing workflow. + +The rebuild indexes the current non-deleted catalog, including inactive products so `$inactive` works. It does +not seed products or download the model. It prepares documents and calls Ollama **before** starting a database +transaction when invoked outside an existing transaction. Only publication takes a short framework-managed unit +of work: lock the product table against writes, verify that its searchable documents still match the captured +catalog, replace the selected profile/recipe, and commit. A model failure leaves the previous index untouched; +a concurrent searchable catalog edit rejects publication. Deleted-product entries disappear on rebuild, and +physical deletion also cascades through the foreign key. Other profiles remain separate. If a consumer calls +rebuild inside an existing unit of work, it inherits that transaction and publication remains subject to the +parent's eventual commit or rollback. + +For a semantic request, the repository checks eligible source hashes first, so a known broken index fails before paying for inference. It embeds the query, uses `PostgresUnitOfWork.TransactionAsync` with `PostgresDatabaseArgs.IsolationLevel = RepeatableRead` to open a database snapshot, and checks hashes again to cover writes during inference. It then joins the compatible projection, orders by exact cosine distance and product ID (a deterministic tie-breaker), and applies paging/count. Any missing or stale eligible document yields **HTTP 503** with a rebuild instruction. There is no silent keyword fallback, incomplete result set or successful empty response masquerading as inference failure. + +The framework manages commit, rollback and cleanup, and enlists the EF context in the shared database transaction. A nested unit of work uses a savepoint, not a new independent transaction: it inherits the parent's isolation. Semantic search therefore explicitly rejects an existing parent unless it is `RepeatableRead` or `Serializable`; silently accepting `ReadCommitted` would lose the consistency guarantee. For standalone requests, inference occurs before opening the snapshot. When a caller already owns a compatible parent transaction, that caller owns its lifetime, including time spent on inference. The search callback does not enqueue events, but any events already queued by a consumer follow the normal root-unit-of-work outbox rules. + +This guard deliberately recomputes hashes over eligible products and reads stored fingerprints, twice for successful searches. That is suitable for a 32-row teaching catalog, **not a high-volume indexing strategy**. Price and inventory come from live relational data, are not in the document hash, and do not require new embeddings. Name, description, tags and reference-data labels do. Labels use the existing orchestrator/cache; rebuilding and querying must use consistent language/cache state. This is not a substitute for reference-data invalidation and asynchronous indexing. + +API integration tests rebuild with controlled vectors, isolating distance ordering and failure behavior from model +quality. Test.Aspire's MCP lifecycle asserts that real semantic cleaning intent returns Nano cleaner through +MCP -> Products API -> PostgreSQL. It runs before the checkout scenario that creates a new product: until +incremental indexing exists, that catalog mutation intentionally makes the prepared projection stale. This ordering +is explicit fixture management, not evidence of automatic indexing. Future mutation-to-search tests must exercise +the asynchronous indexing path without inserting a rebuild between mutation and search. + +There is no relevance cutoff: every eligible vector is a candidate. `$count=true` reports the eligible catalog size, including poor semantic matches. An unrelated query still returns neighbors; clients must not interpret a returned product as guaranteed suitable. + +### Next milestone: asynchronous maintenance + Product CRUD already commits product mutations and outgoing events together using the transactional outbox. An **outbox** stores an event in the same database transaction as its business change. A **relay** forwards committed events to the broker. A **subscriber** handles delivered events. @@ -585,6 +643,21 @@ The experiment did not modify catalog data or change the original smoke expectat With the researched descriptions, the original smoke examples now pass: the cleaner ranks first for the cleaning query (cosine distance **0.2958**) and the Shimano derailleur ranks first for the gear-changing query (**0.3442**). This is still a three-product illustration, not proof of full-catalog relevance. +### Exact retrieval against the seeded catalog + +The subsequent pgvector milestone stores **31 non-deleted products**, including two inactive products. Normal semantic requests rank the **29 active products**. The real-model API smoke keeps the original queries rather than choosing new ones to make the database demo look better: + +| Query | First candidate | Second candidate | Third candidate | +|---|---|---|---| +| something to clean mud off my bike | Muc-Off Nano Tech Bike Cleaner | Muc-Off Drivetrain Cleaner | Muc-Off MTB Tubeless Sealant | +| a component for changing gears | Shimano Deore XT M8100 Rear Derailleur | SRAM GX Eagle Rear Derailleur | OneUp EDC Tool V2 | + +Both expected first products survive the larger candidate set. A price-under-20 cleaning request returns the Nano cleaner, and GraphQL-lite returns the same first gear-changing candidate as REST. `tools/test-product-search.ps1` reproduces these checks through the API, not by calculating distances in PowerShell. + +The live MCP HTTP transport was also exercised against this API: `search_products` returned the Nano cleaner first for the original cleaning intent, including its description and live price. Temporarily invalidating its indexed fingerprint produced a tool error rather than a successful partial result; the fingerprint was restored afterwards. This bounded manual check is distinct from the full Test.Aspire topology. + +The lower ranks expose the remaining quality problem: sealant is not a cleaner, and a repair multi-tool ranks above some drivetrain components. Exact search does not fix that; it faithfully returns the model's nearest candidates. These examples motivate hybrid retrieval and a curated evaluation set, not an invented confidence threshold or a claim that search is finished. + Measure document and query embedding separately. Caching repeated query embeddings can help, but keys must include the profile and relevant preprocessing; user text may be sensitive, so retention/logging needs care. Trace embedding time, database time, indexing lag, retries, and failures. Avoid logging raw vectors, credentials, or full user queries by default. @@ -594,7 +667,7 @@ Trace embedding time, database time, indexing lag, retries, and failures. Avoid | Stage | Learning objective | Evidence required | |---|---|---| | 1. Documents and embeddings | See how verified catalog content becomes vectors. | Selected profile, reproducible builder, provider configuration, provenance. | -| 2. Exact pgvector retrieval | Understand distance ordering with real filters. | Real database tests for ordering, eligibility, profile isolation, and paging policy. | +| 2. Exact pgvector retrieval (implemented) | Understand distance ordering with real filters. | Real database tests for ordering, eligibility, profile isolation, paging, freshness and atomic rebuild failure. | | 3. Full-text and RRF | Compare exact terminology with semantic intent. | Inspectable branch ranks and curated hybrid comparisons. | | 4. Indexing and backfill | Maintain rebuildable derived state. | Duplicate/out-of-order handling, stale-write protection, updates/deletes, recovery. | | 5. HNSW benchmark | Measure speed versus recall at meaningful scale. | Exact baseline, larger corpus, real query plans, latency and recall results. | @@ -625,7 +698,7 @@ ANN recall and human relevance are different measurements. A fast index can fait ## Implementation map -These existing files are the starting points, not claims that the proposed functionality already lives there: +These files own the implemented behavior and the foundations for later milestones: | Existing surface | Relevance | |---|---| @@ -634,7 +707,7 @@ These existing files are the starting points, not claims that the proposed funct | [Ollama client](../src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs) | Task prefixes, model profile, explicit truncation policy, and response validation. | | [Embedding client tests](../tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs) | Mocked success/error responses and exact request construction. | | [Live embedding smoke check](../../tools/test-product-embeddings.ps1) | Explicit local-model probe and retained relevance expectations. | -| [Product query configuration](../src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs) | Current keyword fields and future server-selected ranked mode. | +| [Product query configuration](../src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs) | Server-selected semantic mode and existing filter/order fields. | | [Product repository](../src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs) | Existing filters, joins, ordering, mapping, and paging to preserve. | | [Product read service](../src/Contoso.Products.Application/ProductReadService.cs) | Current query delegation through the Application boundary. | | [Product service](../src/Contoso.Products.Application/ProductService.cs) | Atomic product mutations and outbox events. | @@ -643,8 +716,12 @@ These existing files are the starting points, not claims that the proposed funct | [DbContext](../src/Contoso.Products.Infrastructure/Repositories/ProductsDbContext.cs) | Hand-authored EF/Npgsql composition; generated partials remain generator-owned. | | [Subscribe host](../src/Contoso.Products.Subscribe/Program.cs) | Existing message receiving/composition foundations. | | [MCP tools](../src/Contoso.Mcp/Tools/CommerceTools.cs) | API-only search forwarding and tool description. | -| [Compose infrastructure](../../docker-compose.yml) | PostgreSQL image and potential local inference resource. | -| [Product query tests](../tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs) | Current keyword API behavior and schema expectations. | +| [Compose infrastructure](../../docker-compose.yml) | Pinned PostgreSQL/pgvector image and optional local inference resource. | +| [Search projection](../src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs) | Exact distances, compatibility/freshness checks and atomic manual rebuild. | +| [Search migration](../src/Contoso.Products.Database/Migrations/20261013-182033-create-products-product_search-table.pgsql) | Derived table, vector extension and constraints. | +| [Product query tests](../tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs) | Semantic API behavior and schema expectations. | +| [Semantic database tests](../tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs) | Controlled-vector ranking, eligibility, paging, failures and rebuild behavior. | +| [Real API smoke](../../tools/test-product-search.ps1) | Real-model, full-catalog REST/GraphQL-lite checks. | See also [layer boundaries](layers.md), [Infrastructure](infrastructure-layer.md), [messaging hosts](hosts-layer.md), [tooling](tooling.md), [testing](testing.md), and [Aspire](aspire.md). diff --git a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs index 4e53a82cd..07f5f370f 100644 --- a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs +++ b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs @@ -3,11 +3,11 @@ namespace Contoso.Mcp.Tools; [McpServerToolType] public static class CommerceTools { - /// Searches products by free-text keywords. - [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products using free-text keywords; each word must appear in the product name or be the beginning of the product SKU. Returns up to the requested number of matching product summaries.")] + /// Searches products by semantic relevance. + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products by meaning using semantic search over names, descriptions and product classifications. Returns up to the requested number of product summaries, closest first; results are candidates, not guaranteed matches. Requires a current search index.")] public static Task SearchProductsAsync( [FromServices] IHttpClientFactory clientFactory, - [Description("One or more keywords to find in product names, or the beginning of a product SKU.")] string searchTerm, + [Description("Describe the product or intended use to search for. Exact SKU matching is not guaranteed by semantic search.")] string searchTerm, [Description("Maximum number of products to return (1-25).")] int take = 10, CancellationToken cancellationToken = default) { diff --git a/samples/src/Contoso.Products.Api/GlobalUsing.cs b/samples/src/Contoso.Products.Api/GlobalUsing.cs index 551e5bf37..c725063cf 100644 --- a/samples/src/Contoso.Products.Api/GlobalUsing.cs +++ b/samples/src/Contoso.Products.Api/GlobalUsing.cs @@ -1,6 +1,7 @@ global using Contoso.Products.Application; global using Contoso.Products.Application.Interfaces; global using Contoso.Products.Contracts; +global using Contoso.Products.Infrastructure.Clients.Ollama; global using Contoso.Products.Infrastructure.Repositories; global using CoreEx; global using CoreEx.AspNetCore.Mvc; diff --git a/samples/src/Contoso.Products.Api/Program.cs b/samples/src/Contoso.Products.Api/Program.cs index 54cff7212..2fbca5674 100644 --- a/samples/src/Contoso.Products.Api/Program.cs +++ b/samples/src/Contoso.Products.Api/Program.cs @@ -10,7 +10,7 @@ namespace Contoso.Products.Api; public class Program { - private static void Main(string[] args) + private static async Task Main(string[] args) { // Create the web builder. var builder = WebApplication.CreateBuilder(args); @@ -29,7 +29,7 @@ private static void Main(string[] args) // Add all the dynamically registered services. builder.Services.AddDynamicServicesUsing(); - builder.AddTypedHttpClient("Ollama"); + builder.AddProductEmbeddings(); // Add L1/L2 caching services. builder.Services.AddMemoryCache(); // Adds the in-memory cache - L1. @@ -89,7 +89,7 @@ private static void Main(string[] args) .UseOtlpExporter(); // Build the application. - var app = builder.Build(); + await using var app = builder.Build(); // Configure the pipeline/middleware (order is important). app.UseCoreExExceptionHandler(); @@ -109,6 +109,6 @@ private static void Main(string[] args) app.MapHealthChecks(); // Secure by default: detailed endpoints are disabled unless explicitly enabled (HealthCheckOptions.AreDetailedEndpointsEnabled) and secured (detailedGroupConfigure, e.g. g => g.RequireAuthorization()); basic live/startup/ready checks stay anonymous for orchestrator probes. // Run the application. - app.Run(); + await app.RunAsync().ConfigureAwait(false); } } diff --git a/samples/src/Contoso.Products.Database/Migrations/20261013-182033-create-products-product_search-table.pgsql b/samples/src/Contoso.Products.Database/Migrations/20261013-182033-create-products-product_search-table.pgsql new file mode 100644 index 000000000..c8781be88 --- /dev/null +++ b/samples/src/Contoso.Products.Database/Migrations/20261013-182033-create-products-product_search-table.pgsql @@ -0,0 +1,13 @@ +-- Derived search data is maintained explicitly, not as a CRUD entity in DbEx. +CREATE EXTENSION IF NOT EXISTS vector; + +CREATE TABLE "products"."product_search" ( + "product_id" VARCHAR(50) NOT NULL REFERENCES "products"."product" ("product_id") ON DELETE CASCADE, + "profile" VARCHAR(250) NOT NULL, + "recipe" VARCHAR(50) NOT NULL, + "source_hash" CHAR(64) NOT NULL, + "document" TEXT NOT NULL, + "embedding" vector(768) NOT NULL CHECK (vector_norm("embedding") > 0), + "indexed_on" TIMESTAMPTZ NOT NULL, + PRIMARY KEY ("product_id", "profile", "recipe") +); diff --git a/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHostingExtensions.cs b/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHostingExtensions.cs new file mode 100644 index 000000000..87fdfb940 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHostingExtensions.cs @@ -0,0 +1,23 @@ +namespace Contoso.Products.Infrastructure.Clients.Ollama; + +/// Registers the Aspire Ollama client and Products-specific embedding validation. +public static class OllamaHostingExtensions +{ + /// Gets the named HTTP client used by the pinned Aspire OllamaSharp integration. + public const string HttpClientName = "Ollama_httpClient"; + + /// Adds embeddings using the configured external Ollama endpoint, service discovery and resilience. + public static void AddProductEmbeddings(this IHostApplicationBuilder builder) + { + var config = builder.Configuration.GetRequiredSection("Ollama"); + builder.AddOllamaApiClient("Ollama", settings => + { + settings.Endpoint = new Uri(config["BaseAddress"].ThrowIfNullOrEmpty()); + settings.SelectedModel = OllamaHttpClient.ModelName; + }); + + builder.AddNamedHttpClient(HttpClientName, client => client.Timeout = Timeout.InfiniteTimeSpan, configurationSectionName: "Ollama"); + + builder.Services.AddSingleton(); + } +} diff --git a/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs b/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs index bd9e84ab4..b913a1641 100644 --- a/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs +++ b/samples/src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs @@ -3,8 +3,7 @@ namespace Contoso.Products.Infrastructure.Clients.Ollama; /// /// Provides validated, full-dimensional Nomic retrieval embeddings through the local Ollama API. /// -/// The typed client configured for the Ollama endpoint. -public sealed class OllamaHttpClient(HttpClient httpClient) +public sealed class OllamaHttpClient { /// Gets the versioned model tag used for this sample's embedding profile. public const string ModelName = "nomic-embed-text:v1.5"; @@ -15,15 +14,24 @@ public sealed class OllamaHttpClient(HttpClient httpClient) /// Gets the profile identifying model, dimensions and task-prefix recipe. public const string Profile = "ollama:nomic-embed-text:v1.5:768:retrieval-prefix-v1"; - // Ollama defaults to truncation when the false value is omitted; do not use CoreEx's omit-default serialization here. - private static readonly JsonSerializerOptions _requestOptions = new(JsonSerializerDefaults.Web); - - private static readonly Validator _validator = Validator.Create() + private static readonly Validator _validator = Validator.Create() .HasProperty(r => r.Model, c => c.Mandatory().Error("The embedding response model does not match the requested profile.").WhenValue(v => v != ModelName)) .HasProperty(r => r.Embeddings, c => c.Mandatory().Error("The embedding response must contain one finite, non-zero 768-dimensional vector.") .WhenValue(v => v is not [{ Length: Dimensions } vector] || vector.Any(x => !float.IsFinite(x)) || !vector.Any(x => x != 0))); - private readonly HttpClient _httpClient = httpClient.ThrowIfNull(); + private readonly IOllamaApiClient _client; + + /// Creates the validated wrapper around the Aspire-registered OllamaSharp client. + /// The SDK client configured for the Ollama endpoint. + public OllamaHttpClient(IOllamaApiClient client) + { + _client = client.ThrowIfNull(); + if (client is not OllamaApiClient sdk) + throw new ArgumentException("The embedding wrapper requires the Aspire OllamaSharp client.", nameof(client)); + + // The SDK's EmbedResponse omits model; retain it to validate each response against the pinned profile. + sdk.IncomingJsonSerializerOptions.Converters.Add(new EmbedResponseConverter()); + } /// Embeds product text using Nomic's document instruction. public Task> EmbedDocumentAsync(string text, CancellationToken ct = default) => EmbedAsync(text, "search_document: ", ct); @@ -32,22 +40,41 @@ public sealed class OllamaHttpClient(HttpClient httpClient) public Task> EmbedQueryAsync(string text, CancellationToken ct = default) => EmbedAsync(text, "search_query: ", ct); /// Requests one embedding without silently truncating oversized input. - private Task> EmbedAsync(string text, string prefix, CancellationToken ct) + private async Task> EmbedAsync(string text, string prefix, CancellationToken ct) { text = text.ThrowIfNull().Trim().ThrowIfNullOrEmpty(nameof(text)); - return _httpClient.SendValue(HttpMethod.Post, "api/embed", new { model = ModelName, input = prefix + text, truncate = false, keep_alive = "5m" }, jsonSerializerOptions: _requestOptions) - .WithValidator(_validator) - .ToResultAsync(ct) - .ThenAs(r => r.Embeddings is [{ Length: Dimensions } vector] ? vector : throw new HttpRequestException("The validated embedding response contains no vector.")); + try + { + var response = await _client.EmbedAsync(new EmbedRequest { Model = ModelName, Input = [prefix + text], Truncate = false, KeepAlive = "5m" }, ct).ConfigureAwait(false); + if (response is not ValidatedEmbedResponse validated) + return Result.Fail(new HttpRequestException("The embedding response contains no validated model metadata.")); + + var validation = await _validator.ValidateAsync(validated, ct).ConfigureAwait(false); + if (validation.HasErrors) + return Result.Fail(new HttpRequestException("The embedding response is invalid.", validation.ToException())); + + return validated.Embeddings is [{ Length: Dimensions } vector] + ? Result.Ok(vector) + : Result.Fail(new HttpRequestException("The validated embedding response contains no vector.")); + } + catch (Exception ex) when (ex is HttpRequestException or OllamaException or JsonException) + { + return Result.Fail(ex is HttpRequestException ? ex : new HttpRequestException("Ollama could not produce a valid embedding response.", ex)); + } } - /// Represents only the fields consumed from Ollama's embedding response. - private sealed class EmbedResponse + private sealed class ValidatedEmbedResponse : EmbedResponse { - /// Gets or sets the model that produced the vector. + [JsonPropertyName("model")] public string? Model { get; set; } + } + + private sealed class EmbedResponseConverter : JsonConverter + { + public override EmbedResponse? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => JsonSerializer.Deserialize(ref reader, options); - /// Gets or sets the embedding batch; this client requires exactly one vector. - public float[][]? Embeddings { get; set; } + public override void Write(Utf8JsonWriter writer, EmbedResponse value, JsonSerializerOptions options) + => throw new NotSupportedException("Embedding responses are read-only."); } } diff --git a/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj b/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj index 49255747e..e69dd2fc3 100644 --- a/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj +++ b/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj @@ -7,6 +7,7 @@ + - \ No newline at end of file + diff --git a/samples/src/Contoso.Products.Infrastructure/GlobalUsing.cs b/samples/src/Contoso.Products.Infrastructure/GlobalUsing.cs index de18e5b6c..5363f06e1 100644 --- a/samples/src/Contoso.Products.Infrastructure/GlobalUsing.cs +++ b/samples/src/Contoso.Products.Infrastructure/GlobalUsing.cs @@ -1,5 +1,6 @@ global using Contoso.Products.Application.Adapters.Embeddings; global using Contoso.Products.Application.Repositories; +global using Contoso.Products.Application.Search; global using Contoso.Products.Infrastructure.Clients.Ollama; global using Contoso.Products.Infrastructure.Mapping; global using CoreEx; @@ -16,9 +17,19 @@ global using CoreEx.Events; global using CoreEx.Events.Publishing; global using CoreEx.Mapping; +global using CoreEx.RefData; global using CoreEx.Results; global using CoreEx.Validation; global using Microsoft.EntityFrameworkCore; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Hosting; global using Microsoft.Extensions.Logging; +global using OllamaSharp; +global using OllamaSharp.Models; +global using OllamaSharp.Models.Exceptions; +global using System.Data; +global using System.Globalization; global using System.Text.Json; +global using System.Text.Json.Serialization; global using ExecutionContext = CoreEx.ExecutionContext; diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs index c3f734bce..9eea70424 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs @@ -25,11 +25,8 @@ public ProductQueryArgsConfig() .AddField(nameof(Contracts.ProductBase.Text)) .AddField(nameof(Contracts.ProductBase.Brand))); - // Configure the free-text $search; keyword mode (unranked) where each term must match the Text (contains) or Sku (starts-with). A semantic/hybrid (vector) search would - // instead use WithMode(QuerySearchMode.Semantic) and be executed by the repository (see QueryArgsParseResult.IsRankedSearch). + // Semantic search is executed by the repository; ordinary queries retain the configured SKU ordering. WithSearch(search => search - .WithDefaultModelPrefix("Product") - .AddKeywordField(nameof(Contracts.ProductBase.Text), c => c.AsUpperCase()) - .AddKeywordField(nameof(Contracts.ProductBase.Sku), c => c.WithOperator(QueryFilterOperator.StartsWith).AsUpperCase())); + .WithMode(QuerySearchMode.Semantic)); } } diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs index 4bcc7b21d..94b9e899c 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs @@ -1,9 +1,11 @@ namespace Contoso.Products.Infrastructure.Repositories; [ScopedService] -public class ProductRepository(ProductsEfDb ef) : IProductRepository +public class ProductRepository(ProductsEfDb ef, ProductSearchIndex searchIndex, PostgresUnitOfWork unitOfWork) : IProductRepository { private readonly ProductsEfDb _ef = ef.ThrowIfNull(); + private readonly ProductSearchIndex _searchIndex = searchIndex.ThrowIfNull(); + private readonly PostgresUnitOfWork _unitOfWork = unitOfWork.ThrowIfNull(); public Task GetAsync(string id, CancellationToken ct = default) => _ef.Products.GetAsync(id, ct); @@ -18,7 +20,6 @@ public class ProductRepository(ProductsEfDb ef) : IProductRepository public async Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken ct = default) { var parsed = ProductQueryArgsConfig.Default.Parse(query).ThrowOnError(); - var products = _ef.Products.Model.Query(); if (query?.IsIncludeInactive is false) products = products.Where(p => !p.IsInactive); @@ -40,21 +41,57 @@ from i in ig.DefaultIfEmpty() QtyOnHand = i == null ? 0 : i.QtyOnHand }; - return await q.Where(parsed).OrderBy(parsed).ToMappedItemsResultAsync(x => new Contracts.ProductLite + q = q.Where(parsed); + float[]? vector = null; + if (parsed.IsRankedSearch) { - Id = x.Product.Id, - Sku = x.Product.Sku, - Text = x.Product.Text, - Description = x.Product.Description, - CategoryCode = x.CategoryCode, - SubCategoryCode = x.SubCategoryCode, - BrandCode = x.Product.BrandCode, - UnitOfMeasureCode = x.Product.UnitOfMeasureCode, - Price = x.Product.Price, - IsInactive = x.Product.IsInactive, - IsNonStocked = x.Product.IsNonStocked, - QtyOnHand = x.QtyOnHand - }, paging, cancellationToken: ct); + EnsureSnapshotIsolation(); + // Fail fast before inference, then recheck in the retrieval snapshot to cover writes during inference. + await _searchIndex.EnsureFreshAsync(await q.Select(x => x.Product).AsNoTracking().ToListAsync(ct).ConfigureAwait(false), ct).ConfigureAwait(false); + vector = await _searchIndex.EmbedQueryAsync((parsed.SearchResult?.Text).ThrowIfNull(), ct).ConfigureAwait(false); + } + + return vector is null + ? await ReadAsync(ct).ConfigureAwait(false) + : await _unitOfWork.TransactionAsync(_unitOfWork.Database.DbArgs with { IsolationLevel = IsolationLevel.RepeatableRead }, ReadAsync, ct).ConfigureAwait(false); + + async Task> ReadAsync(CancellationToken token) + { + if (vector is not null) + { + EnsureSnapshotIsolation(); + await _searchIndex.EnsureFreshAsync(await q.Select(x => x.Product).AsNoTracking().ToListAsync(token).ConfigureAwait(false), token).ConfigureAwait(false); + q = (from candidate in q + join match in _searchIndex.Matches(vector) on candidate.Product.Id equals match.Id + orderby match.Distance, candidate.Product.Id + select candidate); + } + else + q = q.OrderBy(parsed); + + return await q.ToMappedItemsResultAsync(x => new Contracts.ProductLite + { + Id = x.Product.Id, + Sku = x.Product.Sku, + Text = x.Product.Text, + Description = x.Product.Description, + CategoryCode = x.CategoryCode, + SubCategoryCode = x.SubCategoryCode, + BrandCode = x.Product.BrandCode, + UnitOfMeasureCode = x.Product.UnitOfMeasureCode, + Price = x.Product.Price, + IsInactive = x.Product.IsInactive, + IsNonStocked = x.Product.IsNonStocked, + QtyOnHand = x.QtyOnHand + }, paging, cancellationToken: token).ConfigureAwait(false); + } + } + + /// Rejects a parent transaction that cannot provide a consistent PostgreSQL read snapshot. + private void EnsureSnapshotIsolation() + { + if (_unitOfWork.Database.CurrentTransaction is { } transaction && transaction.IsolationLevel is not (IsolationLevel.RepeatableRead or IsolationLevel.Serializable)) + throw new InvalidOperationException("Semantic product search requires a RepeatableRead or Serializable parent transaction."); } public Task> GetForReservationAsync(string[] ids, CancellationToken ct = default) diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs new file mode 100644 index 000000000..207944195 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs @@ -0,0 +1,130 @@ +namespace Contoso.Products.Infrastructure.Repositories; + +/// Maintains and queries the rebuildable, exact pgvector search projection. +/// The Products database accessor. +/// The compatible document/query embedding provider. +/// The reference-data labels used by the canonical document. +/// The shared PostgreSQL transaction coordinator. +[ScopedService] +public sealed class ProductSearchIndex(ProductsEfDb ef, IProductEmbeddingAdapter embeddings, ReferenceDataOrchestrator referenceData, PostgresUnitOfWork unitOfWork) +{ + private readonly ProductsEfDb _ef = ef.ThrowIfNull(); + private readonly IProductEmbeddingAdapter _embeddings = embeddings.ThrowIfNull(); + private readonly ReferenceDataOrchestrator _referenceData = referenceData.ThrowIfNull(); + private readonly PostgresUnitOfWork _unitOfWork = unitOfWork.ThrowIfNull(); + + /// Embeds search text before opening a database snapshot. + public async Task EmbedQueryAsync(string text, CancellationToken ct = default) + { + var vector = (await _embeddings.EmbedQueryAsync(text, ct).ConfigureAwait(false)).ThrowOnError().Value; + ValidateVector(vector); + return vector; + } + + /// Returns composable exact distances for the selected model profile and document recipe. + /// Vector text is a bound SQL parameter, not interpolated SQL syntax; no provider-specific EF extension is required. + public IQueryable Matches(float[] vector) + { + var serialized = SerializeVector(vector); + return _ef.DbContext.Database.SqlQuery($""" + SELECT product_id AS "Id", + embedding <=> CAST({serialized} AS vector(768)) AS "Distance" + FROM products.product_search + WHERE profile = {_embeddings.Profile} AND recipe = {ProductSearchDocument.TemplateVersion} + """); + } + + /// Refuses incomplete or stale eligible results rather than silently dropping products. + public async Task EnsureFreshAsync(IReadOnlyCollection products, CancellationToken ct = default) + { + if (products.Count == 0) + return; + await LoadLabelsAsync(ct).ConfigureAwait(false); + var hashes = await _ef.DbContext.Database.SqlQuery($""" + SELECT product_id AS "Id", source_hash AS "SourceHash" + FROM products.product_search + WHERE profile = {_embeddings.Profile} AND recipe = {ProductSearchDocument.TemplateVersion} + """).ToDictionaryAsync(x => x.Id, x => x.SourceHash, ct).ConfigureAwait(false); + + if (products.Any(product => !hashes.TryGetValue(product.Id, out var hash) || hash != Document(product).SourceHash)) + throw new TransientException("Product search is missing or stale. Rebuild the search projection before searching."); + } + + /// Builds embeddings outside transactions and atomically replaces this profile's projection. + /// A short publish-time lock rejects concurrent catalog edits instead of installing stale documents. + public async Task RebuildAsync(CancellationToken ct = default) + { + await LoadLabelsAsync(ct).ConfigureAwait(false); + var products = await _ef.Products.Model.Query().AsNoTracking().OrderBy(p => p.Id).ToListAsync(ct).ConfigureAwait(false); + var documents = products.Select(product => (Product: product, Document: Document(product))).ToArray(); + var vectors = new List(documents.Length); + foreach (var item in documents) + { + var vector = (await _embeddings.EmbedDocumentAsync(item.Document.Text, ct).ConfigureAwait(false)).ThrowOnError().Value; + vectors.Add(SerializeVector(vector)); + } + + return await _unitOfWork.TransactionAsync(async token => + { + await _ef.DbContext.Database.ExecuteSqlRawAsync("LOCK TABLE products.product IN SHARE MODE", token).ConfigureAwait(false); + var current = await _ef.Products.Model.Query().AsNoTracking().OrderBy(p => p.Id).ToListAsync(token).ConfigureAwait(false); + if (!current.Select(p => (p.Id, Document(p).SourceHash)).SequenceEqual(documents.Select(d => (d.Product.Id, d.Document.SourceHash)))) + throw new ConcurrencyException("The product catalog changed during embedding. Rebuild the search projection again."); + + await _ef.DbContext.Database.ExecuteSqlInterpolatedAsync($""" + DELETE FROM products.product_search WHERE profile = {_embeddings.Profile} AND recipe = {ProductSearchDocument.TemplateVersion} + """, token).ConfigureAwait(false); + for (var i = 0; i < documents.Length; i++) + { + var item = documents[i]; + await _ef.DbContext.Database.ExecuteSqlInterpolatedAsync($""" + INSERT INTO products.product_search (product_id, profile, recipe, source_hash, document, embedding, indexed_on) + VALUES ({item.Product.Id}, {_embeddings.Profile}, {ProductSearchDocument.TemplateVersion}, + {item.Document.SourceHash}, {item.Document.Text}, CAST({vectors[i]} AS vector(768)), {Runtime.UtcNow}) + """, token).ConfigureAwait(false); + } + + return documents.Length; + }, ct).ConfigureAwait(false); + } + + /// Loads the labels before synchronous generated navigation getters are used. + private async Task LoadLabelsAsync(CancellationToken ct) + => await _referenceData.PrefetchAsync([typeof(Contracts.Brand), typeof(Contracts.Category), typeof(Contracts.SubCategory)], ct).ConfigureAwait(false); + + /// Uses the same category derivation as the application contract. + private static ProductSearchDocument Document(Persistence.Product product) + => ProductSearchDocument.Create(ProductMapper.From.Map(product)); + + /// Validates vectors at the persistence and query boundaries. + private void ValidateVector(float[] vector) + { + if (_embeddings.Dimensions != OllamaHttpClient.Dimensions || vector.Length != OllamaHttpClient.Dimensions || vector.Any(x => !float.IsFinite(x)) || !vector.Any(x => x != 0)) + throw new InvalidOperationException("Product search requires a finite, non-zero 768-dimensional embedding."); + } + + /// Formats validated vectors independently of locale. + private string SerializeVector(float[] vector) + { + ValidateVector(vector); + return $"[{string.Join(",", vector.Select(x => x.ToString("R", CultureInfo.InvariantCulture)))}]"; + } + + /// Represents an unmapped, scalar SQL projection; the vector stays in PostgreSQL. + public sealed class SearchMatch + { + /// Gets or sets the product identifier. + public string Id { get; set; } = string.Empty; + /// Gets or sets cosine distance; lower is closer. + public double Distance { get; set; } + } + + /// Represents the indexed content fingerprint without loading vector values. + public sealed class SearchFingerprint + { + /// Gets or sets the product identifier. + public string Id { get; set; } = string.Empty; + /// Gets or sets the indexed document fingerprint. + public string SourceHash { get; set; } = string.Empty; + } +} diff --git a/samples/src/Contoso.Products.Subscribe/GlobalUsing.cs b/samples/src/Contoso.Products.Subscribe/GlobalUsing.cs index 2a5358dc1..09420ff62 100644 --- a/samples/src/Contoso.Products.Subscribe/GlobalUsing.cs +++ b/samples/src/Contoso.Products.Subscribe/GlobalUsing.cs @@ -1,8 +1,9 @@ global using Contoso.Products.Application.Interfaces; +global using Contoso.Products.Infrastructure.Clients.Ollama; global using CoreEx; global using CoreEx.DependencyInjection; global using CoreEx.Events; global using CoreEx.Events.Subscribing; global using CoreEx.Json; global using CoreEx.Results; -global using CoreEx.Validation; \ No newline at end of file +global using CoreEx.Validation; diff --git a/samples/src/Contoso.Products.Subscribe/Program.cs b/samples/src/Contoso.Products.Subscribe/Program.cs index 48b68c424..7fb6d5aca 100644 --- a/samples/src/Contoso.Products.Subscribe/Program.cs +++ b/samples/src/Contoso.Products.Subscribe/Program.cs @@ -32,6 +32,7 @@ private static void Main(string[] args) // Add all the dynamically registered services. builder.Services.AddDynamicServicesUsing(); + builder.AddProductEmbeddings(); // Add L1/L2 caching services. builder.Services.AddMemoryCache(); // Adds the in-memory cache - L1. diff --git a/samples/src/Contoso.Products.Subscribe/appsettings.Development.json b/samples/src/Contoso.Products.Subscribe/appsettings.Development.json index 51595f3b7..40e9f6fc2 100644 --- a/samples/src/Contoso.Products.Subscribe/appsettings.Development.json +++ b/samples/src/Contoso.Products.Subscribe/appsettings.Development.json @@ -1,4 +1,18 @@ { + "Ollama": { + "BaseAddress": "http://localhost:11434/", + "Resilience": { + "AttemptTimeout": { + "Timeout": "00:01:00" + }, + "TotalRequestTimeout": { + "Timeout": "00:02:00" + }, + "CircuitBreaker": { + "SamplingDuration": "00:02:00" + } + } + }, "Logging": { "LogLevel": { "Default": "Information", @@ -34,4 +48,4 @@ } } } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.Products.Test.Api/GlobalUsing.cs b/samples/tests/Contoso.Products.Test.Api/GlobalUsing.cs index 41ac99fa2..9fbbabb09 100644 --- a/samples/tests/Contoso.Products.Test.Api/GlobalUsing.cs +++ b/samples/tests/Contoso.Products.Test.Api/GlobalUsing.cs @@ -1,12 +1,20 @@ global using AwesomeAssertions; +global using Contoso.Products.Application.Adapters.Embeddings; +global using Contoso.Products.Application.Repositories; global using Contoso.Products.Contracts; +global using Contoso.Products.Infrastructure.Clients.Ollama; +global using Contoso.Products.Infrastructure.Repositories; global using CoreEx; +global using CoreEx.Data; +global using CoreEx.Database.Postgres; global using CoreEx.Http.Abstractions; global using CoreEx.Json; +global using CoreEx.Results; global using DbMigration = Contoso.Products.Database.Program; global using Microsoft.EntityFrameworkCore; global using Microsoft.Extensions.DependencyInjection; global using NUnit.Framework; +global using System.Data; global using System.Net; global using System.Net.Http.Json; global using System.Text.Json; diff --git a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs index e48c12777..00b3d6cb4 100644 --- a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs +++ b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs @@ -102,21 +102,20 @@ public void Product_Query_FilterByBrandAndSubCategory() public void Product_Query_Search_MultipleTerms() { var r = Test.Http() - .Run(HttpMethod.Get, "/api/products?$search=rear derailleur") + .Run(HttpMethod.Get, "/api/products?$search=rear derailleur&$take=3") .AssertOK() .Value; r.Should().NotBeNull() .And.HaveCount(3) - .And.BeInAscendingOrder(x => x.Sku) .And.OnlyContain(p => p.Text!.Contains("Rear Derailleur")); } [Test] - public void Product_Query_Search_SkuPrefix() + public void Product_Query_Search_SkuIntent() { var r = Test.Http() - .Run(HttpMethod.Get, "/api/products?$search=sram-xx1") + .Run(HttpMethod.Get, "/api/products?$search=sram-xx1&$take=1") .AssertOK() .Value; @@ -128,14 +127,9 @@ public void Product_Query_Search_SkuPrefix() [Test] public void Product_Query_Search_WithFilterAndOrderBy() { - var r = Test.Http() + Test.Http() .Run(HttpMethod.Get, "/api/products?$search=cleaner&$filter=brand eq 'MUCOFF'&$orderby=text desc") - .AssertOK() - .Value; - - r.Should().NotBeNull() - .And.HaveCount(2) - .And.BeInDescendingOrder(x => x.Text); + .AssertBadRequest(); } [Test] @@ -169,7 +163,7 @@ public void Product_Query_FilterByDecimalPrice(string filter) public void Product_Query_Search_WithPriceFilter() { var products = Test.Http() - .Run(HttpMethod.Get, "/api/products?$search=cleaner&$filter=price lt 20") + .Run(HttpMethod.Get, "/api/products?$search=cleaner&$filter=price lt 20&$take=1") .AssertOK() .Value!; @@ -178,11 +172,11 @@ public void Product_Query_Search_WithPriceFilter() } [Test] - public void Product_Query_Search_TooManyTerms() + public void Product_Query_Search_TooLong() { Test.Http() - .Run(HttpMethod.Get, "/api/products?$search=a b c d e f g h i") + .Run(HttpMethod.Get, $"/api/products?$search={new string('a', 201)}") .AssertBadRequest() - .AssertErrors(new ApiError("$search", "Statement must not contain more than 8 distinct terms.")); + .AssertErrors(new ApiError("$search", "Statement must not exceed 200 characters in length.")); } } diff --git a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs new file mode 100644 index 000000000..3efac42fd --- /dev/null +++ b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs @@ -0,0 +1,249 @@ +namespace Contoso.Products.Test.Api; + +public partial class ReadTests +{ + [TestCase(IsolationLevel.RepeatableRead)] + [TestCase(IsolationLevel.Serializable)] + public void Product_Search_EnlistsInCompatibleParent(IsolationLevel isolation) => Test.Scoped(test => test.Run(async _ => + { + var ef = test.Services.GetRequiredService(); + var uow = test.Services.GetRequiredService(); + var repository = test.Services.GetRequiredService(); + await uow.TransactionAsync(new PostgresDatabaseArgs { IsolationLevel = isolation }, async token => + { + var parent = uow.Database.CurrentTransaction; + var result = await repository.QueryAsync(new QueryArgs { Search = "changing gears" }, PagingArgs.CreateWithCount(take: 1), token).ConfigureAwait(false); + result.Items.Should().ContainSingle().Which.Sku.Should().Be("SHIMANO-XT-M8100-RD"); + result.Paging!.TotalCount.Should().Be(29); + uow.Database.CurrentTransaction.Should().BeSameAs(parent); + ef.DbContext.Database.CurrentTransaction.Should().NotBeNull(); + uow.Events.IsEmpty.Should().BeTrue(); + }).ConfigureAwait(false); + uow.Database.CurrentTransaction.Should().BeNull(); + ef.DbContext.Database.CurrentTransaction.Should().BeNull(); + }).AssertSuccess()); + + [Test] + public void Product_Search_RejectsWeakParentWithoutInference() => Test.Scoped(test => test.Run(async _ => + { + var ef = test.Services.GetRequiredService(); + var uow = test.Services.GetRequiredService(); + var repository = test.Services.GetRequiredService(); + await uow.TransactionAsync(async token => + { + var parent = uow.Database.CurrentTransaction; + var queries = _embeddings.QueryCallCount; + Func search = () => repository.QueryAsync(new QueryArgs { Search = "cleaner" }, null, token); + await search.Should().ThrowAsync() + .WithMessage("Semantic product search requires a RepeatableRead or Serializable parent transaction.").ConfigureAwait(false); + _embeddings.QueryCallCount.Should().Be(queries); + uow.Database.CurrentTransaction.Should().BeSameAs(parent); + (await repository.QueryAsync(null, PagingArgs.Create(take: 1), token).ConfigureAwait(false)).Items.Should().ContainSingle(); + }).ConfigureAwait(false); + uow.Database.CurrentTransaction.Should().BeNull(); + ef.DbContext.Database.CurrentTransaction.Should().BeNull(); + }).AssertSuccess()); + + [Test] + public void Product_Search_ExactDistancePagingAndCount() + { + var result = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=a component for changing gears&$skip=1&$take=2&$count=true") + .AssertOK(); + + result.Value!.Select(p => p.Id).Should().Equal(22.ToGuid().ToString(), 23.ToGuid().ToString()); + result.Response.Headers.Should().ContainKey("X-Paging-Total-Count").WhoseValue.Should().ContainSingle().Which.Should().Be("29"); + } + + [Test] + public void Product_Search_FiltersBeforeTake() + { + var result = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=a component for changing gears&$filter=brand eq 'SRAM' and price lt 160&$take=1&$count=true") + .AssertOK(); + + result.Value.Should().ContainSingle().Which.Sku.Should().Be("SRAM-GX-EAGLE-RD"); + result.Response.Headers.Should().ContainKey("X-Paging-Total-Count").WhoseValue.Should().ContainSingle().Which.Should().Be("1"); + } + + [Test] + public void Product_Search_InactiveAndDeletedEligibility() + { + var result = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=mountain bike&$inactive&$take=50") + .AssertOK() + .Value!; + + result.Should().HaveCount(31).And.Contain(p => p.Sku == "SPEC-EPIC-8-PRO"); + result.Should().NotContain(p => p.Id == 18.ToGuid().ToString()); + } + + [Test] + public void Product_Search_NoEligibleProducts() + { + Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=clean mud off my bike&$filter=price lt 0") + .AssertOK() + .Value.Should().BeEmpty(); + } + + [Test] + public void Product_Search_GraphQLMatchesRest() + { + var graphql = Test.Http() + .Run(HttpMethod.Post, "/api/query", new { query = "{ products(search: \"a component for changing gears\", where: {price: {lt: 130}}, first: 2) { edges { node { id sku } } } }" }) + .AssertOK() + .Value; + + graphql.TryGetProperty("errors", out _).Should().BeFalse(); + var rest = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=a component for changing gears&$filter=price lt 130&$take=2") + .AssertOK() + .Value!; + graphql.GetProperty("data").GetProperty("products").GetProperty("edges").EnumerateArray() + .Select(edge => edge.GetProperty("node").GetProperty("id").GetString()).Should().Equal(rest.Select(p => p.Id)); + } + + [TestCase("DELETE FROM products.product_search WHERE profile = 'tests:controlled-vectors:768'")] + [TestCase("UPDATE products.product_search SET profile = 'incompatible' WHERE profile = 'tests:controlled-vectors:768'")] + [TestCase("UPDATE products.product_search SET recipe = 'old-recipe' WHERE profile = 'tests:controlled-vectors:768'")] + [TestCase("UPDATE products.product_search SET source_hash = repeat('0', 64) WHERE profile = 'tests:controlled-vectors:768'")] + public void Product_Search_InvalidProjectionIsUnavailable(string sql) + { + try + { + ExecuteSearchSql(sql); + var queries = _embeddings.QueryCallCount; + Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=cleaner") + .Assert(HttpStatusCode.ServiceUnavailable); + _embeddings.QueryCallCount.Should().Be(queries); + Test.Http() + .Run(HttpMethod.Get, "/api/products?$take=1") + .AssertOK() + .Value.Should().ContainSingle(); + } + finally + { + ExecuteSearchSql("DELETE FROM products.product_search WHERE profile = 'incompatible' OR recipe = 'old-recipe'"); + RebuildSearch(); + } + } + + [TestCase("zero")] + [TestCase("nonFinite")] + [TestCase("wrongDimensions")] + public void Product_Search_InvalidVectorIsNotAValidResult(string shape) + { + _embeddings.QueryVector = new float[shape == "wrongDimensions" ? 767 : 768]; + if (shape != "zero") + _embeddings.QueryVector[0] = shape == "nonFinite" ? float.NaN : 1; + try + { + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.InternalServerError); + } + finally + { + _embeddings.QueryVector = null; + } + } + + [Test] + public void Product_Search_QueryEmbeddingFailureHasNoFallback() + { + _embeddings.FailQuery = true; + try + { + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.InternalServerError); + } + finally + { + _embeddings.FailQuery = false; + } + } + + [Test] + public void Product_Search_DescriptionEditRequiresRebuild() + { + var product = Test.Http().Run(HttpMethod.Get, $"/api/products/{29.ToGuid()}").AssertOK().Value!; + try + { + ExecuteSearchSql("UPDATE products.product SET description = 'Changed searchable meaning.' WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.ServiceUnavailable); + } + finally + { + Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().Database.ExecuteSqlInterpolatedAsync( + $"UPDATE products.product SET description = {product.Description} WHERE sku = 'MUCOFF-NANO-CLEANER'").ConfigureAwait(false)).AssertSuccess()); + RebuildSearch(); + } + } + + [Test] + public void Product_Search_PriceChangeIsLiveWithoutReembedding() + { + try + { + ExecuteSearchSql("UPDATE products.product SET price = 99 WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=cleaner&$filter=sku eq 'MUCOFF-NANO-CLEANER'") + .AssertOK().Value.Should().ContainSingle().Which.Price.Should().Be(99); + Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=cleaner&$filter=sku eq 'MUCOFF-NANO-CLEANER' and price lt 20") + .AssertOK().Value.Should().BeEmpty(); + } + finally + { + ExecuteSearchSql("UPDATE products.product SET price = 16.99 WHERE sku = 'MUCOFF-NANO-CLEANER'"); + } + } + + [Test] + public void Product_Search_FailedRebuildPreservesPublishedIndex() + { + _embeddings.DocumentCallCount = 0; + _embeddings.FailDocumentOnCall = 2; + try + { + Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false)).AssertException()); + Test.Http().Run(HttpMethod.Get, "/api/products?$search=changing gears&$take=3").AssertOK() + .Value.Should().HaveCount(3).And.OnlyContain(p => p.Text!.Contains("Rear Derailleur")); + } + finally + { + _embeddings.FailDocumentOnCall = null; + } + } + + [Test] + public void Product_Search_ConcurrentCatalogEditRejectsRebuild() + { + var product = Test.Http().Run(HttpMethod.Get, $"/api/products/{29.ToGuid()}").AssertOK().Value!; + try + { + Test.Scoped(test => test.Run(async _ => + { + _embeddings.BeforeDocument = async ct => + { + _embeddings.BeforeDocument = null; + await test.Services.GetRequiredService().Database.ExecuteSqlRawAsync( + "UPDATE products.product SET description = 'Concurrent edit during embedding.' WHERE sku = 'MUCOFF-NANO-CLEANER'", ct).ConfigureAwait(false); + }; + await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false); + }).AssertException()); + } + finally + { + _embeddings.BeforeDocument = null; + Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().Database.ExecuteSqlInterpolatedAsync( + $"UPDATE products.product SET description = {product.Description} WHERE sku = 'MUCOFF-NANO-CLEANER'").ConfigureAwait(false)).AssertSuccess()); + } + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner&$take=1").AssertOK().Value.Should().ContainSingle(); + } + + private void ExecuteSearchSql(string sql) => Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().Database.ExecuteSqlRawAsync(sql).ConfigureAwait(false)).AssertSuccess()); +} diff --git a/samples/tests/Contoso.Products.Test.Api/ReadTests.cs b/samples/tests/Contoso.Products.Test.Api/ReadTests.cs index 7b42a4caf..f8fe15444 100644 --- a/samples/tests/Contoso.Products.Test.Api/ReadTests.cs +++ b/samples/tests/Contoso.Products.Test.Api/ReadTests.cs @@ -2,10 +2,16 @@ namespace Contoso.Products.Test.Api; public partial class ReadTests : WithApiTester { + private readonly TestEmbeddingAdapter _embeddings = new(); + [OneTimeSetUp] public async Task OneTimeSetUpAsync() { + Test.ReplaceScoped(_ => _embeddings); await Test.MigratePostgresDataAsync(["read-data.seed.yaml"], DbMigration.ConfigureMigrationArgs).ConfigureAwait(false); await Test.ClearFusionCacheAsync().ConfigureAwait(false); + RebuildSearch(); } -} \ No newline at end of file + + private void RebuildSearch() => Test.Scoped(test => test.Run(async _ => await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false)).AssertSuccess()); +} diff --git a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json index 9de99dca1..2de327ac6 100644 --- a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json +++ b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json @@ -81,9 +81,8 @@ "default": "sku asc" }, "search": { - "mode": "keyword", - "ranked": false, - "maxLength": 200, - "maxTerms": 8 + "mode": "semantic", + "ranked": true, + "maxLength": 200 } } diff --git a/samples/tests/Contoso.Products.Test.Api/TestEmbeddingAdapter.cs b/samples/tests/Contoso.Products.Test.Api/TestEmbeddingAdapter.cs new file mode 100644 index 000000000..572d027b0 --- /dev/null +++ b/samples/tests/Contoso.Products.Test.Api/TestEmbeddingAdapter.cs @@ -0,0 +1,45 @@ +namespace Contoso.Products.Test.Api; + +internal sealed class TestEmbeddingAdapter : IProductEmbeddingAdapter +{ + public string Profile => "tests:controlled-vectors:768"; + public int Dimensions => OllamaHttpClient.Dimensions; + public int? FailDocumentOnCall { get; set; } + public int DocumentCallCount { get; set; } + public int QueryCallCount { get; set; } + public float[]? QueryVector { get; set; } + public bool FailQuery { get; set; } + public Func? BeforeDocument { get; set; } + + public async Task> EmbedDocumentAsync(string text, CancellationToken ct = default) + { + if (BeforeDocument is not null) + await BeforeDocument(ct).ConfigureAwait(false); + if (++DocumentCallCount == FailDocumentOnCall) + throw new HttpRequestException("Controlled embedding failure."); + return await EmbedAsync(text, ct).ConfigureAwait(false); + } + public Task> EmbedQueryAsync(string text, CancellationToken ct = default) + { + QueryCallCount++; + if (FailQuery) + return Task.FromException>(new HttpRequestException("Controlled query embedding failure.")); + return QueryVector is null ? EmbedAsync(text, ct) : Task.FromResult(Result.Ok(QueryVector)); + } + + private static Task> EmbedAsync(string text, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + var vector = new float[OllamaHttpClient.Dimensions]; + vector[0] = 0.1f; + if (text.Contains("derailleur", StringComparison.OrdinalIgnoreCase) || text.Contains("changing gears", StringComparison.OrdinalIgnoreCase)) + vector[1] = 1; + else if (text.Contains("sram-xx1", StringComparison.OrdinalIgnoreCase)) + vector[2] = 1; + else if (text.Contains("cleaner", StringComparison.OrdinalIgnoreCase) || text.Contains("clean mud", StringComparison.OrdinalIgnoreCase)) + vector[3] = 1; + else + vector[4] = 1; + return Task.FromResult(Result.Ok(vector)); + } +} diff --git a/samples/tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs b/samples/tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs index cca7a37d3..629c4d00a 100644 --- a/samples/tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs +++ b/samples/tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs @@ -8,7 +8,7 @@ public class OllamaHttpClientTests : WithGenericTester public void OneTimeSetUp() { var factory = MockHttpClientFactory.Create(); - _request = factory.CreateClient("Ollama").Request(HttpMethod.Post, "api/embed"); + _request = factory.CreateClient(OllamaHostingExtensions.HttpClientName).Request(HttpMethod.Post, "api/embed"); _request.WithAnyBody(); Test.ReplaceHttpClientFactory(factory); } @@ -84,8 +84,9 @@ public async Task Embed_UsesCorrectInstructionAndDisablesTruncation(bool query, using (json) { json.RootElement.GetProperty("model").GetString().Should().Be(OllamaHttpClient.ModelName); - json.RootElement.GetProperty("input").GetString().Should().Be(prefix + "cleaner"); + json.RootElement.GetProperty("input").EnumerateArray().Should().ContainSingle().Which.GetString().Should().Be(prefix + "cleaner"); json.RootElement.GetProperty("truncate").GetBoolean().Should().BeFalse(); + json.RootElement.GetProperty("keep_alive").GetString().Should().Be("5m"); } return new HttpResponseMessage(HttpStatusCode.OK) @@ -94,7 +95,8 @@ public async Task Embed_UsesCorrectInstructionAndDisablesTruncation(bool query, }; }); using var http = new HttpClient(handler) { BaseAddress = new Uri("http://localhost:11434/") }; - var client = new OllamaHttpClient(http); + using var sdk = new OllamaApiClient(http); + var client = new OllamaHttpClient(sdk); var result = query ? await client.EmbedQueryAsync(" cleaner ") : await client.EmbedDocumentAsync(" cleaner "); result.IsSuccess.Should().BeTrue(); } @@ -102,11 +104,12 @@ public async Task Embed_UsesCorrectInstructionAndDisablesTruncation(bool query, [TestCase(null)] [TestCase("")] [TestCase(" ")] - public void Embed_BlankInputDoesNotCallProvider(string? text) + public async Task Embed_BlankInputDoesNotCallProvider(string? text) { - using var http = new HttpClient(); - Action action = () => new OllamaHttpClient(http).EmbedQueryAsync(text!); - action.Should().Throw(); + using var sdk = new OllamaApiClient(new Uri("http://localhost:11434/")); + var client = new OllamaHttpClient(sdk); + Func action = () => client.EmbedQueryAsync(text!); + await action.Should().ThrowAsync(); } [Test] @@ -118,7 +121,8 @@ public async Task Embed_NonFiniteCoordinate_ReturnsDependencyFailure() Content = new StringContent($"{{\"model\":\"{OllamaHttpClient.ModelName}\",\"embeddings\":[[{values}]]}}") })); using var http = new HttpClient(handler) { BaseAddress = new Uri("http://localhost:11434/") }; - var result = await new OllamaHttpClient(http).EmbedQueryAsync("cleaner"); + using var sdk = new OllamaApiClient(http); + var result = await new OllamaHttpClient(sdk).EmbedQueryAsync("cleaner"); result.IsFailure.Should().BeTrue(); result.Error.Should().BeOfType().Which.InnerException.Should().BeOfType(); } @@ -129,11 +133,54 @@ public async Task Embed_CancellationIsPropagated() using var cancellation = new CancellationTokenSource(); cancellation.Cancel(); using var http = new HttpClient { BaseAddress = new Uri("http://localhost:11434/") }; - var client = new OllamaHttpClient(http); + using var sdk = new OllamaApiClient(http); + var client = new OllamaHttpClient(sdk); Func action = () => client.EmbedQueryAsync("cleaner", cancellation.Token); await action.Should().ThrowAsync(); } + [Test] + public async Task Registration_UsesAspireClientWithServiceDiscoveryAndResilience() + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration["Ollama:BaseAddress"] = "http://ollama"; + builder.Configuration["Ollama:Resilience:Retry:Delay"] = "00:00:00.001"; + builder.Configuration["Ollama:Resilience:Retry:UseJitter"] = "false"; + builder.Configuration["Ollama:Resilience:Retry:MaxRetryAttempts"] = "1"; + builder.Configuration["Services:ollama:http:0"] = "http://localhost:11434"; + builder.AddProductEmbeddings(); + var requests = 0; + builder.Services.AddHttpClient(OllamaHostingExtensions.HttpClientName).ConfigurePrimaryHttpMessageHandler(() => new InspectHandler((request, _) => + { + request.RequestUri.Should().Be(new Uri("http://localhost:11434/api/embed")); + requests++; + return Task.FromResult(new HttpResponseMessage(requests == 1 ? HttpStatusCode.ServiceUnavailable : HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(new { model = OllamaHttpClient.ModelName, embeddings = new[] { Enumerable.Repeat(0.1f, 768).ToArray() } })) + }); + })); + + using var host = builder.Build(); + host.Services.GetRequiredService().Should().BeOfType(); + using var http = host.Services.GetRequiredService().CreateClient(OllamaHostingExtensions.HttpClientName); + http.Timeout.Should().Be(Timeout.InfiniteTimeSpan); + var result = await host.Services.GetRequiredService().EmbedQueryAsync("cleaner"); + result.IsSuccess.Should().BeTrue(); + requests.Should().Be(2); + } + + [TestCase("null")] + [TestCase("{")] + public async Task Embed_InvalidJson_ReturnsDependencyFailure(string body) + { + using var handler = new InspectHandler((_, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) })); + using var http = new HttpClient(handler) { BaseAddress = new Uri("http://localhost:11434/") }; + using var sdk = new OllamaApiClient(http); + var result = await new OllamaHttpClient(sdk).EmbedQueryAsync("cleaner"); + result.IsFailure.Should().BeTrue(); + result.Error.Should().BeOfType(); + } + private sealed class InspectHandler(Func> inspect) : HttpMessageHandler { protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => inspect(request, cancellationToken); diff --git a/samples/tests/Contoso.Products.Test.Unit/EntryPoint.cs b/samples/tests/Contoso.Products.Test.Unit/EntryPoint.cs index f94052716..f57b6f7e0 100644 --- a/samples/tests/Contoso.Products.Test.Unit/EntryPoint.cs +++ b/samples/tests/Contoso.Products.Test.Unit/EntryPoint.cs @@ -9,7 +9,8 @@ public static void ConfigureApplication(IHostApplicationBuilder builder) builder.Services.AddMemoryCache(); builder.Services.AddReferenceDataOrchestrator(); - builder.AddTypedHttpClient("Ollama", client => client.BaseAddress = new Uri("http://localhost:11434/")); + builder.Configuration["Ollama:BaseAddress"] = "http://localhost:11434/"; + builder.AddProductEmbeddings(); // Reuse the "real" database configured reference data. var jdr = JsonDataReader.ParseYaml("ref-data.seed.yaml", JsonDataReaderOptions.CreateForReferenceData(JsonPropertyNamingConvention.SnakeCase)); diff --git a/samples/tests/Contoso.Products.Test.Unit/GlobalUsing.cs b/samples/tests/Contoso.Products.Test.Unit/GlobalUsing.cs index 2985ef222..4f2720cda 100644 --- a/samples/tests/Contoso.Products.Test.Unit/GlobalUsing.cs +++ b/samples/tests/Contoso.Products.Test.Unit/GlobalUsing.cs @@ -15,6 +15,7 @@ global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Hosting; global using Moq; +global using OllamaSharp; global using System.Net; global using System.Text.Json; global using UnitTestEx; diff --git a/src/CoreEx.Database.Postgres/Extended/PostgresUnitOfWorkInvoker.cs b/src/CoreEx.Database.Postgres/Extended/PostgresUnitOfWorkInvoker.cs index ce57a8bad..dc4824263 100644 --- a/src/CoreEx.Database.Postgres/Extended/PostgresUnitOfWorkInvoker.cs +++ b/src/CoreEx.Database.Postgres/Extended/PostgresUnitOfWorkInvoker.cs @@ -20,8 +20,8 @@ public class PostgresUnitOfWorkInvoker : InvokerBase protected async override Task OnInvokeAsync(InvokerTracer tracer, PostgresUnitOfWork unitOfWork, PostgresDatabaseArgs args, Func> func, CancellationToken cancellationToken) - => await DatabaseInvoker.OrchestrateUnitOfWorkTransactionAsync(tracer, unitOfWork, + => await DatabaseInvoker.OrchestrateUnitOfWorkTransactionAsync(tracer, unitOfWork, args, () => base.OnInvokeAsync(tracer, unitOfWork, args, func, cancellationToken), outboxEnqueued => PostgresMetrics.OutboxEnqueued.Add(outboxEnqueued), cancellationToken).ConfigureAwait(false); -} \ No newline at end of file +} diff --git a/src/CoreEx.Database.SqlServer/Extended/SqlServerUnitOfWorkInvoker.cs b/src/CoreEx.Database.SqlServer/Extended/SqlServerUnitOfWorkInvoker.cs index d6bd7ef4f..823ae6236 100644 --- a/src/CoreEx.Database.SqlServer/Extended/SqlServerUnitOfWorkInvoker.cs +++ b/src/CoreEx.Database.SqlServer/Extended/SqlServerUnitOfWorkInvoker.cs @@ -20,8 +20,8 @@ public class SqlServerUnitOfWorkInvoker : InvokerBase protected async override Task OnInvokeAsync(InvokerTracer tracer, SqlServerUnitOfWork unitOfWork, SqlServerDatabaseArgs args, Func> func, CancellationToken cancellationToken) - => await DatabaseInvoker.OrchestrateUnitOfWorkTransactionAsync(tracer, unitOfWork, + => await DatabaseInvoker.OrchestrateUnitOfWorkTransactionAsync(tracer, unitOfWork, args, () => base.OnInvokeAsync(tracer, unitOfWork, args, func, cancellationToken), outboxEnqueued => SqlServerMetrics.OutboxEnqueued.Add(outboxEnqueued), cancellationToken).ConfigureAwait(false); -} \ No newline at end of file +} diff --git a/src/CoreEx.Database/Abstractions/DatabaseArgs.cs b/src/CoreEx.Database/Abstractions/DatabaseArgs.cs index 98f7c51b6..bff3398b6 100644 --- a/src/CoreEx.Database/Abstractions/DatabaseArgs.cs +++ b/src/CoreEx.Database/Abstractions/DatabaseArgs.cs @@ -11,6 +11,7 @@ public abstract record class DatabaseArgs : DatabaseArgsBase /// Gets or sets the . /// /// Defaults to . - /// This is used to specify the transaction isolation level for the likes of the . + /// Specifies the isolation level when an creates a root transaction. Nested invocations use save-points and inherit the existing transaction's isolation level; + /// this setting does not change or validate the parent's isolation level. public IsolationLevel IsolationLevel { get; init; } = IsolationLevel.ReadCommitted; } diff --git a/src/CoreEx.Database/Abstractions/DatabaseInvoker.cs b/src/CoreEx.Database/Abstractions/DatabaseInvoker.cs index 967e135b6..ad7772408 100644 --- a/src/CoreEx.Database/Abstractions/DatabaseInvoker.cs +++ b/src/CoreEx.Database/Abstractions/DatabaseInvoker.cs @@ -160,8 +160,25 @@ private static class DefaultRetryPipeline /// The . /// The result of the . /// This is intended to be used by the invoker to provide database-agnostic transaction handling (where applicable). - public static async Task OrchestrateUnitOfWorkTransactionAsync(InvokerTracer tracer, IDatabaseUnitOfWork unitOfWork, Func> work, Action? emitOutboxMetrics, CancellationToken cancellationToken) + public static Task OrchestrateUnitOfWorkTransactionAsync(InvokerTracer tracer, IDatabaseUnitOfWork unitOfWork, Func> work, Action? emitOutboxMetrics, CancellationToken cancellationToken) + => OrchestrateUnitOfWorkTransactionAsync(tracer, unitOfWork, unitOfWork.Database.DbArgs, work, emitOutboxMetrics, cancellationToken); + + /// + /// Provides standardized database transaction handling using the specified arguments for a new root transaction. + /// + /// The result . + /// The . + /// The . + /// The arguments specifying the root transaction isolation level. + /// The work to be performed within the unit-of-work. + /// The action to emit outbox metrics (where applicable). + /// The . + /// The result of the . + /// Nested invocations use save-points and inherit the parent transaction's isolation level; they cannot change it. + public static async Task OrchestrateUnitOfWorkTransactionAsync(InvokerTracer tracer, IDatabaseUnitOfWork unitOfWork, DatabaseArgs args, + Func> work, Action? emitOutboxMetrics, CancellationToken cancellationToken) { + args.ThrowIfNull(); var txn = unitOfWork.Database.CurrentTransaction; var isRootTxn = txn is null; var savePoint = isRootTxn ? string.Empty : unitOfWork.Database.GetNextSavePointName(); @@ -242,7 +259,7 @@ async Task RollbackAsync(Exception exception) tracer.Logger.LogDebug("Unit-of-work transaction; creating (root) transaction. [DatabaseId: {DatabaseId}]", unitOfWork.Database.DatabaseId); var conn = await unitOfWork.Database.GetConnectionAsync(cancellationToken).ConfigureAwait(false); - txn = await conn.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + txn = await conn.BeginTransactionAsync(args.IsolationLevel, cancellationToken).ConfigureAwait(false); unitOfWork.Database.UseTransaction(txn); } else @@ -316,4 +333,4 @@ async Task RollbackAsync(Exception exception) } } } -} \ No newline at end of file +} diff --git a/src/CoreEx.Database/README.md b/src/CoreEx.Database/README.md index f43920439..2cb55ec42 100644 --- a/src/CoreEx.Database/README.md +++ b/src/CoreEx.Database/README.md @@ -47,6 +47,23 @@ The outbox sub-namespace implements the [Transactional Outbox pattern](https://m | **[`DatabaseOutboxRelayHostedServiceBase`](./Outbox/DatabaseOutboxRelayHostedServiceBase.cs)** | `TimerHostedServiceBase` subclass that drives `IDatabaseOutboxRelay.RelayAsync` on a timer. | | **[`DatabaseOutboxPublisherBase`](./Outbox/DatabaseOutboxPublisherBase.cs)** | Abstract base `IEventPublisher` that writes `EventData` entries to the outbox table within the current database transaction. | +## Transaction isolation + +The SQL Server and PostgreSQL `IUnitOfWork.TransactionAsync` overloads accepting provider arguments honor +`SqlServerDatabaseArgs.IsolationLevel` or `PostgresDatabaseArgs.IsolationLevel` when creating a root transaction. +Overloads without explicit arguments use the database's configured arguments; the default is `ReadCommitted`. +The framework owns commit, rollback and disposal, and `EfDb` enlists its shared connection through +`IDatabase.UseTransactionChanged`. + +Nested calls use savepoints and **inherit the parent's isolation level**, regardless of their arguments. +They do not upgrade isolation or independently commit the parent. An operation needing a particular isolation +guarantee must verify an existing parent's level. Isolation semantics are provider-specific: PostgreSQL +`RepeatableRead` provides a stable MVCC snapshot, whereas SQL Server `RepeatableRead` does not prevent phantoms +and is not a substitute for SQL Server `Snapshot`. + +Read-only callbacks need not enqueue events. At the root boundary, a configured outbox publishes any queued +events before commit; nested calls leave publication to their parent. + ## Namespaces | Namespace | Description | @@ -68,4 +85,4 @@ The outbox sub-namespace implements the [Transactional Outbox pattern](https://m ## AI Usage Guide -An [`AGENTS.md`](./AGENTS.md) file is included with this package. AI coding assistants (GitHub Copilot, Claude, Cursor, etc.) that support workspace-injected package documentation will automatically surface concise usage guidance, code examples, and `Do Not` rules for this package without requiring a local CoreEx checkout. \ No newline at end of file +An [`AGENTS.md`](./AGENTS.md) file is included with this package. AI coding assistants (GitHub Copilot, Claude, Cursor, etc.) that support workspace-injected package documentation will automatically surface concise usage guidance, code examples, and `Do Not` rules for this package without requiring a local CoreEx checkout. diff --git a/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs b/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs index 9b9ccd8b1..1564022ed 100644 --- a/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs +++ b/src/CoreEx/CoreExExtensions.ApplicationBuilder.cs @@ -67,8 +67,21 @@ public static IHostApplicationBuilder AddHostSettings(this IHostApplicationBuild /// /// public static IHttpClientBuilder AddNamedHttpClient(this IHostApplicationBuilder builder, string name, Action? configureClient = null, Action? configureIdempotency = null) + => builder.AddNamedHttpClient(name, configureClient, name, configureIdempotency); + + /// + /// Adds an opinionated named using a separate configuration section, with idempotency key handling, service discovery, and standard resilience. + /// + /// The . + /// The name registered with . + /// An optional action to configure the . + /// The section containing BaseAddress and optional Resilience settings. + /// An optional action to configure the . + /// The for fluent-style method-chaining. + /// Use this overload when an SDK registers its own HTTP client name. Existing registrations for that name are augmented, not replaced. + public static IHttpClientBuilder AddNamedHttpClient(this IHostApplicationBuilder builder, string name, Action? configureClient, string configurationSectionName, Action? configureIdempotency = null) { - var config = builder.ThrowIfNull().Configuration.GetSection(name) ?? throw new ArgumentException($"Unable to find configuration section for '{name}'."); + var config = builder.ThrowIfNull().Configuration.GetSection(configurationSectionName.ThrowIfNullOrEmpty()); // Registers the underlying service endpoint resolver providers (configuration-based + pass-through); safe/idempotent to call more than once per host. builder.Services.AddServiceDiscovery(); @@ -82,7 +95,7 @@ public static IHttpClientBuilder AddNamedHttpClient(this IHostApplicationBuilder configureClient?.Invoke(client); if (client.BaseAddress is null) - throw new ArgumentException($"Unable to find '{nameof(HttpClient.BaseAddress)}' configuration for '{name}'."); + throw new ArgumentException($"Unable to find '{nameof(HttpClient.BaseAddress)}' configuration for '{configurationSectionName}'."); }); cb.AddIdempotencyKeyHandler((sp, handler) => configureIdempotency?.Invoke(sp, handler)); diff --git a/src/CoreEx/README.md b/src/CoreEx/README.md index e49dd71ed..7fa59093e 100644 --- a/src/CoreEx/README.md +++ b/src/CoreEx/README.md @@ -33,7 +33,7 @@ All other CoreEx packages (`CoreEx.Events`, `CoreEx.Database`, `CoreEx.AspNetCor - 🃏 **Wildcard parsing**: `Wildcard` provides standardized `*` and `?` wildcard parsing, validation, and `Regex` compilation for consistent wildcard-based filtering. - 🏷️ **Reference data interfaces**: Core `IReferenceData`, `IReferenceDataCollection`, and `ReferenceDataOrchestrator` types establish the reference data contract used by `CoreEx.RefData` and data-layer packages. - 📐 **Schema versioning**: `SchemaAttribute` and `Schema` utilities for associating a semantic version with entity types, supporting schema-aware messaging and validation. -- 🌐 **HTTP client and request builder**: `client.Send(...)` and `client.SendValue(...)` return an `HttpRequestBuilder` providing fluent request setup (`WithQuery`, `WithPaging`, headers) and terminal execution with automatic response disposal (`GetValueAsync`, `ToResultAsync`, `GetStringAsync`, etc.). `builder.AddNamedHttpClient` / `AddTypedHttpClient` configure service discovery, resilience, and idempotency key handling. +- 🌐 **HTTP client and request builder**: `client.Send(...)` and `client.SendValue(...)` return an `HttpRequestBuilder` providing fluent request setup (`WithQuery`, `WithPaging`, headers) and terminal execution with automatic response disposal (`GetValueAsync`, `ToResultAsync`, `GetStringAsync`, etc.). `builder.AddNamedHttpClient` / `AddTypedHttpClient` configure service discovery, resilience, and idempotency key handling. For SDK-owned client names, `builder.AddNamedHttpClient("Sdk_httpClient", configureClient: null, configurationSectionName: "Service")` augments the SDK registration using `Service:BaseAddress` and `Service:Resilience`; existing overloads continue using the client name as the configuration section. ## Key types diff --git a/src/CoreEx/RefData/README.md b/src/CoreEx/RefData/README.md index c9afb7c8a..8b2c65c3f 100644 --- a/src/CoreEx/RefData/README.md +++ b/src/CoreEx/RefData/README.md @@ -21,6 +21,22 @@ The `IReferenceDataContext` scoping mechanism allows per-request filtering: e.g. - 🏷️ **Code validation**: `IReferenceDataCodeCollection` provides `ContainsCode(string)` for fast O(1) code existence checks used by validation rules. - 📡 **OpenTelemetry tracing**: `ReferenceDataOrchestratorInvoker` wraps every collection load with an `InvokerTracer` span, tagging the cache type, cache state, and item count. +## Prefetching reference data + +Use `PrefetchAsync(IEnumerable, cancellationToken)` to warm required collections before using synchronous +reference-data navigation properties: + +```csharp +await orchestrator.PrefetchAsync([typeof(Brand), typeof(Category), typeof(SubCategory)], cancellationToken).ConfigureAwait(false); +``` + +The type-based overload removes duplicates and validates every registration before loading any collection. +An unregistered type throws rather than silently skipping a required dependency. Loading is bounded by +`PrefetchMaxDegreeOfParallelism` and uses the existing cache-backed provider scopes. +The existing `PrefetchAsync(IEnumerable, cancellationToken)` retains its behavior of ignoring unknown +names and deduplicating names case-insensitively. For an empty input, use `Array.Empty()` or +`Array.Empty()` to make the intended overload explicit. + ## Key types | Type | Description | @@ -42,4 +58,4 @@ The `IReferenceDataContext` scoping mechanism allows per-request filtering: e.g. - **[`CoreEx.Caching`](../Caching/README.md)** - `IHybridCache` backs `IReferenceDataCache` for L1/L2 reference data caching. - **[`CoreEx.Json`](../Json/README.md)** - `JsonReferenceDataConverter` uses `ReferenceDataOrchestrator.Current` to deserialize code strings to typed reference data instances. - **[`CoreEx.Invokers`](../Invokers/README.md)** - `ReferenceDataOrchestratorInvoker` emits OpenTelemetry spans for every collection load. -- **[`CoreEx.RefData`](../../CoreEx.RefData/README.md)** - The separate `CoreEx.RefData` NuGet package extends these primitives with `ReferenceDataBase` and additional entity/collection base types. \ No newline at end of file +- **[`CoreEx.RefData`](../../CoreEx.RefData/README.md)** - The separate `CoreEx.RefData` NuGet package extends these primitives with `ReferenceDataBase` and additional entity/collection base types. diff --git a/src/CoreEx/RefData/ReferenceDataOrchestrator.cs b/src/CoreEx/RefData/ReferenceDataOrchestrator.cs index 267615f21..85453f529 100644 --- a/src/CoreEx/RefData/ReferenceDataOrchestrator.cs +++ b/src/CoreEx/RefData/ReferenceDataOrchestrator.cs @@ -67,7 +67,7 @@ public static bool TryGetCurrent([NotNullWhen(true)] out ReferenceDataOrchestrat public ILogger Logger { get; } = logger.ThrowIfNull(); /// - /// Gets or sets the to use when performing a . + /// Gets or sets the to use when performing a reference-data prefetch. /// /// Defaults to 2 to minimize potential impact public int PrefetchMaxDegreeOfParallelism { get; set; } = 2; @@ -522,6 +522,8 @@ public Task QueryAsync(Type refType, QueryArgs? queryArgs, PagingA /// As the reference data is a great candidate for caching this will force a prefetch of the cache items. public async Task> PrefetchAsync(IEnumerable names, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + // Get the distinct list of known names to prefetch. var list = names.Where(ContainsName).Distinct(StringComparer.OrdinalIgnoreCase); @@ -532,6 +534,33 @@ public async Task> PrefetchAsync(IEnumerable names, return list; } + /// + /// Prefetches all of the specified, registered types. + /// + /// The list of reference-data types. + /// The . + /// The actual distinct list of types used. + /// Validates all types before loading any collections. Unlike name-based prefetch, unregistered types throw . + /// Loading uses and the existing cache-backed, separately scoped provider execution. + public async Task> PrefetchAsync(IEnumerable types, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + // Get the distinct list of known types to prefetch, validating that they are all registered. + var list = types.ThrowIfNull().Select(type => type.ThrowIfNull()).Distinct().ToArray(); + foreach (var type in list) + { + if (!ContainsType(type)) + throw new InvalidOperationException($"Reference data collection for type '{type.FullName}' does not exist."); + } + + // Go get 'em all! + await Parallel.ForEachAsync(list, new ParallelOptions { MaxDegreeOfParallelism = PrefetchMaxDegreeOfParallelism, CancellationToken = cancellationToken }, + async (type, ct) => await GetByTypeRequiredAsync(type, ct).ConfigureAwait(false)).ConfigureAwait(false); + + return list; + } + /// /// Gets the reference data items for the specified . /// diff --git a/tests/CoreEx.Database.Postgres.Test.Unit/EntityFrameworkUowTests.cs b/tests/CoreEx.Database.Postgres.Test.Unit/EntityFrameworkUowTests.cs index a1a4e86b0..ad371abd1 100644 --- a/tests/CoreEx.Database.Postgres.Test.Unit/EntityFrameworkUowTests.cs +++ b/tests/CoreEx.Database.Postgres.Test.Unit/EntityFrameworkUowTests.cs @@ -6,6 +6,60 @@ namespace CoreEx.Database.Postgres.Test.Unit; public class EntityFrameworkUowTests : DatabaseTestBase { + [TestCase(IsolationLevel.ReadCommitted, "read committed")] + [TestCase(IsolationLevel.RepeatableRead, "repeatable read")] + [TestCase(IsolationLevel.Serializable, "serializable")] + public void Root_Uses_Explicit_Isolation(IsolationLevel isolation, string serverIsolation) => Test.ScopedType(test => test.Run(async _ => + { + var ef = ExecutionContext.GetRequiredService(); + var uow = ExecutionContext.GetRequiredService(); + var result = await uow.TransactionAsync(new PostgresDatabaseArgs { IsolationLevel = isolation }, async token => + { + uow.Database.CurrentTransaction!.IsolationLevel.Should().Be(isolation); + ef.DbContext.Database.CurrentTransaction!.GetDbTransaction().Should().BeSameAs(uow.Database.CurrentTransaction); + return await uow.Database.Statement("SHOW transaction_isolation").ScalarAsync(token).ConfigureAwait(false); + }).ConfigureAwait(false); + + result.Should().Be(serverIsolation); + uow.Database.CurrentTransaction.Should().BeNull(); + ef.DbContext.Database.CurrentTransaction.Should().BeNull(); + }).AssertSuccess()); + + [Test] + public void Root_Uses_Configured_Isolation() => Test.ScopedType(test => test.Run(async _ => + { + var uow = ExecutionContext.GetRequiredService(); + uow.Database.DbArgs = uow.Database.DbArgs with { IsolationLevel = IsolationLevel.Serializable }; + await uow.TransactionAsync(async token => + { + (await uow.Database.Statement("SHOW transaction_isolation").ScalarAsync(token).ConfigureAwait(false)).Should().Be("serializable"); + }).ConfigureAwait(false); + uow.Database.CurrentTransaction.Should().BeNull(); + }).AssertSuccess()); + + [TestCase(IsolationLevel.ReadCommitted, IsolationLevel.RepeatableRead)] + [TestCase(IsolationLevel.Serializable, IsolationLevel.ReadCommitted)] + public void Nested_Inherits_Parent_Isolation(IsolationLevel parentIsolation, IsolationLevel childIsolation) => Test.ScopedType(test => test.Run(async _ => + { + var ef = ExecutionContext.GetRequiredService(); + var uow = ExecutionContext.GetRequiredService(); + await uow.TransactionAsync(new PostgresDatabaseArgs { IsolationLevel = parentIsolation }, async token => + { + var parent = uow.Database.CurrentTransaction; + await uow.TransactionAsync(new PostgresDatabaseArgs { IsolationLevel = childIsolation }, _ => + { + uow.Database.CurrentTransaction.Should().BeSameAs(parent); + parent!.IsolationLevel.Should().Be(parentIsolation); + ef.DbContext.Database.CurrentTransaction!.GetDbTransaction().Should().BeSameAs(parent); + return Task.CompletedTask; + }, token).ConfigureAwait(false); + uow.Database.CurrentTransaction.Should().BeSameAs(parent); + (await uow.Database.Statement("SELECT 1").ScalarAsync(token).ConfigureAwait(false)).Should().Be(1); + }).ConfigureAwait(false); + uow.Database.CurrentTransaction.Should().BeNull(); + ef.DbContext.Database.CurrentTransaction.Should().BeNull(); + }).AssertSuccess()); + [Test] public void Rollback_On_Exception() => Test.ScopedType(test => test.Run(async _ => { @@ -96,4 +150,4 @@ await uow.TransactionAsync(async () => var m = await ef.Table.GetAsync(9.ToGuid()).ConfigureAwait(false); m.Should().BeNull(); }).AssertSuccess()); -} \ No newline at end of file +} diff --git a/tests/CoreEx.Database.Postgres.Test.Unit/GlobalUsing.cs b/tests/CoreEx.Database.Postgres.Test.Unit/GlobalUsing.cs new file mode 100644 index 000000000..0b83088c7 --- /dev/null +++ b/tests/CoreEx.Database.Postgres.Test.Unit/GlobalUsing.cs @@ -0,0 +1,2 @@ +global using Microsoft.EntityFrameworkCore.Storage; +global using System.Data; diff --git a/tests/CoreEx.Database.SqlServer.Test.Unit/EntityFrameworkUowTests.cs b/tests/CoreEx.Database.SqlServer.Test.Unit/EntityFrameworkUowTests.cs index 23ccd5b74..78d76166e 100644 --- a/tests/CoreEx.Database.SqlServer.Test.Unit/EntityFrameworkUowTests.cs +++ b/tests/CoreEx.Database.SqlServer.Test.Unit/EntityFrameworkUowTests.cs @@ -6,6 +6,60 @@ namespace CoreEx.Database.SqlServer.Test.Unit; public class EntityFrameworkUowTests : DatabaseTestBase { + [TestCase(IsolationLevel.ReadCommitted, 2)] + [TestCase(IsolationLevel.RepeatableRead, 3)] + [TestCase(IsolationLevel.Serializable, 4)] + public void Root_Uses_Explicit_Isolation(IsolationLevel isolation, int serverIsolation) => Test.ScopedType(test => test.Run(async _ => + { + var ef = ExecutionContext.GetRequiredService(); + var uow = ExecutionContext.GetRequiredService(); + var result = await uow.TransactionAsync(new SqlServerDatabaseArgs { IsolationLevel = isolation }, async token => + { + uow.Database.CurrentTransaction!.IsolationLevel.Should().Be(isolation); + ef.DbContext.Database.CurrentTransaction!.GetDbTransaction().Should().BeSameAs(uow.Database.CurrentTransaction); + return await uow.Database.Statement("SELECT CAST(transaction_isolation_level AS int) FROM sys.dm_exec_sessions WHERE session_id = @@SPID").ScalarAsync(token).ConfigureAwait(false); + }).ConfigureAwait(false); + + result.Should().Be(serverIsolation); + uow.Database.CurrentTransaction.Should().BeNull(); + ef.DbContext.Database.CurrentTransaction.Should().BeNull(); + }).AssertSuccess()); + + [Test] + public void Root_Uses_Configured_Isolation() => Test.ScopedType(test => test.Run(async _ => + { + var uow = ExecutionContext.GetRequiredService(); + uow.Database.DbArgs = uow.Database.DbArgs with { IsolationLevel = IsolationLevel.Serializable }; + await uow.TransactionAsync(async token => + { + (await uow.Database.Statement("SELECT CAST(transaction_isolation_level AS int) FROM sys.dm_exec_sessions WHERE session_id = @@SPID").ScalarAsync(token).ConfigureAwait(false)).Should().Be(4); + }).ConfigureAwait(false); + uow.Database.CurrentTransaction.Should().BeNull(); + }).AssertSuccess()); + + [TestCase(IsolationLevel.ReadCommitted, IsolationLevel.RepeatableRead)] + [TestCase(IsolationLevel.Serializable, IsolationLevel.ReadCommitted)] + public void Nested_Inherits_Parent_Isolation(IsolationLevel parentIsolation, IsolationLevel childIsolation) => Test.ScopedType(test => test.Run(async _ => + { + var ef = ExecutionContext.GetRequiredService(); + var uow = ExecutionContext.GetRequiredService(); + await uow.TransactionAsync(new SqlServerDatabaseArgs { IsolationLevel = parentIsolation }, async token => + { + var parent = uow.Database.CurrentTransaction; + await uow.TransactionAsync(new SqlServerDatabaseArgs { IsolationLevel = childIsolation }, _ => + { + uow.Database.CurrentTransaction.Should().BeSameAs(parent); + parent!.IsolationLevel.Should().Be(parentIsolation); + ef.DbContext.Database.CurrentTransaction!.GetDbTransaction().Should().BeSameAs(parent); + return Task.CompletedTask; + }, token).ConfigureAwait(false); + uow.Database.CurrentTransaction.Should().BeSameAs(parent); + (await uow.Database.Statement("SELECT 1").ScalarAsync(token).ConfigureAwait(false)).Should().Be(1); + }).ConfigureAwait(false); + uow.Database.CurrentTransaction.Should().BeNull(); + ef.DbContext.Database.CurrentTransaction.Should().BeNull(); + }).AssertSuccess()); + [Test] public void Rollback_On_Exception() => Test.ScopedType(test => test.Run(async _ => { @@ -96,4 +150,4 @@ await uow.TransactionAsync(async () => var m = await ef.Table.GetAsync(9.ToGuid()).ConfigureAwait(false); m.Should().BeNull(); }).AssertSuccess()); -} \ No newline at end of file +} diff --git a/tests/CoreEx.Database.SqlServer.Test.Unit/GlobalUsing.cs b/tests/CoreEx.Database.SqlServer.Test.Unit/GlobalUsing.cs new file mode 100644 index 000000000..0b83088c7 --- /dev/null +++ b/tests/CoreEx.Database.SqlServer.Test.Unit/GlobalUsing.cs @@ -0,0 +1,2 @@ +global using Microsoft.EntityFrameworkCore.Storage; +global using System.Data; diff --git a/tests/CoreEx.RefData.Test.Unit/ReferenceDataOrchestratorTests.cs b/tests/CoreEx.RefData.Test.Unit/ReferenceDataOrchestratorTests.cs index 0640ee613..afdd55dca 100644 --- a/tests/CoreEx.RefData.Test.Unit/ReferenceDataOrchestratorTests.cs +++ b/tests/CoreEx.RefData.Test.Unit/ReferenceDataOrchestratorTests.cs @@ -660,6 +660,66 @@ public async Task PrefetchAsync_AllUnknownNames_ReturnsEmpty() names.Should().BeEmpty(); } + [Test] + public async Task PrefetchAsync_Types_ReturnsDistinctTypesAndPopulatesCache() + { + var orch = ReferenceDataOrchestrator.Current; + var types = await orch.PrefetchAsync([typeof(DummyRefData), typeof(DummyRefData2), typeof(DummyRefData)]).ConfigureAwait(false); + + types.Should().Equal(typeof(DummyRefData), typeof(DummyRefData2)); + var cache = ExecutionContext.GetRequiredService(); + (await cache.GetOrCreateAsync(typeof(DummyRefDataCollection), (_, _) => throw new InvalidOperationException("Prefetch did not populate the cache.")).ConfigureAwait(false)).Should().NotBeNull(); + (await cache.GetOrCreateAsync(typeof(DummyRefData2Collection), (_, _) => throw new InvalidOperationException("Prefetch did not populate the cache.")).ConfigureAwait(false)).Should().NotBeNull(); + } + + [TestCase(typeof(InvalidRefData))] + [TestCase(typeof(string))] + public async Task PrefetchAsync_UnregisteredTypeRejectsBeforeLoading(Type invalidType) + { + var orch = ReferenceDataOrchestrator.Current; + Func act = () => orch.PrefetchAsync([typeof(DummyRefData), invalidType]); + await act.Should().ThrowAsync() + .WithMessage($"Reference data collection for type '{invalidType.FullName}' does not exist.").ConfigureAwait(false); + await AssertPrefetchCacheEmptyAsync().ConfigureAwait(false); + } + + [Test] + public async Task PrefetchAsync_EmptyTypesReturnsEmpty() + { + var types = await ReferenceDataOrchestrator.Current.PrefetchAsync(Array.Empty()).ConfigureAwait(false); + types.Should().BeEmpty(); + } + + [Test] + public async Task PrefetchAsync_NullTypesRejected() + { + Func act = () => ReferenceDataOrchestrator.Current.PrefetchAsync((IEnumerable)null!); + await act.Should().ThrowAsync().ConfigureAwait(false); + act = () => ReferenceDataOrchestrator.Current.PrefetchAsync(new Type[] { typeof(DummyRefData), null! }); + await act.Should().ThrowAsync().ConfigureAwait(false); + } + + [Test] + public async Task PrefetchAsync_CanceledTypesDoesNotPopulateCache() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + Func act = () => ReferenceDataOrchestrator.Current.PrefetchAsync([typeof(DummyRefData)], cancellation.Token); + await act.Should().ThrowAsync().ConfigureAwait(false); + await AssertPrefetchCacheEmptyAsync().ConfigureAwait(false); + } + + private static async Task AssertPrefetchCacheEmptyAsync() + { + var created = false; + await ExecutionContext.GetRequiredService().GetOrCreateAsync(typeof(DummyRefDataCollection), (_, _) => + { + created = true; + return Task.FromResult(new DummyRefDataCollection()); + }).ConfigureAwait(false); + created.Should().BeTrue(); + } + // ── ConvertFromMapping ───────────────────────────────────────────────── private class MappingProvider(IReferenceDataCollection collection) : IReferenceDataProvider diff --git a/tests/CoreEx.Test.Unit/Http/HttpClientConfigurationTests.cs b/tests/CoreEx.Test.Unit/Http/HttpClientConfigurationTests.cs index 1a61d9523..1306dd877 100644 --- a/tests/CoreEx.Test.Unit/Http/HttpClientConfigurationTests.cs +++ b/tests/CoreEx.Test.Unit/Http/HttpClientConfigurationTests.cs @@ -2,9 +2,11 @@ namespace CoreEx.Test.Unit.Http; public class HttpClientConfigurationTests { - [TestCase(false)] - [TestCase(true)] - public async Task NamedClient_BindsOnlyResilienceSection(bool configureResilience) + [TestCase(false, false)] + [TestCase(true, false)] + [TestCase(false, true)] + [TestCase(true, true)] + public async Task NamedClient_BindsOnlyResilienceSection(bool configureResilience, bool separateSection) { var builder = Host.CreateApplicationBuilder(); var settings = new Dictionary @@ -19,12 +21,26 @@ public async Task NamedClient_BindsOnlyResilienceSection(bool configureResilienc } builder.Configuration.AddInMemoryCollection(settings); - builder.AddNamedHttpClient("Embedding"); + var name = separateSection ? "Sdk_httpClient" : "Embedding"; + if (separateSection) + { + builder.Configuration[$"{name}:BaseAddress"] = "http://wrong-endpoint/"; + builder.Configuration[$"{name}:Resilience:AttemptTimeout:Timeout"] = "invalid"; + builder.Services.AddHttpClient(name, client => client.DefaultRequestHeaders.Add("X-Sdk", "preserved")); + builder.AddNamedHttpClient(name, client => client.Timeout = Timeout.InfiniteTimeSpan, "Embedding"); + } + else + builder.AddNamedHttpClient(name); using var host = builder.Build(); await host.StartAsync().ConfigureAwait(false); - using var client = host.Services.GetRequiredService().CreateClient("Embedding"); + using var client = host.Services.GetRequiredService().CreateClient(name); client.BaseAddress.Should().Be(new Uri("http://localhost:11434/")); + if (separateSection) + { + client.DefaultRequestHeaders.GetValues("X-Sdk").Should().ContainSingle().Which.Should().Be("preserved"); + client.Timeout.Should().Be(Timeout.InfiniteTimeSpan); + } await host.StopAsync().ConfigureAwait(false); } } diff --git a/tools/test-product-search.ps1 b/tools/test-product-search.ps1 new file mode 100644 index 000000000..3562c3844 --- /dev/null +++ b/tools/test-product-search.ps1 @@ -0,0 +1,46 @@ +[CmdletBinding()] +param( + [uri] $BaseAddress = 'http://localhost:5207/' +) + +$ErrorActionPreference = 'Stop' +$schema = Invoke-RestMethod -Uri ([uri]::new($BaseAddress, 'api/products/$query')) -TimeoutSec 30 +if ($schema.search.mode -ne 'semantic' -or !$schema.search.ranked) { + throw 'This smoke check requires the semantic Products API.' +} + +$queries = @( + @{ Text = 'something to clean mud off my bike'; Expected = 'MUCOFF-NANO-CLEANER' }, + @{ Text = 'a component for changing gears'; Expected = 'SHIMANO-XT-M8100-RD' } +) +$failures = [Collections.Generic.List[string]]::new() +foreach ($query in $queries) { + $path = 'api/products?$search=' + [uri]::EscapeDataString($query.Text) + '&$take=5&$count=true' + $response = Invoke-WebRequest -Uri ([uri]::new($BaseAddress, $path)) -TimeoutSec 120 + $products = @($response.Content | ConvertFrom-Json) + Write-Host "`nQuery: $($query.Text)" + $products | Format-Table sku, text, price + if ($products.Count -eq 0 -or $products[0].sku -ne $query.Expected) { + $failures.Add("Query '$($query.Text)' expected $($query.Expected) first.") + } + if (!$response.Headers.ContainsKey('X-Paging-Total-Count')) { + $failures.Add("Query '$($query.Text)' returned no eligible-product count.") + } +} + +$filtered = Invoke-RestMethod -Uri ([uri]::new($BaseAddress, 'api/products?$search=clean%20mud%20off%20my%20bike&$filter=price%20lt%2020&$take=1')) -TimeoutSec 120 +if (@($filtered).Count -ne 1 -or $filtered[0].sku -ne 'MUCOFF-NANO-CLEANER' -or $filtered[0].price -ge 20) { + $failures.Add('The price-constrained cleaning query did not return the cleaner under 20.') +} + +$body = @{ query = '{ products(search: "a component for changing gears", first: 1) { edges { node { sku } } } }' } | ConvertTo-Json +$graphql = Invoke-RestMethod -Uri ([uri]::new($BaseAddress, 'api/query')) -Method Post -ContentType 'application/json' -Body $body -TimeoutSec 120 +if ($graphql.errors -or $graphql.data.products.edges[0].node.sku -ne 'SHIMANO-XT-M8100-RD') { + $failures.Add('GraphQL-lite did not return the same first gear-changing candidate as REST.') +} + +if ($failures.Count -gt 0) { + throw ($failures -join "`n") +} +Write-Host "`nVerified illustrative full-catalog pgvector rankings, a live price constraint and GraphQL-lite parity." +Write-Host 'Nearest candidates are not guaranteed relevant; this is a learning smoke check, not a broad quality benchmark.' From 036e198a7d644d81cf47c30cc405e5834785f5b9 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Wed, 14 Oct 2026 11:47:37 -0700 Subject: [PATCH 07/25] Add eventual product search indexing and composable EF SQL bridge Maintain search projections through globbed product events, publish explicit backfill readiness, and load SQL from embedded resources. Add a reusable IEfDb SqlQuery bridge with parameter binding and shared transaction coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .../coreex-conventions.instructions.md | 2 + .../aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs | 2 +- .../E2ETest.ProductSearch.cs | 70 +++++++ samples/aspire/Contoso.Test.Aspire/E2ETest.cs | 5 +- samples/docs/local-dev.md | 13 +- samples/docs/search.md | 74 ++++--- .../Embeddings/IProductSearchSyncAdapter.cs | 8 + ...ducts-product_search_readiness-table.pgsql | 7 + .../Embeddings/ProductSearchSyncAdapter.cs | 11 + .../Contoso.Products.Infrastructure.csproj | 3 + .../Repositories/ProductRepository.cs | 31 ++- .../Repositories/ProductSearchIndex.cs | 195 +++++++++++++----- .../Sql/ProductSearchDeleteProduct.pgsql | 1 + .../Sql/ProductSearchDeleteProfile.pgsql | 1 + .../Sql/ProductSearchIsReady.pgsql | 4 + .../Sql/ProductSearchLockSource.pgsql | 1 + .../Sql/ProductSearchMarkReady.pgsql | 3 + .../Sql/ProductSearchMatches.pgsql | 4 + .../Sql/ProductSearchSourceHash.pgsql | 2 + .../Sql/ProductSearchUpsert.pgsql | 7 + .../Contoso.Products.Subscribe/GlobalUsing.cs | 1 + .../Subscribers/ProductSearchSubscriber.cs | 12 ++ .../ReadTests.ProductSemanticSearch.cs | 145 ++++++++++++- .../GlobalUsing.cs | 8 +- .../ProductSearchTests.cs | 153 ++++++++++++++ .../SubscriberTests.cs | 4 +- src/CoreEx.EntityFrameworkCore/AGENTS.md | 4 + .../EfDbExtensions.cs | 27 ++- src/CoreEx.EntityFrameworkCore/README.md | 17 +- .../EntityFrameworkSqlQueryTests.cs | 87 ++++++++ .../GlobalUsing.cs | 4 + 31 files changed, 776 insertions(+), 130 deletions(-) create mode 100644 samples/aspire/Contoso.Test.Aspire/E2ETest.ProductSearch.cs create mode 100644 samples/src/Contoso.Products.Application/Adapters/Embeddings/IProductSearchSyncAdapter.cs create mode 100644 samples/src/Contoso.Products.Database/Migrations/20261014-172545-create-products-product_search_readiness-table.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductSearchSyncAdapter.cs create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchDeleteProduct.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchDeleteProfile.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchIsReady.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchLockSource.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchMarkReady.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchMatches.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchSourceHash.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchUpsert.pgsql create mode 100644 samples/src/Contoso.Products.Subscribe/Subscribers/ProductSearchSubscriber.cs create mode 100644 samples/tests/Contoso.Products.Test.Subscribe/ProductSearchTests.cs create mode 100644 tests/CoreEx.Database.Postgres.Test.Unit/EntityFrameworkSqlQueryTests.cs diff --git a/.github/instructions/coreex-conventions.instructions.md b/.github/instructions/coreex-conventions.instructions.md index bca366dac..a1dbbaa40 100644 --- a/.github/instructions/coreex-conventions.instructions.md +++ b/.github/instructions/coreex-conventions.instructions.md @@ -162,6 +162,8 @@ Guid id = Runtime.NewGuid(); // new Guid **Every `async`/`Task`-returning method takes a `CancellationToken` and passes it on.** Add it as the **last parameter** — `CancellationToken cancellationToken = default` on public/library methods (default so callers aren't forced to supply one). Flow it through to **every** downstream awaitable call (repositories, `HttpClient`, EF Core, `_unitOfWork`, other services) — never call an overload that accepts a token without passing it, and never silently drop it. +**Naming:** use `cancellationToken` for declared method parameters, including interfaces, implementations, private methods and local functions. Reserve `ct` for local delegate/lambda parameters within methods; do not shorten a method parameter to `ct`. + ```csharp public async Task CreateAsync(Employee value, CancellationToken cancellationToken = default) { diff --git a/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs b/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs index 6ba539dfe..9b440cb0f 100644 --- a/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs +++ b/samples/aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs @@ -4,7 +4,7 @@ public partial class E2ETest { private string _mcpEndpoint = string.Empty; - [Test, Order(0)] + [Test] public async Task Contoso_Mcp_FullBasketLifecycle() { Test.Checkpoint("Increase inventory for the seeded, active Product used by the MCP basket scenario."); diff --git a/samples/aspire/Contoso.Test.Aspire/E2ETest.ProductSearch.cs b/samples/aspire/Contoso.Test.Aspire/E2ETest.ProductSearch.cs new file mode 100644 index 000000000..86abd4f1b --- /dev/null +++ b/samples/aspire/Contoso.Test.Aspire/E2ETest.ProductSearch.cs @@ -0,0 +1,70 @@ +namespace Contoso.Test.Aspire; + +public partial class E2ETest +{ + [Test] + public async Task ProductSearch_LifecycleIndexesThroughOutboxAsync() + { + var product = Test.Http(_productsApi) + .Run(HttpMethod.Post, "/api/products", new Product + { + Sku = "SEARCH-E2E-CLEANER", + Text = "Search lifecycle bike cleaner", + Description = "Removes mud from bicycle frames.", + Price = 12m, + SubCategoryCode = "CL", + UnitOfMeasureCode = "EA" + }, r => r.WithIdempotencyKey()) + .AssertCreated().Value!; + + await using var source = Npgsql.NpgsqlDataSource.Create(_productsConnectionString); + await WaitForDocumentAsync(source, product.Id!, product.Description).ConfigureAwait(false); + SearchProduct(product.Sku!, includeInactive: false).Should().BeEmpty(); + SearchProduct(product.Sku!, includeInactive: true).Should().ContainSingle(); + + product.Description = "A citrus degreaser for oily bicycle drivetrains."; + product = Test.Http(_productsApi).Run(HttpMethod.Put, $"/api/products/{product.Id}", product).AssertOK().Value!; + await WaitForDocumentAsync(source, product.Id!, product.Description).ConfigureAwait(false); + + // Delete while inactive: logical deletion must clean the derived table without a physical cascade. + Test.Http(_productsApi).Run(HttpMethod.Delete, $"/api/products/{product.Id}").AssertNoContent(); + SearchProduct(product.Sku!, includeInactive: true).Should().BeEmpty(); + await WaitForDocumentAsync(source, product.Id!, null).ConfigureAwait(false); + Test.Http(_productsApi).Run(HttpMethod.Get, $"/api/products/{product.Id}").AssertNotFound(); + + var active = Test.Http(_productsApi) + .Run(HttpMethod.Post, "/api/products", new Product + { + Sku = "SEARCH-E2E-ACTIVE", + Text = "Search lifecycle trail bicycle", + Description = "A bicycle for forest trails.", + Price = 900m, + SubCategoryCode = "XC", + UnitOfMeasureCode = "EA" + }, r => r.WithIdempotencyKey()) + .AssertCreated().Value!; + Test.Http(_productsApi).Run(HttpMethod.Post, $"/api/products/{active.Id}/activate").AssertOK(); + await WaitForDocumentAsync(source, active.Id!, active.Description).ConfigureAwait(false); + SearchProduct(active.Sku!, includeInactive: false).Should().ContainSingle().Which.IsInactive.Should().BeFalse(); + } + + private ProductLite[] SearchProduct(string sku, bool includeInactive) => Test.Http(_productsApi) + .Run(HttpMethod.Get, $"/api/products?$search=bicycle&$filter=sku eq '{sku}'{(includeInactive ? "&$inactive" : "")}") + .AssertOK().Value!; + + private async Task WaitForDocumentAsync(Npgsql.NpgsqlDataSource source, string id, string? expected, CancellationToken cancellationToken = default) + { + for (var attempt = 0; attempt < 90; attempt++) + { + await using var command = source.CreateCommand("SELECT document FROM products.product_search WHERE product_id = $1 AND profile = $2 AND recipe = $3"); + command.Parameters.AddWithValue(id); + command.Parameters.AddWithValue(OllamaHttpClient.Profile); + command.Parameters.AddWithValue(Contoso.Products.Application.Search.ProductSearchDocument.TemplateVersion); + var document = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) as string; + if (expected is null ? document is null : document?.Contains(expected, StringComparison.Ordinal) == true) + return; + Test.Delay(1000, $"Waiting for product '{id}' search projection after outbox delivery; poll {attempt + 1}/90."); + } + Assert.Fail($"Search projection for product '{id}' did not {(expected is null ? "disappear" : "contain the current description")} after 90 polls."); + } +} diff --git a/samples/aspire/Contoso.Test.Aspire/E2ETest.cs b/samples/aspire/Contoso.Test.Aspire/E2ETest.cs index 9f9c581d6..14fd192be 100644 --- a/samples/aspire/Contoso.Test.Aspire/E2ETest.cs +++ b/samples/aspire/Contoso.Test.Aspire/E2ETest.cs @@ -6,11 +6,14 @@ public partial class E2ETest : WithAspireTester private readonly string _shoppingApi = "shopping-api"; private readonly string _customersApi = "customers-api"; private string _serviceBusConnectionString = string.Empty; + private string _productsConnectionString = string.Empty; protected override async Task OnBeforeStartAsync(DistributedApplication app) { _serviceBusConnectionString = await AspireTesterBase.GetConnectionStringAsync(app, "ServiceBus").ConfigureAwait(false) ?? throw new InvalidOperationException("The ServiceBus resource has no connection string."); + _productsConnectionString = await AspireTesterBase.GetConnectionStringAsync(app, "Postgres").ConfigureAwait(false) + ?? throw new InvalidOperationException("The Postgres resource has no connection string."); // Migrate the Products, Shopping and Customers databases and seed each with test data. await app.MigratePostgresDataAsync("Postgres", ["mutate-data.seed.yaml"], Contoso.Products.Database.Program.ConfigureMigrationArgs); @@ -38,7 +41,7 @@ protected override async Task OnAfterStartAsync(DistributedApplication app) _mcpEndpoint = app.GetEndpoint("contoso-mcp", "http").ToString().TrimEnd('/'); } - [Test, Order(1)] + [Test] public async Task CreateOrderAndConfirmAsync() { Test.Checkpoint("Create and activate a new Product; this should sync to the Shopping domain via the Products domain's outbox and Service Bus."); diff --git a/samples/docs/local-dev.md b/samples/docs/local-dev.md index 9814fa031..93a1e0757 100644 --- a/samples/docs/local-dev.md +++ b/samples/docs/local-dev.md @@ -96,16 +96,9 @@ Aspire exposes the Compose-owned Ollama endpoint as an external resource and use configuration for both fixture preparation and API calls. Start the `search` Compose profile and install the model above before running Aspire E2E tests; CI does this explicitly too. -`ProductSearchIndex.RebuildAsync` remains callable through a configured Infrastructure service scope. There is -currently no standalone indexing console or HTTP maintenance endpoint. Starting Aspire manually does not seed or -rebuild the catalog. After prepared-fixture execution, `.\tools\test-product-search.ps1` can exercise a normally -running API (default `http://localhost:5207/`). Searchable CRUD changes require another explicit rebuild until -incremental indexing is implemented. Missing/stale eligible embeddings produce HTTP 503 rather than an incomplete -successful search. Ordinary product reads and structured filters remain available. - -To prepare and verify a real-model catalog without subsequently running the E2E test that adds an unindexed product, -run the MCP lifecycle on its own. This is test setup: it reseeds all three domain databases and resets broker state, -so use it only against disposable local infrastructure. +`ProductSearchIndex.RebuildAsync` remains callable through a configured Infrastructure service scope. There is currently no standalone indexing console or HTTP maintenance endpoint. Starting Aspire manually does not seed or rebuild the catalog. After prepared-fixture execution, `.\tools\test-product-search.ps1` can exercise a normally running API (default `http://localhost:5207/`). Subsequent product changes flow through the transactional outbox, Products Relay and the Products Subscribe host's single globbed product handler. Keep both hosts running to maintain the projection; logical deletion explicitly removes projection rows rather than relying on a cascade. Search is eventually consistent: older vectors can rank current product fields, and new products appear once indexed. Live price/activity/deletion filters apply immediately. Counts describe eligible indexed candidates. Only uninitialized/incompatible readiness or a failed query-embedding dependency yields CoreEx `TransientException` (HTTP 503), not ordinary indexing lag. Ordinary reads and structured filters remain available. + +To prepare and verify a real-model seed catalog, run the MCP lifecycle on its own. This is test setup: it reseeds all three domain databases and resets broker state, so use it only against disposable local infrastructure. ```powershell dotnet test samples\aspire\Contoso.Test.Aspire --filter "FullyQualifiedName~E2ETest.Contoso_Mcp_FullBasketLifecycle" diff --git a/samples/docs/search.md b/samples/docs/search.md index e9e460f2b..845586239 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -19,10 +19,10 @@ The goal is not merely to store some numbers and label the result "AI". It is to | Local embedding provider | Implemented | Ollama/Nomic document and query instructions, response validation, and an Application adapter boundary are available. | | Separate search projection and pgvector semantic retrieval | Implemented | PostgreSQL stores 768-dimensional embeddings and computes exact, filtered cosine-distance rankings. | | PostgreSQL full-text retrieval and hybrid ranking | Proposed | The current sample does not perform full-text or hybrid search. | -| Explicit rebuild / asynchronous search indexing | Rebuild implemented; asynchronous processing proposed | Test setup invokes reusable Infrastructure logic to rebuild the catalog atomically. Missing/stale eligible documents return HTTP 503 until rebuilt. | +| Explicit backfill / asynchronous search indexing | Implemented | Test setup publishes an atomic initial backfill; one globbed product-event subscriber reconciles subsequent changes, including logical deletion. | | Real-model relevance evaluation and HNSW benchmark | Initial probes implemented; broader evaluation proposed | Two illustrative semantic expectations pass after descriptive enrichment. Full-catalog relevance and index performance remain unproven. | -The implemented baseline is PostgreSQL as the search store, one embedding per product/profile/recipe, a rebuildable search projection, and indexing outside product-write transactions. It uses local Ollama with Nomic v1.5 and 768 dimensions. Exact retrieval ranks every eligible product; count means the number of eligible products, not the number judged relevant. Full-text/hybrid retrieval windows and asynchronous maintenance remain future work. +The implemented baseline is PostgreSQL as the search store, one embedding per product/profile/recipe, a rebuildable search projection, and indexing outside product-write transactions. It uses local Ollama with Nomic v1.5 and 768 dimensions. Exact retrieval ranks eligible **indexed** products; count means the number of those candidates, not the number judged relevant or the complete catalog size. Full-text/hybrid retrieval windows remain future work. ## Reading map @@ -346,9 +346,9 @@ This is the implemented semantic flow. Requests without `$search` do not call th The existing `ProductRepository.QueryAsync` joins product, subcategory, and inventory data. Ranked retrieval must preserve that behavior and all configured filters rather than quietly implementing a reduced API. -Application orchestration depends on application-owned interfaces. `IProductEmbeddingAdapter` owns the model boundary, its Ollama implementation lives in Infrastructure, and `ProductSearchIndex` handles derived storage and rebuilds there. +Application orchestration depends on application-owned interfaces. `IProductEmbeddingAdapter` owns the model boundary; `IProductSearchSyncAdapter` exposes current-state reconciliation to the subscriber. Their implementations live in Infrastructure, where `ProductSearchIndex` handles derived storage and backfills. -The sample deliberately uses bound vector-text parameters with an explicit PostgreSQL `vector(768)` cast, rather than adding pgvector EF/Npgsql packages. EF Core's unmapped `SqlQuery` exposes only product IDs, fingerprints and distances; the repository composes it with its existing relational joins and filters. Vectors never enter CRUD contracts. This SQL-only derived table is intentionally outside `dbex.yaml`'s generated CRUD models; its DDL is still owned by a DbEx migration. +The sample deliberately uses bound vector-text parameters with an explicit PostgreSQL `vector(768)` cast, rather than adding pgvector EF/Npgsql packages. SQL lives in named, embedded `.pgsql` files under Infrastructure's `Repositories/Sql/`, loaded through `SqlStatement.FromResource`. `Matches()` passes its resource to CoreEx's `IEfDb.SqlQuery` bridge with separate parameter values; EF binds the `{0}`/`{1}`/`{2}` placeholders rather than formatting values into SQL. The query omits a trailing semicolon so EF can compose it with live relational joins, filters, ranking, paging and count. Maintenance statements execute with bound parameters via `ProductsEfDb.Database` (CoreEx `IDatabase`). These commands share the existing unit-of-work transaction with EF; no separate connection or transaction is introduced. Vectors never enter CRUD contracts. These SQL-only derived tables are intentionally outside `dbex.yaml`'s generated CRUD models; their DDL is still owned by DbEx migrations. ### What MCP adds @@ -362,7 +362,7 @@ Search does not resolve authentication or authorization. The sample's existing a ## Keeping search current -### Current milestone: an explicit, atomic rebuild +### Initial readiness: an explicit, atomic backfill Rebuilding is reusable Infrastructure logic, **not an API startup responsibility**. The earlier API `--rebuild-search` maintenance mode has been removed. There is no HTTP rebuild endpoint or standalone indexing @@ -391,9 +391,7 @@ CoreEx results; cancellation still propagates. No provider types leak through `I The SDK's direct embedding API is used instead of `IEmbeddingGenerator` to retain the provider-specific truncation policy and response model metadata; portability remains at the application-owned adapter boundary. -The Products Subscribe host also registers the Ollama client because its shared repositories and dynamically -registered services include the embedding adapter. Ordinary reservation handlers do not call the model; this -registration satisfies their service graph, not an indexing workflow. +The Products Subscribe host registers the Ollama client for asynchronous product indexing. Ordinary reservation handlers do not call the model and consume a separate command queue. The rebuild indexes the current non-deleted catalog, including inactive products so `$inactive` works. It does not seed products or download the model. It prepares documents and calls Ollama **before** starting a database @@ -405,28 +403,36 @@ physical deletion also cascades through the foreign key. Other profiles remain s rebuild inside an existing unit of work, it inherits that transaction and publication remains subject to the parent's eventual commit or rollback. -For a semantic request, the repository checks eligible source hashes first, so a known broken index fails before paying for inference. It embeds the query, uses `PostgresUnitOfWork.TransactionAsync` with `PostgresDatabaseArgs.IsolationLevel = RepeatableRead` to open a database snapshot, and checks hashes again to cover writes during inference. It then joins the compatible projection, orders by exact cosine distance and product ID (a deterministic tie-breaker), and applies paging/count. Any missing or stale eligible document yields **HTTP 503** with a rebuild instruction. There is no silent keyword fallback, incomplete result set or successful empty response masquerading as inference failure. +`products.product_search_readiness` records successful initial preparation for each profile/recipe. The readiness marker commits atomically with backfill publication, even for an empty catalog. Incremental events never mark a partially indexed catalog ready. A failed initial backfill leaves it unready; a failed later backfill preserves the previous published projection and readiness. + +For a semantic request, the repository checks this small readiness record before inference and again inside its `RepeatableRead` snapshot. It joins compatible vectors to **live** product data, orders by exact cosine distance and product ID (a deterministic tie-breaker), and applies paging/count. There are no full-catalog freshness scans. + +| Condition | Search behavior | +|---|---| +| Updated product, older compatible vector | HTTP 200; older meaning can affect ranking, but returned fields and filters use current relational state. | +| New product without a vector | HTTP 200; omitted until indexing succeeds. | +| Deleted or inactive product with a retained vector | Immediately excluded by live eligibility (inactive can be explicitly requested). | +| Ready index with no eligible indexed candidates | HTTP 200 with an empty result and count zero. | +| Initial backfill never completed for the configured profile/recipe | CoreEx `TransientException`, HTTP 503. | +| Query embedding dependency unavailable, timed out, or returned an invalid response | CoreEx `TransientException`, HTTP 503, preserving inner diagnostics. | + +Caller cancellation propagates; a broken internal vector contract remains a programming error, not a successful empty result. There is no silent keyword fallback. The framework manages commit, rollback and cleanup, and enlists the EF context in the shared database transaction. A nested unit of work uses a savepoint, not a new independent transaction: it inherits the parent's isolation. Semantic search therefore explicitly rejects an existing parent unless it is `RepeatableRead` or `Serializable`; silently accepting `ReadCommitted` would lose the consistency guarantee. For standalone requests, inference occurs before opening the snapshot. When a caller already owns a compatible parent transaction, that caller owns its lifetime, including time spent on inference. The search callback does not enqueue events, but any events already queued by a consumer follow the normal root-unit-of-work outbox rules. -This guard deliberately recomputes hashes over eligible products and reads stored fingerprints, twice for successful searches. That is suitable for a 32-row teaching catalog, **not a high-volume indexing strategy**. Price and inventory come from live relational data, are not in the document hash, and do not require new embeddings. Name, description, tags and reference-data labels do. Labels use the existing orchestrator/cache; rebuilding and querying must use consistent language/cache state. This is not a substitute for reference-data invalidation and asynchronous indexing. +Price, inventory and activity changes do not require new embeddings. Name, description, tags and reference-data labels do. Labels use the existing orchestrator/cache; reference-data-driven invalidation and reindexing are **not implemented**. A label-only change needs an explicit backfill with appropriate cache invalidation. -API integration tests rebuild with controlled vectors, isolating distance ordering and failure behavior from model -quality. Test.Aspire's MCP lifecycle asserts that real semantic cleaning intent returns Nano cleaner through -MCP -> Products API -> PostgreSQL. It runs before the checkout scenario that creates a new product: until -incremental indexing exists, that catalog mutation intentionally makes the prepared projection stale. This ordering -is explicit fixture management, not evidence of automatic indexing. Future mutation-to-search tests must exercise -the asynchronous indexing path without inserting a rebuild between mutation and search. +API integration tests rebuild with controlled vectors, isolating distance ordering and failure behavior from model quality. Subscriber integration tests cover duplicate/out-of-order notifications, logical deletion, content-hash skips, concurrent edits and provider retries. Test.Aspire's MCP lifecycle asserts real semantic cleaning intent; its product-search lifecycle drives create/update/activate/delete through HTTP and polls the stored document and search results through the real relay, broker, subscriber and model, with no intervening rebuild. The MCP and checkout scenarios no longer require ordering to avoid freshness failures. -There is no relevance cutoff: every eligible vector is a candidate. `$count=true` reports the eligible catalog size, including poor semantic matches. An unrelated query still returns neighbors; clients must not interpret a returned product as guaranteed suitable. +There is no relevance cutoff: every eligible compatible vector is a candidate. `$count=true` reports eligible indexed candidates, including poor semantic matches and excluding not-yet-indexed products. An unrelated query still returns neighbors; clients must not interpret a returned product as guaranteed suitable. -### Next milestone: asynchronous maintenance +### Asynchronous maintenance Product CRUD already commits product mutations and outgoing events together using the transactional outbox. An **outbox** stores an event in the same database transaction as its business change. A **relay** forwards committed events to the broker. A **subscriber** handles delivered events. -The proposed indexing flow reuses that foundation: +The indexing flow reuses that foundation: ```mermaid sequenceDiagram @@ -450,28 +456,28 @@ This is **eventual consistency**: the product write can succeed before its searc The API must not wait for external inference inside a product-write transaction. That would tie business writes to model availability and hold database resources during inference. -The existing Products Subscribe host is a natural home for the worker. A dedicated subscription/receiver may isolate slower indexing from reservation commands without adding another host. Broker subscription rules, handler registration, concurrency, retries, and dead-letter behavior all need explicit wiring. +`ProductSearchSubscriber` uses `[Subscribe("contoso.products.product.**")]` to handle both versioned payload events and key-only deletion events. It delegates only the product key through the application-owned sync boundary. The existing `contoso` topic's `products` subscription feeds it; assembly discovery registers the handler. Reservation commands use a separate receiver/queue, although they still share the process and infrastructure. The usual CoreEx transient error handling requests broker retry; operators must monitor failures/dead letters. ### Duplicate delivery is normal At-least-once delivery means an event can arrive multiple times. The handler must be **idempotent**: retrying the same work must not corrupt state. -The proposed handler should: +The implemented handler: -1. Load current authoritative product data instead of blindly using an old event payload. -2. Construct the canonical document and source hash. -3. Skip work if the stored hash/profile already matches. -4. Generate the embedding outside a database transaction. -5. Recheck source freshness before writing. -6. Store only if the result still corresponds to current source state; otherwise schedule/retry fresh work. +1. Loads current authoritative product data instead of blindly using an old event payload. +2. Constructs the canonical document and source hash. +3. Skips inference if the stored hash/profile already matches. +4. Generates any required embedding outside a database transaction. +5. Rechecks source freshness before writing. +6. Stores only if the result still corresponds to current source state; otherwise returns `TransientException` for retry. -The final check/write requires a concurrency strategy, not just two unrelated reads. A short transaction with an appropriate lock/version check can prevent an older worker overwriting a newer projection. +The final check/write uses a short unit of work and PostgreSQL `SHARE` lock on the product table, also used by backfill publication. This blocks product writes during the final check/upsert, **not** during model inference. Inference cannot run inside a caller-owned parent transaction for incremental sync. Concurrent publishers may share the lock, but all must verify the same current source. This coarse table lock is deliberately simple for the teaching catalog; production scale needs measured contention and a finer-grained publication strategy. -If reference-data descriptions contribute to the document, changes to those descriptions must also trigger or schedule reindexing of affected products. +Reference-data changes should eventually schedule affected products too; that wiring remains future work. ### Deactivation and deletion -Live eligibility must exclude deleted or inactive products when appropriate, even while their old vectors remain stored. Deletion should also clean up the projection. +Live eligibility excludes deleted or inactive products when appropriate, even while their old vectors remain stored. On a missing/logically deleted product, the subscriber deletes projection rows for **all** profiles and recipes. An old update arriving afterward reloads current state and cannot resurrect the projection. The foreign key cascade helps physical deletion only; it cannot detect a logical delete. Search eligibility must not rely solely on yesterday's indexed status. @@ -479,7 +485,7 @@ Search eligibility must not rely solely on yesterday's indexed status. Events do not automatically index pre-existing products. A **backfill** processes existing records; **reindexing** rebuilds records after a model/template change or repair. -Provide an explicit resumable operation using the same builder and profile as event processing. It should batch work, skip current records, report failures, and support reconciliation after missed events or dead-lettered work. +The sample's atomic backfill uses the same document/profile as incremental processing, but is not a resumable, batched operational job. Production adoption still needs explicit reconciliation after missed/dead-lettered work, operational scheduling, lag/coverage monitoring and a controlled profile rollout. ### Provider failures @@ -669,7 +675,7 @@ Trace embedding time, database time, indexing lag, retries, and failures. Avoid | 1. Documents and embeddings | See how verified catalog content becomes vectors. | Selected profile, reproducible builder, provider configuration, provenance. | | 2. Exact pgvector retrieval (implemented) | Understand distance ordering with real filters. | Real database tests for ordering, eligibility, profile isolation, paging, freshness and atomic rebuild failure. | | 3. Full-text and RRF | Compare exact terminology with semantic intent. | Inspectable branch ranks and curated hybrid comparisons. | -| 4. Indexing and backfill | Maintain rebuildable derived state. | Duplicate/out-of-order handling, stale-write protection, updates/deletes, recovery. | +| 4. Indexing and backfill (implemented; operational recovery remains) | Maintain rebuildable derived state. | Duplicate/out-of-order handling, stale-write protection, updates/logical deletes and explicit readiness. Resumable reconciliation remains future work. | | 5. HNSW benchmark | Measure speed versus recall at meaningful scale. | Exact baseline, larger corpus, real query plans, latency and recall results. | | 6. MCP demonstration | Reuse the domain capability from an AI client. | Same API behavior, accurate tool descriptions, cross-host demonstration. | @@ -717,7 +723,9 @@ These files own the implemented behavior and the foundations for later milestone | [Subscribe host](../src/Contoso.Products.Subscribe/Program.cs) | Existing message receiving/composition foundations. | | [MCP tools](../src/Contoso.Mcp/Tools/CommerceTools.cs) | API-only search forwarding and tool description. | | [Compose infrastructure](../../docker-compose.yml) | Pinned PostgreSQL/pgvector image and optional local inference resource. | -| [Search projection](../src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs) | Exact distances, compatibility/freshness checks and atomic manual rebuild. | +| [Search projection](../src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs) | Exact distances, readiness, atomic backfill and current-state reconciliation. | +| [Product-event subscriber](../src/Contoso.Products.Subscribe/Subscribers/ProductSearchSubscriber.cs) | One globbed handler for all product lifecycle notifications. | +| [Indexing integration tests](../tests/Contoso.Products.Test.Subscribe/ProductSearchTests.cs) | Logical-delete cleanup, idempotence, inference races and retries. | | [Search migration](../src/Contoso.Products.Database/Migrations/20261013-182033-create-products-product_search-table.pgsql) | Derived table, vector extension and constraints. | | [Product query tests](../tests/Contoso.Products.Test.Api/ReadTests.ProductQuery.cs) | Semantic API behavior and schema expectations. | | [Semantic database tests](../tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs) | Controlled-vector ranking, eligibility, paging, failures and rebuild behavior. | diff --git a/samples/src/Contoso.Products.Application/Adapters/Embeddings/IProductSearchSyncAdapter.cs b/samples/src/Contoso.Products.Application/Adapters/Embeddings/IProductSearchSyncAdapter.cs new file mode 100644 index 000000000..9e81c4caa --- /dev/null +++ b/samples/src/Contoso.Products.Application/Adapters/Embeddings/IProductSearchSyncAdapter.cs @@ -0,0 +1,8 @@ +namespace Contoso.Products.Application.Adapters.Embeddings; + +/// Reconciles a product's derived search document with its current authoritative state. +public interface IProductSearchSyncAdapter +{ + /// Indexes current content or removes all projections when the product is deleted or missing. + Task SynchronizeAsync(string id, CancellationToken cancellationToken = default); +} diff --git a/samples/src/Contoso.Products.Database/Migrations/20261014-172545-create-products-product_search_readiness-table.pgsql b/samples/src/Contoso.Products.Database/Migrations/20261014-172545-create-products-product_search_readiness-table.pgsql new file mode 100644 index 000000000..3bd9724d2 --- /dev/null +++ b/samples/src/Contoso.Products.Database/Migrations/20261014-172545-create-products-product_search_readiness-table.pgsql @@ -0,0 +1,7 @@ +-- Derived search readiness is published atomically with a completed backfill. +CREATE TABLE "products"."product_search_readiness" ( + "profile" VARCHAR(250) NOT NULL, + "recipe" VARCHAR(50) NOT NULL, + "ready_on" TIMESTAMPTZ NOT NULL, + PRIMARY KEY ("profile", "recipe") +); diff --git a/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductSearchSyncAdapter.cs b/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductSearchSyncAdapter.cs new file mode 100644 index 000000000..7e9d777aa --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductSearchSyncAdapter.cs @@ -0,0 +1,11 @@ +namespace Contoso.Products.Infrastructure.Adapters.Embeddings; + +/// Exposes product search reconciliation through the application-owned boundary. +[ScopedService] +public sealed class ProductSearchSyncAdapter(Repositories.ProductSearchIndex index) : IProductSearchSyncAdapter +{ + private readonly Repositories.ProductSearchIndex _index = index.ThrowIfNull(); + + /// + public Task SynchronizeAsync(string id, CancellationToken cancellationToken = default) => Result.GoAsync(() => _index.SynchronizeAsync(id, cancellationToken)); +} diff --git a/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj b/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj index e69dd2fc3..1b3edfdc3 100644 --- a/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj +++ b/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj @@ -10,4 +10,7 @@ + + + diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs index 94b9e899c..0d3ffc119 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs @@ -7,17 +7,17 @@ public class ProductRepository(ProductsEfDb ef, ProductSearchIndex searchIndex, private readonly ProductSearchIndex _searchIndex = searchIndex.ThrowIfNull(); private readonly PostgresUnitOfWork _unitOfWork = unitOfWork.ThrowIfNull(); - public Task GetAsync(string id, CancellationToken ct = default) => _ef.Products.GetAsync(id, ct); + public Task GetAsync(string id, CancellationToken cancellationToken = default) => _ef.Products.GetAsync(id, cancellationToken); - public Task> CreateAsync(Contracts.Product product, CancellationToken ct = default) => _ef.Products.CreateAsync(product, ct); + public Task> CreateAsync(Contracts.Product product, CancellationToken cancellationToken = default) => _ef.Products.CreateAsync(product, cancellationToken); - public Task> UpdateAsync(Contracts.Product product, CancellationToken ct = default) => _ef.Products.UpdateAsync(product, ct); + public Task> UpdateAsync(Contracts.Product product, CancellationToken cancellationToken = default) => _ef.Products.UpdateAsync(product, cancellationToken); - public Task DeleteAsync(string id, CancellationToken ct = default) => _ef.Products.DeleteAsync(id, ct); + public Task DeleteAsync(string id, CancellationToken cancellationToken = default) => _ef.Products.DeleteAsync(id, cancellationToken); - public Task QuerySchemaAsync(CancellationToken ct = default) => Task.FromResult(ProductQueryArgsConfig.Default.ToJsonSchema()); + public Task QuerySchemaAsync(CancellationToken cancellationToken = default) => Task.FromResult(ProductQueryArgsConfig.Default.ToJsonSchema()); - public async Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken ct = default) + public async Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken cancellationToken = default) { var parsed = ProductQueryArgsConfig.Default.Parse(query).ThrowOnError(); var products = _ef.Products.Model.Query(); @@ -46,21 +46,20 @@ from i in ig.DefaultIfEmpty() if (parsed.IsRankedSearch) { EnsureSnapshotIsolation(); - // Fail fast before inference, then recheck in the retrieval snapshot to cover writes during inference. - await _searchIndex.EnsureFreshAsync(await q.Select(x => x.Product).AsNoTracking().ToListAsync(ct).ConfigureAwait(false), ct).ConfigureAwait(false); - vector = await _searchIndex.EmbedQueryAsync((parsed.SearchResult?.Text).ThrowIfNull(), ct).ConfigureAwait(false); + await _searchIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + vector = await _searchIndex.EmbedQueryAsync((parsed.SearchResult?.Text).ThrowIfNull(), cancellationToken).ConfigureAwait(false); } return vector is null - ? await ReadAsync(ct).ConfigureAwait(false) - : await _unitOfWork.TransactionAsync(_unitOfWork.Database.DbArgs with { IsolationLevel = IsolationLevel.RepeatableRead }, ReadAsync, ct).ConfigureAwait(false); + ? await ReadAsync(cancellationToken).ConfigureAwait(false) + : await _unitOfWork.TransactionAsync(_unitOfWork.Database.DbArgs with { IsolationLevel = IsolationLevel.RepeatableRead }, ReadAsync, cancellationToken).ConfigureAwait(false); - async Task> ReadAsync(CancellationToken token) + async Task> ReadAsync(CancellationToken cancellationToken) { if (vector is not null) { EnsureSnapshotIsolation(); - await _searchIndex.EnsureFreshAsync(await q.Select(x => x.Product).AsNoTracking().ToListAsync(token).ConfigureAwait(false), token).ConfigureAwait(false); + await _searchIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); q = (from candidate in q join match in _searchIndex.Matches(vector) on candidate.Product.Id equals match.Id orderby match.Distance, candidate.Product.Id @@ -83,7 +82,7 @@ join match in _searchIndex.Matches(vector) on candidate.Product.Id equals match. IsInactive = x.Product.IsInactive, IsNonStocked = x.Product.IsNonStocked, QtyOnHand = x.QtyOnHand - }, paging, cancellationToken: token).ConfigureAwait(false); + }, paging, cancellationToken: cancellationToken).ConfigureAwait(false); } } @@ -94,7 +93,7 @@ private void EnsureSnapshotIsolation() throw new InvalidOperationException("Semantic product search requires a RepeatableRead or Serializable parent transaction."); } - public Task> GetForReservationAsync(string[] ids, CancellationToken ct = default) + public Task> GetForReservationAsync(string[] ids, CancellationToken cancellationToken = default) { var products = _ef.Products.Model.Query(); var list = ids.ToList(); // PostgreSQL EF has issues translating array.Contains, but can do list.Contains; see: https://github.com/npgsql/efcore.pg/issues/3461. @@ -110,6 +109,6 @@ where list.Contains(p.Id) IsNonStocked = p.IsNonStocked }; - return q.ToDictionaryAsync(x => x.Id, x => x, ct); + return q.ToDictionaryAsync(x => x.Id, x => x, cancellationToken); } } diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs index 207944195..bf3d3d4d0 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs @@ -5,92 +5,190 @@ namespace Contoso.Products.Infrastructure.Repositories; /// The compatible document/query embedding provider. /// The reference-data labels used by the canonical document. /// The shared PostgreSQL transaction coordinator. +/// The indexing diagnostics logger. [ScopedService] -public sealed class ProductSearchIndex(ProductsEfDb ef, IProductEmbeddingAdapter embeddings, ReferenceDataOrchestrator referenceData, PostgresUnitOfWork unitOfWork) +public sealed class ProductSearchIndex(ProductsEfDb ef, IProductEmbeddingAdapter embeddings, ReferenceDataOrchestrator referenceData, PostgresUnitOfWork unitOfWork, ILogger logger) { + private static readonly SqlStatement _isReady = SqlStatement.FromResource("ProductSearchIsReady.pgsql"); + private static readonly SqlStatement _sourceHash = SqlStatement.FromResource("ProductSearchSourceHash.pgsql"); + private static readonly SqlStatement _lockSource = SqlStatement.FromResource("ProductSearchLockSource.pgsql"); + private static readonly SqlStatement _deleteProduct = SqlStatement.FromResource("ProductSearchDeleteProduct.pgsql"); + private static readonly SqlStatement _deleteProfile = SqlStatement.FromResource("ProductSearchDeleteProfile.pgsql"); + private static readonly SqlStatement _markReady = SqlStatement.FromResource("ProductSearchMarkReady.pgsql"); + private static readonly SqlStatement _upsert = SqlStatement.FromResource("ProductSearchUpsert.pgsql"); + private static readonly SqlStatement _matches = SqlStatement.FromResource("ProductSearchMatches.pgsql"); + private readonly ProductsEfDb _ef = ef.ThrowIfNull(); private readonly IProductEmbeddingAdapter _embeddings = embeddings.ThrowIfNull(); private readonly ReferenceDataOrchestrator _referenceData = referenceData.ThrowIfNull(); private readonly PostgresUnitOfWork _unitOfWork = unitOfWork.ThrowIfNull(); + private readonly ILogger _logger = logger.ThrowIfNull(); /// Embeds search text before opening a database snapshot. - public async Task EmbedQueryAsync(string text, CancellationToken ct = default) + public async Task EmbedQueryAsync(string text, CancellationToken cancellationToken = default) { - var vector = (await _embeddings.EmbedQueryAsync(text, ct).ConfigureAwait(false)).ThrowOnError().Value; - ValidateVector(vector); - return vector; + try + { + var vector = (await _embeddings.EmbedQueryAsync(text, cancellationToken).ConfigureAwait(false)).ThrowOnError().Value; + ValidateVector(vector); + return vector; + } + catch (Exception ex) when (ex is HttpRequestException or Polly.Timeout.TimeoutRejectedException || ex is OperationCanceledException && !cancellationToken.IsCancellationRequested) + { + throw new TransientException("Product search is temporarily unavailable. Please try again.", ex); + } } /// Returns composable exact distances for the selected model profile and document recipe. - /// Vector text is a bound SQL parameter, not interpolated SQL syntax; no provider-specific EF extension is required. + /// Kept as composable EF SQL so live filters, ranking, paging and count execute together in PostgreSQL. Vector text is a bound parameter. public IQueryable Matches(float[] vector) { var serialized = SerializeVector(vector); - return _ef.DbContext.Database.SqlQuery($""" - SELECT product_id AS "Id", - embedding <=> CAST({serialized} AS vector(768)) AS "Distance" - FROM products.product_search - WHERE profile = {_embeddings.Profile} AND recipe = {ProductSearchDocument.TemplateVersion} - """); + return _ef.SqlQuery(_matches, serialized, _embeddings.Profile, ProductSearchDocument.TemplateVersion); } - /// Refuses incomplete or stale eligible results rather than silently dropping products. - public async Task EnsureFreshAsync(IReadOnlyCollection products, CancellationToken ct = default) + /// Distinguishes a completed, possibly empty index from one never prepared for this profile. + public async Task EnsureReadyAsync(CancellationToken cancellationToken = default) { - if (products.Count == 0) - return; - await LoadLabelsAsync(ct).ConfigureAwait(false); - var hashes = await _ef.DbContext.Database.SqlQuery($""" - SELECT product_id AS "Id", source_hash AS "SourceHash" - FROM products.product_search - WHERE profile = {_embeddings.Profile} AND recipe = {ProductSearchDocument.TemplateVersion} - """).ToDictionaryAsync(x => x.Id, x => x.SourceHash, ct).ConfigureAwait(false); - - if (products.Any(product => !hashes.TryGetValue(product.Id, out var hash) || hash != Document(product).SourceHash)) - throw new TransientException("Product search is missing or stale. Rebuild the search projection before searching."); + var ready = await _ef.Database.Statement(_isReady) + .Param("Profile", _embeddings.Profile, dbType: DbType.String) + .Param("Recipe", ProductSearchDocument.TemplateVersion, dbType: DbType.String) + .ScalarAsync(cancellationToken).ConfigureAwait(false); + + if (!ready) + throw new TransientException("Product search is not initialized for the configured model and document recipe. Complete a search backfill before searching."); + } + + /// Reconciles current product state, avoiding inference for unchanged documents and removing logical deletions. + public async Task SynchronizeAsync(string id, CancellationToken cancellationToken = default) + { + id = id.ThrowIfNullOrEmpty(); + if (_unitOfWork.Database.CurrentTransaction is not null) + throw new InvalidOperationException("Search synchronization must run outside a parent transaction so inference does not hold database locks."); + + await LoadLabelsAsync(cancellationToken).ConfigureAwait(false); + var product = await _ef.Products.Model.Query().AsNoTracking().SingleOrDefaultAsync(p => p.Id == id, cancellationToken).ConfigureAwait(false); + var document = product is null ? null : Document(product); + string? vector = null; + + if (document is not null) + { + var hash = await ReadSourceHashAsync(id, cancellationToken).ConfigureAwait(false); + if (hash != document.SourceHash) + vector = await EmbedDocumentAsync(document.Text, cancellationToken).ConfigureAwait(false); + } + + await _unitOfWork.TransactionAsync(async token => + { + // Shares the rebuild's publish lock; source writes cannot race the final check and publication. + await _ef.Database.Statement(_lockSource).NonQueryAsync(token).ConfigureAwait(false); + var current = await _ef.Products.Model.Query().AsNoTracking().SingleOrDefaultAsync(p => p.Id == id, token).ConfigureAwait(false); + + if (current is null) + { + await _ef.Database.Statement(_deleteProduct).Param("ProductId", id, dbType: DbType.String).NonQueryAsync(token).ConfigureAwait(false); + + if (_logger.IsEnabled(LogLevel.Debug)) + _logger.LogDebug("Removed search projections for deleted or missing product {ProductId}.", id); + return; + } + + if (document is null || Document(current).SourceHash != document.SourceHash) + throw new TransientException("The product changed during search indexing. Retry using its current state."); + + if (vector is not null) + { + await UpsertAsync(id, document, vector, token).ConfigureAwait(false); + if (_logger.IsEnabled(LogLevel.Information)) + _logger.LogInformation("Indexed product {ProductId} for search profile {Profile} and recipe {Recipe}.", id, _embeddings.Profile, ProductSearchDocument.TemplateVersion); + } + else + { + // A concurrent rebuild may have removed the row after the initial hash comparison. + var hash = await ReadSourceHashAsync(id, token).ConfigureAwait(false); + if (hash != document.SourceHash) + throw new TransientException("The search projection changed during indexing. Retry using its current state."); + + if (_logger.IsEnabled(LogLevel.Debug)) + _logger.LogDebug("Search document for product {ProductId} is already current.", id); + } + }, cancellationToken).ConfigureAwait(false); } /// Builds embeddings outside transactions and atomically replaces this profile's projection. /// A short publish-time lock rejects concurrent catalog edits instead of installing stale documents. - public async Task RebuildAsync(CancellationToken ct = default) + public async Task RebuildAsync(CancellationToken cancellationToken = default) { - await LoadLabelsAsync(ct).ConfigureAwait(false); - var products = await _ef.Products.Model.Query().AsNoTracking().OrderBy(p => p.Id).ToListAsync(ct).ConfigureAwait(false); + await LoadLabelsAsync(cancellationToken).ConfigureAwait(false); + var products = await _ef.Products.Model.Query().AsNoTracking().OrderBy(p => p.Id).ToListAsync(cancellationToken).ConfigureAwait(false); var documents = products.Select(product => (Product: product, Document: Document(product))).ToArray(); var vectors = new List(documents.Length); foreach (var item in documents) - { - var vector = (await _embeddings.EmbedDocumentAsync(item.Document.Text, ct).ConfigureAwait(false)).ThrowOnError().Value; - vectors.Add(SerializeVector(vector)); - } + vectors.Add(await EmbedDocumentAsync(item.Document.Text, cancellationToken).ConfigureAwait(false)); return await _unitOfWork.TransactionAsync(async token => { - await _ef.DbContext.Database.ExecuteSqlRawAsync("LOCK TABLE products.product IN SHARE MODE", token).ConfigureAwait(false); + await _ef.Database.Statement(_lockSource).NonQueryAsync(token).ConfigureAwait(false); var current = await _ef.Products.Model.Query().AsNoTracking().OrderBy(p => p.Id).ToListAsync(token).ConfigureAwait(false); if (!current.Select(p => (p.Id, Document(p).SourceHash)).SequenceEqual(documents.Select(d => (d.Product.Id, d.Document.SourceHash)))) throw new ConcurrencyException("The product catalog changed during embedding. Rebuild the search projection again."); - await _ef.DbContext.Database.ExecuteSqlInterpolatedAsync($""" - DELETE FROM products.product_search WHERE profile = {_embeddings.Profile} AND recipe = {ProductSearchDocument.TemplateVersion} - """, token).ConfigureAwait(false); + await _ef.Database.Statement(_deleteProfile) + .Param("Profile", _embeddings.Profile, dbType: DbType.String) + .Param("Recipe", ProductSearchDocument.TemplateVersion, dbType: DbType.String) + .NonQueryAsync(token).ConfigureAwait(false); + for (var i = 0; i < documents.Length; i++) { var item = documents[i]; - await _ef.DbContext.Database.ExecuteSqlInterpolatedAsync($""" - INSERT INTO products.product_search (product_id, profile, recipe, source_hash, document, embedding, indexed_on) - VALUES ({item.Product.Id}, {_embeddings.Profile}, {ProductSearchDocument.TemplateVersion}, - {item.Document.SourceHash}, {item.Document.Text}, CAST({vectors[i]} AS vector(768)), {Runtime.UtcNow}) - """, token).ConfigureAwait(false); + await UpsertAsync(item.Product.Id, item.Document, vectors[i], token).ConfigureAwait(false); } + await _ef.Database.Statement(_markReady) + .Param("Profile", _embeddings.Profile, dbType: DbType.String) + .Param("Recipe", ProductSearchDocument.TemplateVersion, dbType: DbType.String) + .Param("ReadyOn", Runtime.UtcNow, dbType: DbType.DateTimeOffset) + .NonQueryAsync(token).ConfigureAwait(false); return documents.Length; - }, ct).ConfigureAwait(false); + }, cancellationToken).ConfigureAwait(false); + } + + /// Reads the stored fingerprint, or null when this product has no compatible projection. + private Task ReadSourceHashAsync(string id, CancellationToken cancellationToken) + => _ef.Database.Statement(_sourceHash) + .Param("ProductId", id, dbType: DbType.String) + .Param("Profile", _embeddings.Profile, dbType: DbType.String) + .Param("Recipe", ProductSearchDocument.TemplateVersion, dbType: DbType.String) + .ScalarAsync(cancellationToken); + + /// Publishes a validated vector and its exact source document for this profile and recipe. + private Task UpsertAsync(string id, ProductSearchDocument document, string vector, CancellationToken cancellationToken) + => _ef.Database.Statement(_upsert) + .Param("ProductId", id, dbType: DbType.String) + .Param("Profile", _embeddings.Profile, dbType: DbType.String) + .Param("Recipe", ProductSearchDocument.TemplateVersion, dbType: DbType.String) + .Param("SourceHash", document.SourceHash, dbType: DbType.String) + .Param("Document", document.Text, dbType: DbType.String) + .Param("Embedding", vector, dbType: DbType.String) + .Param("IndexedOn", Runtime.UtcNow, dbType: DbType.DateTimeOffset) + .NonQueryAsync(cancellationToken); + + /// Preserves model diagnostics while making dependency failures retriable for indexing. + private async Task EmbedDocumentAsync(string text, CancellationToken cancellationToken) + { + try + { + return SerializeVector((await _embeddings.EmbedDocumentAsync(text, cancellationToken).ConfigureAwait(false)).ThrowOnError().Value); + } + catch (Exception ex) when (ex is HttpRequestException or Polly.Timeout.TimeoutRejectedException || ex is OperationCanceledException && !cancellationToken.IsCancellationRequested) + { + throw new TransientException("Product search indexing is temporarily unavailable. Please retry.", ex); + } } /// Loads the labels before synchronous generated navigation getters are used. - private async Task LoadLabelsAsync(CancellationToken ct) - => await _referenceData.PrefetchAsync([typeof(Contracts.Brand), typeof(Contracts.Category), typeof(Contracts.SubCategory)], ct).ConfigureAwait(false); + private async Task LoadLabelsAsync(CancellationToken cancellationToken) + => await _referenceData.PrefetchAsync([typeof(Contracts.Brand), typeof(Contracts.Category), typeof(Contracts.SubCategory)], cancellationToken).ConfigureAwait(false); /// Uses the same category derivation as the application contract. private static ProductSearchDocument Document(Persistence.Product product) @@ -118,13 +216,4 @@ public sealed class SearchMatch /// Gets or sets cosine distance; lower is closer. public double Distance { get; set; } } - - /// Represents the indexed content fingerprint without loading vector values. - public sealed class SearchFingerprint - { - /// Gets or sets the product identifier. - public string Id { get; set; } = string.Empty; - /// Gets or sets the indexed document fingerprint. - public string SourceHash { get; set; } = string.Empty; - } } diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchDeleteProduct.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchDeleteProduct.pgsql new file mode 100644 index 000000000..3e0229e62 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchDeleteProduct.pgsql @@ -0,0 +1 @@ +DELETE FROM products.product_search WHERE product_id = @ProductId; diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchDeleteProfile.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchDeleteProfile.pgsql new file mode 100644 index 000000000..08ddccd73 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchDeleteProfile.pgsql @@ -0,0 +1 @@ +DELETE FROM products.product_search WHERE profile = @Profile AND recipe = @Recipe; diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchIsReady.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchIsReady.pgsql new file mode 100644 index 000000000..5925cd37d --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchIsReady.pgsql @@ -0,0 +1,4 @@ +SELECT EXISTS ( + SELECT 1 FROM products.product_search_readiness + WHERE profile = @Profile AND recipe = @Recipe +); diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchLockSource.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchLockSource.pgsql new file mode 100644 index 000000000..0e9a1c0bf --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchLockSource.pgsql @@ -0,0 +1 @@ +LOCK TABLE products.product IN SHARE MODE; diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchMarkReady.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchMarkReady.pgsql new file mode 100644 index 000000000..44d4c76c8 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchMarkReady.pgsql @@ -0,0 +1,3 @@ +INSERT INTO products.product_search_readiness (profile, recipe, ready_on) +VALUES (@Profile, @Recipe, @ReadyOn) +ON CONFLICT (profile, recipe) DO UPDATE SET ready_on = EXCLUDED.ready_on; diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchMatches.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchMatches.pgsql new file mode 100644 index 000000000..b4d86aaae --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchMatches.pgsql @@ -0,0 +1,4 @@ +SELECT product_id AS "Id", + embedding <=> CAST({0} AS vector(768)) AS "Distance" +FROM products.product_search +WHERE profile = {1} AND recipe = {2} diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchSourceHash.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchSourceHash.pgsql new file mode 100644 index 000000000..d7208316f --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchSourceHash.pgsql @@ -0,0 +1,2 @@ +SELECT source_hash FROM products.product_search +WHERE product_id = @ProductId AND profile = @Profile AND recipe = @Recipe; diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchUpsert.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchUpsert.pgsql new file mode 100644 index 000000000..3d1ce9ffa --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductSearchUpsert.pgsql @@ -0,0 +1,7 @@ +INSERT INTO products.product_search (product_id, profile, recipe, source_hash, document, embedding, indexed_on) +VALUES (@ProductId, @Profile, @Recipe, @SourceHash, @Document, CAST(@Embedding AS vector(768)), @IndexedOn) +ON CONFLICT (product_id, profile, recipe) DO UPDATE SET + source_hash = EXCLUDED.source_hash, + document = EXCLUDED.document, + embedding = EXCLUDED.embedding, + indexed_on = EXCLUDED.indexed_on; diff --git a/samples/src/Contoso.Products.Subscribe/GlobalUsing.cs b/samples/src/Contoso.Products.Subscribe/GlobalUsing.cs index 09420ff62..aab4d2cde 100644 --- a/samples/src/Contoso.Products.Subscribe/GlobalUsing.cs +++ b/samples/src/Contoso.Products.Subscribe/GlobalUsing.cs @@ -1,3 +1,4 @@ +global using Contoso.Products.Application.Adapters.Embeddings; global using Contoso.Products.Application.Interfaces; global using Contoso.Products.Infrastructure.Clients.Ollama; global using CoreEx; diff --git a/samples/src/Contoso.Products.Subscribe/Subscribers/ProductSearchSubscriber.cs b/samples/src/Contoso.Products.Subscribe/Subscribers/ProductSearchSubscriber.cs new file mode 100644 index 000000000..9f746f4e1 --- /dev/null +++ b/samples/src/Contoso.Products.Subscribe/Subscribers/ProductSearchSubscriber.cs @@ -0,0 +1,12 @@ +namespace Contoso.Products.Subscribe.Subscribers; + +/// Reconciles all product lifecycle notifications against authoritative state, including logical deletion. +[ScopedService, Subscribe("contoso.products.product.**")] +public sealed class ProductSearchSubscriber(IProductSearchSyncAdapter adapter) : SubscribedBase +{ + private readonly IProductSearchSyncAdapter _adapter = adapter.ThrowIfNull(); + + /// + protected override Task OnReceiveAsync(EventData @event, EventSubscriberArgs args, CancellationToken cancellationToken = default) + => _adapter.SynchronizeAsync(@event.Key.Required(), cancellationToken); +} diff --git a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs index 3efac42fd..6be7ba201 100644 --- a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs +++ b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs @@ -104,11 +104,10 @@ public void Product_Search_GraphQLMatchesRest() .Select(edge => edge.GetProperty("node").GetProperty("id").GetString()).Should().Equal(rest.Select(p => p.Id)); } - [TestCase("DELETE FROM products.product_search WHERE profile = 'tests:controlled-vectors:768'")] - [TestCase("UPDATE products.product_search SET profile = 'incompatible' WHERE profile = 'tests:controlled-vectors:768'")] - [TestCase("UPDATE products.product_search SET recipe = 'old-recipe' WHERE profile = 'tests:controlled-vectors:768'")] - [TestCase("UPDATE products.product_search SET source_hash = repeat('0', 64) WHERE profile = 'tests:controlled-vectors:768'")] - public void Product_Search_InvalidProjectionIsUnavailable(string sql) + [TestCase("DELETE FROM products.product_search_readiness WHERE profile = 'tests:controlled-vectors:768'")] + [TestCase("UPDATE products.product_search_readiness SET profile = 'incompatible' WHERE profile = 'tests:controlled-vectors:768'")] + [TestCase("UPDATE products.product_search_readiness SET recipe = 'old-recipe' WHERE profile = 'tests:controlled-vectors:768'")] + public void Product_Search_UninitializedProfileIsUnavailable(string sql) { try { @@ -125,7 +124,7 @@ public void Product_Search_InvalidProjectionIsUnavailable(string sql) } finally { - ExecuteSearchSql("DELETE FROM products.product_search WHERE profile = 'incompatible' OR recipe = 'old-recipe'"); + ExecuteSearchSql("DELETE FROM products.product_search_readiness WHERE profile = 'incompatible' OR recipe = 'old-recipe'"); RebuildSearch(); } } @@ -154,22 +153,116 @@ public void Product_Search_QueryEmbeddingFailureHasNoFallback() _embeddings.FailQuery = true; try { - Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.InternalServerError); + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.ServiceUnavailable); + } + finally + { + _embeddings.FailQuery = false; + } + } + + [Test] + public void Product_Search_TransientFailurePreservesDiagnosticsAndCancellation() => Test.Scoped(test => test.Run(async _ => + { + var index = test.Services.GetRequiredService(); + _embeddings.FailQuery = true; + try + { + Func search = () => index.EmbedQueryAsync("cleaner"); + var failure = await search.Should().ThrowAsync().ConfigureAwait(false); + failure.Which.InnerException.Should().BeOfType().Which.Message.Should().Be("Controlled query embedding failure."); } finally { _embeddings.FailQuery = false; } + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + Func canceled = () => index.EmbedQueryAsync("cleaner", cancellation.Token); + await canceled.Should().ThrowAsync().ConfigureAwait(false); + }).AssertSuccess()); + + [TestCase("is_inactive")] + [TestCase("is_deleted")] + public void Product_Search_LiveEligibilityExcludesRetainedVectors(string column) + { + try + { + ExecuteSearchSql($"UPDATE products.product SET {column} = true WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner&$filter=sku eq 'MUCOFF-NANO-CLEANER'") + .AssertOK().Value.Should().BeEmpty(); + if (column == "is_inactive") + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner&$inactive&$filter=sku eq 'MUCOFF-NANO-CLEANER'") + .AssertOK().Value.Should().ContainSingle(); + } + finally + { + ExecuteSearchSql($"UPDATE products.product SET {column} = false WHERE sku = 'MUCOFF-NANO-CLEANER'"); + } } [Test] - public void Product_Search_DescriptionEditRequiresRebuild() + public void Product_Search_FailedInitialBackfillDoesNotPublishReadiness() + { + ExecuteSearchSql("DELETE FROM products.product_search_readiness WHERE profile = 'tests:controlled-vectors:768'"); + _embeddings.DocumentCallCount = 0; + _embeddings.FailDocumentOnCall = 2; + try + { + Test.Scoped(test => test.Run(async _ => + { + var index = test.Services.GetRequiredService(); + Func rebuild = () => index.RebuildAsync(); + await rebuild.Should().ThrowAsync().ConfigureAwait(false); + Func ready = () => index.EnsureReadyAsync(); + await ready.Should().ThrowAsync().ConfigureAwait(false); + }).AssertSuccess()); + } + finally + { + _embeddings.FailDocumentOnCall = null; + RebuildSearch(); + } + } + + [Test] + public void Product_Search_EmptyBackfillPublishesReadiness() + { + string[] ids = []; + Test.Scoped(test => test.Run(async _ => + ids = await test.Services.GetRequiredService().Database + .SqlQuery($"SELECT product_id AS \"Value\" FROM products.product WHERE NOT is_deleted").ToArrayAsync().ConfigureAwait(false)).AssertSuccess()); + try + { + ExecuteSearchSql("UPDATE products.product SET is_deleted = true WHERE NOT is_deleted"); + ExecuteSearchSql("DELETE FROM products.product_search_readiness WHERE profile = 'tests:controlled-vectors:768'"); + Test.Scoped(test => test.Run(async _ => + { + var index = test.Services.GetRequiredService(); + (await index.RebuildAsync().ConfigureAwait(false)).Should().Be(0); + await index.EnsureReadyAsync().ConfigureAwait(false); + }).AssertSuccess()); + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").AssertOK().Value.Should().BeEmpty(); + } + finally + { + Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().Database.ExecuteSqlInterpolatedAsync( + $"UPDATE products.product SET is_deleted = false WHERE product_id = ANY ({ids})").ConfigureAwait(false)).AssertSuccess()); + RebuildSearch(); + } + } + + [Test] + public void Product_Search_DescriptionEditReturnsLiveContentWhileIndexLags() { var product = Test.Http().Run(HttpMethod.Get, $"/api/products/{29.ToGuid()}").AssertOK().Value!; try { ExecuteSearchSql("UPDATE products.product SET description = 'Changed searchable meaning.' WHERE sku = 'MUCOFF-NANO-CLEANER'"); - Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.ServiceUnavailable); + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner&$filter=sku eq 'MUCOFF-NANO-CLEANER'") + .AssertOK().Value.Should().ContainSingle().Which.Description.Should().Be("Changed searchable meaning."); } finally { @@ -207,7 +300,7 @@ public void Product_Search_FailedRebuildPreservesPublishedIndex() try { Test.Scoped(test => test.Run(async _ => - await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false)).AssertException()); + await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false)).AssertException()); Test.Http().Run(HttpMethod.Get, "/api/products?$search=changing gears&$take=3").AssertOK() .Value.Should().HaveCount(3).And.OnlyContain(p => p.Text!.Contains("Rear Derailleur")); } @@ -246,4 +339,36 @@ await test.Services.GetRequiredService().Database.ExecuteSqlI private void ExecuteSearchSql(string sql) => Test.Scoped(test => test.Run(async _ => await test.Services.GetRequiredService().Database.ExecuteSqlRawAsync(sql).ConfigureAwait(false)).AssertSuccess()); + + [Test] + public void Product_Search_MissingVectorChangesCandidateCountWithoutUnavailableResponse() + { + try + { + ExecuteSearchSql("DELETE FROM products.product_search WHERE profile = 'tests:controlled-vectors:768' AND product_id = (SELECT product_id FROM products.product WHERE sku = 'MUCOFF-NANO-CLEANER')"); + var response = Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner&$take=50&$count=true").AssertOK(); + response.Value.Should().HaveCount(28).And.NotContain(p => p.Sku == "MUCOFF-NANO-CLEANER"); + response.Response.Headers.Should().ContainKey("X-Paging-Total-Count").WhoseValue.Should().ContainSingle().Which.Should().Be("28"); + } + finally + { + RebuildSearch(); + } + } + + [Test] + public void Product_Search_ReadyEmptyProjectionIsSuccessful() + { + try + { + ExecuteSearchSql("DELETE FROM products.product_search WHERE profile = 'tests:controlled-vectors:768'"); + var response = Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner&$count=true").AssertOK(); + response.Value.Should().BeEmpty(); + response.Response.Headers.Should().ContainKey("X-Paging-Total-Count").WhoseValue.Should().ContainSingle().Which.Should().Be("0"); + } + finally + { + RebuildSearch(); + } + } } diff --git a/samples/tests/Contoso.Products.Test.Subscribe/GlobalUsing.cs b/samples/tests/Contoso.Products.Test.Subscribe/GlobalUsing.cs index 6e413dbee..c637932b0 100644 --- a/samples/tests/Contoso.Products.Test.Subscribe/GlobalUsing.cs +++ b/samples/tests/Contoso.Products.Test.Subscribe/GlobalUsing.cs @@ -1,14 +1,18 @@ +global using AwesomeAssertions; +global using Contoso.Products.Application.Adapters.Embeddings; global using Contoso.Products.Application.Interfaces; global using Contoso.Products.Contracts; +global using Contoso.Products.Infrastructure.Repositories; global using CoreEx; global using CoreEx.Azure.Messaging.ServiceBus; global using CoreEx.Events; global using CoreEx.Events.Subscribing; global using CoreEx.Events.Subscribing.Exceptions; -global using AwesomeAssertions; +global using CoreEx.Results; +global using Microsoft.EntityFrameworkCore; global using Microsoft.Extensions.DependencyInjection; global using NUnit.Framework; global using UnitTestEx; global using UnitTestEx.Expectations; global using DbMigration = Contoso.Products.Database.Program; -global using TestData = Contoso.Products.Test.Common.TestData; \ No newline at end of file +global using TestData = Contoso.Products.Test.Common.TestData; diff --git a/samples/tests/Contoso.Products.Test.Subscribe/ProductSearchTests.cs b/samples/tests/Contoso.Products.Test.Subscribe/ProductSearchTests.cs new file mode 100644 index 000000000..f64a09250 --- /dev/null +++ b/samples/tests/Contoso.Products.Test.Subscribe/ProductSearchTests.cs @@ -0,0 +1,153 @@ +namespace Contoso.Products.Test.Subscribe; + +public class ProductSearchTests : WithApiTester +{ + private readonly ControlledEmbeddings _embeddings = new(); + private readonly string _id = 29.ToGuid().ToString(); + private string? _concurrentSql; + + [OneTimeSetUp] + public void OneTimeSetUp() => Test.ReplaceScoped(_ => _embeddings); + + [SetUp] + public async Task SetUpAsync() + { + await Test.MigratePostgresDataAsync(["mutate-data.seed.yaml"], DbMigration.ConfigureMigrationArgs).ConfigureAwait(false); + await Test.ClearFusionCacheAsync().ConfigureAwait(false); + _embeddings.Calls = 0; + _embeddings.BeforeEmbedding = null; + _embeddings.Fail = false; + _concurrentSql = null; + ExecuteSql("DELETE FROM products.product_search WHERE profile = 'tests:subscriber:768'"); + ExecuteSql("DELETE FROM products.product_search_readiness WHERE profile = 'tests:subscriber:768'"); + } + + [TestCase("created")] + [TestCase("updated")] + [TestCase("activated")] + [TestCase("deactivated")] + [TestCase("deleted")] + public void ProductLifecycle_GlobReconcilesCurrentStateInsteadOfPayload(string action) + { + Receive(action); + Count().Should().Be(1); + _embeddings.Calls.Should().Be(1); + Receive(action); + _embeddings.Calls.Should().Be(1); + Test.Scoped(test => test.Run(async _ => + { + var index = test.Services.GetRequiredService(); + Func ready = () => index.EnsureReadyAsync(); + await ready.Should().ThrowAsync().ConfigureAwait(false); + }).AssertSuccess()); + } + + [Test] + public void Update_ReembedsChangedContentButNotPriceOrActivity() + { + Receive("created"); + ExecuteSql("UPDATE products.product SET price = 19, is_inactive = true WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Receive("updated"); + Receive("deactivated"); + _embeddings.Calls.Should().Be(1); + ExecuteSql("UPDATE products.product SET description = 'Changed search text.' WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Receive("updated"); + _embeddings.Calls.Should().Be(2); + } + + [Test] + public void LogicalDelete_RemovesEveryProfileAndOldUpdateCannotResurrect() + { + Receive("created"); + ExecuteSql("INSERT INTO products.product_search SELECT product_id, 'tests:old-profile', recipe, source_hash, document, embedding, indexed_on FROM products.product_search WHERE profile = 'tests:subscriber:768'"); + ExecuteSql("UPDATE products.product SET is_deleted = true WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Receive("deleted"); + Count(allProfiles: true).Should().Be(0); + Receive("updated"); + Count(allProfiles: true).Should().Be(0); + _embeddings.Calls.Should().Be(1); + } + + [Test] + public void DeleteDuringInference_DoesNotPublishVector() + { + _concurrentSql = "UPDATE products.product SET is_deleted = true WHERE sku = 'MUCOFF-NANO-CLEANER'"; + Receive("created"); + Count(allProfiles: true).Should().Be(0); + } + + [Test] + public void EditDuringInference_RetriesWithoutPublishingObsoleteVector() + { + _concurrentSql = "UPDATE products.product SET description = 'Changed during inference.' WHERE sku = 'MUCOFF-NANO-CLEANER'"; + Receive("created", retry: true); + Count().Should().Be(0); + _concurrentSql = null; + Receive("updated"); + Count().Should().Be(1); + } + + [Test] + public void ProviderFailure_RetriesAndPreservesPublishedVector() + { + Receive("created"); + ExecuteSql("UPDATE products.product SET description = 'Changed before provider failure.' WHERE sku = 'MUCOFF-NANO-CLEANER'"); + _embeddings.Fail = true; + Receive("updated", retry: true); + Count().Should().Be(1); + _embeddings.Fail = false; + Receive("updated"); + Count().Should().Be(1); + _embeddings.Calls.Should().Be(3); + } + + private void Receive(string action, bool retry = false) => Test.Scoped(test => test.Run(async _ => + { + _embeddings.BeforeEmbedding = () => + { + if (_concurrentSql is null) + return; + using var scope = test.Services.CreateScope(); + scope.ServiceProvider.GetRequiredService().Database.ExecuteSqlRaw(_concurrentSql); + }; + var message = Test.CreateCloudEventFrom(EventData.CreateEvent(action).WithKey(_id)).ToServiceBusReceivedMessage(); + var result = await test.Services.GetRequiredKeyedService("subscriber-events").ReceiveAsync(message).ConfigureAwait(false); + if (retry) + result.Error.Should().BeOfType(because: result.Error?.ToString()).Which.InnerException.Should().BeOfType(); + else + result.IsSuccess.Should().BeTrue(); + }).AssertSuccess()); + + private int Count(bool allProfiles = false) + { + var count = 0; + Test.Scoped(test => test.Run(async _ => + { + var db = test.Services.GetRequiredService().Database; + count = allProfiles + ? await db.SqlQuery($"SELECT count(*)::int AS \"Value\" FROM products.product_search WHERE product_id = {_id}").SingleAsync().ConfigureAwait(false) + : await db.SqlQuery($"SELECT count(*)::int AS \"Value\" FROM products.product_search WHERE product_id = {_id} AND profile = 'tests:subscriber:768'").SingleAsync().ConfigureAwait(false); + }).AssertSuccess()); + return count; + } + + private void ExecuteSql(string sql) => Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().Database.ExecuteSqlRawAsync(sql).ConfigureAwait(false)).AssertSuccess()); + + private sealed class ControlledEmbeddings : IProductEmbeddingAdapter + { + public string Profile => "tests:subscriber:768"; + public int Dimensions => 768; + public int Calls { get; set; } + public bool Fail { get; set; } + public Action? BeforeEmbedding { get; set; } + public Task> EmbedDocumentAsync(string text, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + Calls++; + BeforeEmbedding?.Invoke(); + return Task.FromResult(Fail ? Result.Fail(new HttpRequestException("Controlled provider failure.")) : Result.Ok(Enumerable.Repeat(0.1f, 768).ToArray())); + } + public Task> EmbedQueryAsync(string text, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + } +} diff --git a/samples/tests/Contoso.Products.Test.Subscribe/SubscriberTests.cs b/samples/tests/Contoso.Products.Test.Subscribe/SubscriberTests.cs index 185ed2a8c..c3f712ccf 100644 --- a/samples/tests/Contoso.Products.Test.Subscribe/SubscriberTests.cs +++ b/samples/tests/Contoso.Products.Test.Subscribe/SubscriberTests.cs @@ -17,7 +17,7 @@ public async Task OneTimeSetUpAsync() [Test] public void Unsubscribed_Error() => Test.Scoped(test => { - var ed = EventData.CreateEvent("product", "deleted").WithKey("abc"); + var ed = EventData.CreateEvent("unsubscribed", "deleted").WithKey("abc"); var ce = Test.CreateCloudEventFrom(ed); var sbm = ce.ToServiceBusReceivedMessage(); @@ -33,4 +33,4 @@ public void Unsubscribed_Error() => Test.Scoped(test => e.InnerException.Message.Should().Be("No subscriber matched the event."); }).AssertSuccess(); }); -} \ No newline at end of file +} diff --git a/src/CoreEx.EntityFrameworkCore/AGENTS.md b/src/CoreEx.EntityFrameworkCore/AGENTS.md index 796658193..dd970306f 100644 --- a/src/CoreEx.EntityFrameworkCore/AGENTS.md +++ b/src/CoreEx.EntityFrameworkCore/AGENTS.md @@ -68,6 +68,10 @@ public async Task> QueryAsync(QueryArgs? query, PagingArgs? > If the read side projects to a reduced "Lite" contract instead of the full entity, `ProductMapper.From.Map(m)` (an `IBiDirectionMapper`) still returns `Product`, not `ProductLite` — a mapper only knows its own two types. Project inline instead: `m => new ProductLite { Id = m.Id, Sku = m.Sku, ... }`. +## Composable SQL + +Use `ef.SqlQuery(statement, parameters)` for deferred scalar/unmapped EF queries over a CoreEx `SqlStatement`, including cached `SqlStatement.FromResource(...)` resources. EF binds `{0}`, `{1}`, etc. to separately supplied values, or accepts explicit provider `DbParameter` instances with named placeholders. Never format parameter values into SQL. Only determinate text statements are accepted; normal EF/provider composability rules apply (no trailing semicolon for composed SQL, and `Value` as the column name for composed scalar queries). Materialize with cancellation. This uses the existing EF context/transaction and EF diagnostics, not the CoreEx Database execution pipeline. It does not automatically apply tenant/logical-delete model filters. Use `ef.Database.Statement(...)` for immediate commands instead. + ## ValueConverter Bridge Use `ValueConverterBridge` in `OnModelCreating` to reuse CoreEx `IConverter` instances as EF value converters. diff --git a/src/CoreEx.EntityFrameworkCore/EfDbExtensions.cs b/src/CoreEx.EntityFrameworkCore/EfDbExtensions.cs index 0ecca936f..4d62f515a 100644 --- a/src/CoreEx.EntityFrameworkCore/EfDbExtensions.cs +++ b/src/CoreEx.EntityFrameworkCore/EfDbExtensions.cs @@ -5,6 +5,31 @@ namespace CoreEx.EntityFrameworkCore; /// public static partial class EfDbExtensions { + /// + /// Creates a deferred, composable SQL query using the existing Entity Framework context; see . + /// + /// The scalar or unmapped result type supported by Entity Framework. + /// The CoreEx Entity Framework database accessor. + /// The SQL text statement, optionally loaded from an embedded resource. + /// Values for SQL placeholders, or explicit instances. + /// The unexecuted query. + /// Parameters are bound by Entity Framework, never formatted into SQL. Composed queries must satisfy the provider's composable SQL rules (for example, no trailing semicolon); scalar queries + /// composed with LINQ require a column named Value. Execution uses the context's transaction and Entity Framework diagnostics and error handling, not the CoreEx Database command pipeline. + /// No CoreEx model filters (including tenant and logical-delete filters) are automatically added; the caller must enforce eligibility. Pass cancellation when materializing the query. + public static IQueryable SqlQuery(this IEfDb database, SqlStatement statement, params object[] parameters) + { + database.ThrowIfNull(); + statement.ThrowIfNull(); + parameters.ThrowIfNull(); + if (statement.IsIndeterminate) + throw new InvalidOperationException("A SQL query requires a determinate statement."); + + if (statement.CommandType != System.Data.CommandType.Text) + throw new ArgumentException("A SQL query requires a text statement.", nameof(statement)); + + return database.DbContext.Database.SqlQueryRaw(statement.CommandText, parameters); + } + /// /// Creates a from a using the specified . /// @@ -153,4 +178,4 @@ public static async Task> ToMappedItemsResultAsync mapper.Map(source)!, paging, autoCount, cancellationToken).ConfigureAwait(false); } -} \ No newline at end of file +} diff --git a/src/CoreEx.EntityFrameworkCore/README.md b/src/CoreEx.EntityFrameworkCore/README.md index 753c63e27..f2182009d 100644 --- a/src/CoreEx.EntityFrameworkCore/README.md +++ b/src/CoreEx.EntityFrameworkCore/README.md @@ -40,6 +40,21 @@ The central type is `EfDb`, which holds the `DbContext`, bridges its | [`IEfDb`](./IEfDb.cs) | Interface exposing `DbContext`, `IDatabase`, `EfDbOptions`, `ExecutionContext`, and `EfDbInvoker`; implemented by `EfDb`. | | [`IEfDbContext`](./IEfDbContext.cs) | Marker interface for `DbContext` subclasses wiring the `IDatabase` bridge via `BaseDatabase`. | +## Composable SQL statements + +`IEfDb.SqlQuery(SqlStatement, params object[])` bridges CoreEx SQL statements to EF's deferred, composable scalar/unmapped queries. Cache embedded statements and supply values separately: + +```csharp +private static readonly SqlStatement _matches = SqlStatement.FromResource("ProductMatches.pgsql"); + +public IQueryable Matches(string profile) + => _ef.SqlQuery(_matches, profile); +``` + +The resource can contain `SELECT product_id AS "Id" FROM products.product_search WHERE profile = {0}`. EF binds placeholder values; explicit provider `DbParameter` instances with named placeholders are also supported. Do not interpolate values into SQL or call `string.Format`. Only determinate `CommandType.Text` statements are accepted; stored-procedure statements are rejected. Normal EF/provider composability rules apply (including no trailing semicolon for a composed query); SQL is not rewritten. A composed scalar query needs a `Value` column. + +Materialization (for example, `ToListAsync(cancellationToken)`) executes through EF with the context's current transaction and EF diagnostics/error handling, not the CoreEx Database command pipeline. The helper does not apply CoreEx model filters: callers must enforce tenant, logical-delete and other eligibility rules explicitly. For immediate scalar or non-query commands, continue to use `ef.Database.Statement(...)`. + ## Namespaces | Namespace | Description | @@ -55,4 +70,4 @@ The central type is `EfDb`, which holds the `DbContext`, bridges its ## AI Usage Guide -An [`AGENTS.md`](./AGENTS.md) file is included with this package. AI coding assistants (GitHub Copilot, Claude, Cursor, etc.) that support workspace-injected package documentation will automatically surface concise usage guidance, code examples, and `Do Not` rules for this package without requiring a local CoreEx checkout. \ No newline at end of file +An [`AGENTS.md`](./AGENTS.md) file is included with this package. AI coding assistants (GitHub Copilot, Claude, Cursor, etc.) that support workspace-injected package documentation will automatically surface concise usage guidance, code examples, and `Do Not` rules for this package without requiring a local CoreEx checkout. diff --git a/tests/CoreEx.Database.Postgres.Test.Unit/EntityFrameworkSqlQueryTests.cs b/tests/CoreEx.Database.Postgres.Test.Unit/EntityFrameworkSqlQueryTests.cs new file mode 100644 index 000000000..dc48d79c7 --- /dev/null +++ b/tests/CoreEx.Database.Postgres.Test.Unit/EntityFrameworkSqlQueryTests.cs @@ -0,0 +1,87 @@ +namespace CoreEx.Database.Postgres.Test.Unit; + +public class EntityFrameworkSqlQueryTests : DatabaseTestBase +{ + [Test] + public void SqlQuery_BindsValuesWithoutInterpretingSql() => Test.ScopedType(test => test.Run(async _ => + { + var ef = ExecutionContext.GetRequiredService(); + const string value = "O'Brien'; SELECT 99; -- {0}"; + var query = ef.SqlQuery(SqlStatement.FromText("SELECT CAST({0} AS text) AS \"Value\""), value); + (await query.SingleAsync().ConfigureAwait(false)).Should().Be(value); + }).AssertSuccess()); + + [Test] + public void SqlQuery_ForwardsExplicitProviderParameter() => Test.ScopedType(test => test.Run(async _ => + { + var ef = ExecutionContext.GetRequiredService(); + var parameter = new NpgsqlParameter("Input", "named parameter"); + var query = ef.SqlQuery(SqlStatement.FromText("SELECT CAST(@Input AS text) AS \"Value\""), parameter); + (await query.SingleAsync().ConfigureAwait(false)).Should().Be("named parameter"); + }).AssertSuccess()); + + [Test] + public void SqlQuery_ComposesJoinFilterOrderingPagingAndCount() => Test.ScopedType(test => test.Run(async _ => + { + var ef = ExecutionContext.GetRequiredService(); + var statement = SqlStatement.FromText("SELECT n AS \"Number\" FROM generate_series(1, {0}) AS n"); + var numbers = ef.SqlQuery(statement, 6); + var query = from left in numbers + join right in numbers on left.Number equals right.Number + where left.Number > 2 + orderby left.Number descending + select left.Number; + + (await query.CountAsync().ConfigureAwait(false)).Should().Be(4); + (await query.Skip(1).Take(2).ToArrayAsync().ConfigureAwait(false)).Should().Equal(5, 4); + }).AssertSuccess()); + + [Test] + public void SqlQuery_IsDeferredAndUsesSharedTransaction() => Test.ScopedType(test => test.Run(async _ => + { + var ef = ExecutionContext.GetRequiredService(); + var uow = ExecutionContext.GetRequiredService(); + var query = ef.SqlQuery(SqlStatement.FromText("SELECT text AS \"Value\" FROM test.\"table\" WHERE table_id = {0}"), 1.ToGuid()); + var original = await query.SingleAsync().ConfigureAwait(false); + Func work = () => uow.TransactionAsync(async ct => + { + await ef.Database.Statement("UPDATE test.\"table\" SET text = @Text WHERE table_id = @Id") + .Param("Text", "Visible before commit", dbType: DbType.String) + .Param("Id", 1.ToGuid(), dbType: DbType.Guid) + .NonQueryAsync(ct).ConfigureAwait(false); + (await query.SingleAsync(ct).ConfigureAwait(false)).Should().Be("Visible before commit"); + var transaction = ef.DbContext.Database.CurrentTransaction ?? throw new InvalidOperationException("Missing shared transaction."); + transaction.GetDbTransaction().Should().BeSameAs(uow.Database.CurrentTransaction); + throw new InvalidOperationException("Rollback query test."); + }); + + await work.Should().ThrowAsync().WithMessage("Rollback query test.").ConfigureAwait(false); + (await query.SingleAsync().ConfigureAwait(false)).Should().Be(original); + ef.DbContext.Database.CurrentTransaction.Should().BeNull(); + uow.Database.CurrentTransaction.Should().BeNull(); + }).AssertSuccess()); + + [Test] + public void SqlQuery_RejectsUnsupportedStatementsAndNullArguments() => Test.ScopedType(test => test.Run(_ => + { + var ef = ExecutionContext.GetRequiredService(); + Action indeterminate = () => ef.SqlQuery(SqlStatement.Indeterminate); + Action procedure = () => ef.SqlQuery(SqlStatement.StoredProcedure("test.procedure")); + // Null-forgiving values deliberately exercise runtime argument guards. + Action missingStatement = () => ef.SqlQuery(null!); + Action missingParameters = () => ef.SqlQuery(SqlStatement.FromText("SELECT 1"), null!); + Action missingDatabase = () => EfDbExtensions.SqlQuery(null!, SqlStatement.FromText("SELECT 1")); + + indeterminate.Should().Throw().WithMessage("*determinate statement*"); + procedure.Should().Throw().WithParameterName("statement"); + missingStatement.Should().Throw().WithParameterName("statement"); + missingParameters.Should().Throw().WithParameterName("parameters"); + missingDatabase.Should().Throw().WithParameterName("database"); + return Task.CompletedTask; + }).AssertSuccess()); + + public sealed class NumberRow + { + public int Number { get; set; } + } +} diff --git a/tests/CoreEx.Database.Postgres.Test.Unit/GlobalUsing.cs b/tests/CoreEx.Database.Postgres.Test.Unit/GlobalUsing.cs index 0b83088c7..cda9d55a5 100644 --- a/tests/CoreEx.Database.Postgres.Test.Unit/GlobalUsing.cs +++ b/tests/CoreEx.Database.Postgres.Test.Unit/GlobalUsing.cs @@ -1,2 +1,6 @@ +global using CoreEx.Database.Postgres.Test.Unit.Repository; +global using CoreEx.EntityFrameworkCore; +global using Microsoft.EntityFrameworkCore; global using Microsoft.EntityFrameworkCore.Storage; +global using Npgsql; global using System.Data; From 66b711cde9542620fedd3f24a15b5e3c978a4581 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Thu, 15 Oct 2026 09:20:36 -0700 Subject: [PATCH 08/25] Update dependencies and streamline local builds and publishing Adopt DbEx 3.3.1 and patched dependency versions across framework and templates. Add opt-in net10 local builds, build-output cleanup, and leaner template validation. Preserve shared feed packages and publish only packages generated by the current run. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .config/dotnet-tools.json | 2 +- .github/copilot-instructions.md | 2 + Directory.Build.targets | 13 +++ Directory.Packages.props | 109 +++++++++--------- nuget-publish.ps1 | 78 ++++++------- .../Contoso.Aspire/Contoso.Aspire.csproj | 2 +- .../Contoso.Orders.Database.csproj | 2 +- .../Contoso.Products.Api.csproj | 1 + .../Contoso.Shopping.Database.csproj | 2 +- .../CoreEx.Database.SqlServer.csproj | 1 + .../app-name.Aspire/app-name.Aspire.csproj | 4 +- .../CoreEx.Core/_Directory.Packages.props | 30 ++--- src/Directory.Build.targets | 5 +- tools/clean-build-outputs.ps1 | 92 +++++++++++++++ tools/validate-template-pack.ps1 | 23 +++- 15 files changed, 244 insertions(+), 122 deletions(-) create mode 100644 Directory.Build.targets create mode 100644 tools/clean-build-outputs.ps1 diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index b1dbbf049..5804950ea 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-reportgenerator-globaltool": { - "version": "5.5.10", + "version": "5.5.11", "commands": [ "reportgenerator" ] diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c3360c55d..52c90c185 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -37,6 +37,8 @@ solution; the rule applies to consumer solutions where these assets are installe - **Build**: `dotnet build CoreEx.sln` - **Test**: `dotnet test CoreEx.sln` or target specific projects. - **Single test**: `dotnet test --filter "FullyQualifiedName~"` +- **Local target framework (inner loop)**: projects multi-target `net8.0;net9.0;net10.0`. Locally, develop, build, and test on `net10.0` Debug only — set the user environment variable `CoreExLocalTargetFramework=net10.0` (honoured by the root `Directory.Build.targets`; ignored when `CI=true` and when packing). Do **not** use `dotnet build -f net10.0` on the solution: it fails with `NETSDK1005` on non-net10 projects such as `CoreEx.Generator` (netstandard2.0). Avoid local Release builds unless specifically required. CI builds and tests all target frameworks in Release; run a full multi-target build locally (`-p:CoreExLocalTargetFramework=`) only when changing `#if NET…` code, framework-conditional package versions in `Directory.Packages.props`, or framework-sensitive APIs. +- **Reclaiming disk**: run [`tools/clean-build-outputs.ps1`](../tools/clean-build-outputs.ps1) (`-WhatIf` to preview) to remove all `bin`/`obj` `Debug`/`Release` outputs. - **Samples**: docker-compose infrastructure + dotnet run for Database projects + Aspire AppHost. - **`CoreEx.Template` changes**: `dotnet build`/`dotnet test` do **not** exercise `src/CoreEx.Template/content/**` — that's raw `dotnet new` template content, not compiled C#. Any change under `src/CoreEx.Template/content/` (a host's `Program.cs`/`GlobalUsing.cs`/`.csproj`, a `template.json` symbol, etc.) must be validated by actually scaffolding it: run [`tools/validate-template-pack.ps1`](../tools/validate-template-pack.ps1), which packs the template, installs it, scaffolds every parameter combination it knows about into temp directories, and `dotnet build`s the ones flagged `Build = $true`. It also runs in CI (`.github/workflows/CI.yml`). If you add a new template, host, or parameter combination, add a matching scenario to the script's `$testScenarios` array — parameter-conditional bugs (an unconditional `global using`/`ProjectReference` that should have been gated behind a symbol like `has-data-provider` or `implement-servicebus`) only surface when the generated code is actually compiled, which most existing host-template scenarios don't yet do since they scaffold in isolation without their `coreex` solution siblings. - **Linting**: No separate `dotnet format`. Build is the lint pass (nullable, LangVersion=preview, TreatWarningsAsErrors in `src\Directory.Build.props`). diff --git a/Directory.Build.targets b/Directory.Build.targets new file mode 100644 index 000000000..31d0e0d75 --- /dev/null +++ b/Directory.Build.targets @@ -0,0 +1,13 @@ + + + + + $(CoreExLocalTargetFramework) + + + diff --git a/Directory.Packages.props b/Directory.Packages.props index daf9f950f..cd1c41553 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,75 +6,78 @@ - - - - - + + + + + - - + + - + - - + + - - - + + + - + + - - - + + + - - - + + + - + - - - - - + + + + + - - + + - - - - - - + + + + + + - - + + + + - - - - + + + + @@ -83,30 +86,30 @@ - + - - + + - - - - - + + + + + - - - - - + + + + + diff --git a/nuget-publish.ps1 b/nuget-publish.ps1 index 711dc5f72..a84125d47 100644 --- a/nuget-publish.ps1 +++ b/nuget-publish.ps1 @@ -2,6 +2,12 @@ .SYNOPSIS Generates nuget pacakges and publishes if server values are provided + .DESCRIPTION + Packs all supported target frameworks regardless of CoreExLocalTargetFramework. + Uses Release by default; the Visual Studio configuration does not affect this script. + Preserves existing packages except when replacing the same package filename. + Remote publishing uploads only packages produced by this invocation. + .PARAMETER PublishFolder Local folder to publish the built nuget pacakges to @@ -71,44 +77,36 @@ param( $ShouldPublishRemote = (![string]::IsNullOrEmpty($apiKey) -and ![string]::IsNullOrEmpty($NugetServer)) -if ($ShouldPublishRemote) { - Write-Warning "APIKey and NugetServer provided, All existing NuGet packages in publish folder will be deleted.`r`n" -} - -# Create publish folder if it doesn't exist; otherwise, delete existing. +# Create the shared local feed without deleting existing packages. if (!(Test-Path -Path $publishFolder)) { Write-Host "Publish folder doesn't exist - creating folder." -ForegroundColor Yellow - New-Item -ItemType directory -Path $publishFolder -} -else { - if($ShouldPublishRemote) - { - Write-Host 'Deleting existing NuGet packages from publish folder.' -ForegroundColor Yellow - $removeFiles = Join-Path $publishFolder '*.nupkg' - Remove-Item $removeFiles - } + New-Item -ItemType directory -Path $publishFolder -ErrorAction Stop | Out-Null } -# Generate NuGet packages using dotnet pack. -$successPackageRegexp = [regex] 'Successfully\screated\spackage\s''(?.*?)''.' +$stagingFolder = Join-Path ([IO.Path]::GetTempPath()) ("CoreEx-nuget-publish-" + [guid]::NewGuid().ToString('N')) +New-Item -ItemType Directory -Path $stagingFolder -ErrorAction Stop | Out-Null $packageDetailsRegexp = [regex] '(?.*?)\.(?\d*\.\d*\.\d*\.\d*.*)(? + - + diff --git a/src/CoreEx.Template/content/CoreEx.Core/_Directory.Packages.props b/src/CoreEx.Template/content/CoreEx.Core/_Directory.Packages.props index 8beb6b94c..7adc278b3 100644 --- a/src/CoreEx.Template/content/CoreEx.Core/_Directory.Packages.props +++ b/src/CoreEx.Template/content/CoreEx.Core/_Directory.Packages.props @@ -41,30 +41,30 @@ - + - + - + - + - + - - + + - - + + @@ -74,17 +74,17 @@ - + - + - + - + - + @@ -93,6 +93,6 @@ - + diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets index f6f59a9ff..144371187 100644 --- a/src/Directory.Build.targets +++ b/src/Directory.Build.targets @@ -1,4 +1,3 @@ - - - \ No newline at end of file + + diff --git a/tools/clean-build-outputs.ps1 b/tools/clean-build-outputs.ps1 new file mode 100644 index 000000000..dab29c036 --- /dev/null +++ b/tools/clean-build-outputs.ps1 @@ -0,0 +1,92 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS +Removes generated Debug and Release build outputs across the repository to recover disk space. + +.DESCRIPTION +Deletes the per-configuration directories beneath each project's bin and obj directories (for example +bin\Debug, bin\Release, obj\Debug and obj\Release). NuGet restore assets directly within obj (such as +project.assets.json) are retained, so a subsequent build does not require a restore. + +Source, NuGet caches, containers and database volumes are never touched. All outputs are regenerated on +the next build; the trade-off is losing incremental-build state, so that build is slower. + +Use -WhatIf to preview what would be removed. + +.PARAMETER Configuration +The build configurations to remove. Defaults to Debug and Release. + +.PARAMETER ShutdownBuildServers +Shuts down MSBuild/compiler build servers first to release any file locks. + +.EXAMPLE +./clean-build-outputs.ps1 +./clean-build-outputs.ps1 -WhatIf +./clean-build-outputs.ps1 -Configuration Release +./clean-build-outputs.ps1 -ShutdownBuildServers +#> + +[CmdletBinding(SupportsShouldProcess)] +param( + [string[]]$Configuration = @("Debug", "Release"), + [switch]$ShutdownBuildServers = $false +) + +$ErrorActionPreference = "Stop" + +$repoRoot = Split-Path -Parent $PSScriptRoot +$searchRoots = @("src", "tests", "samples", "gen", "tools") | + ForEach-Object { Join-Path $repoRoot $_ } | + Where-Object { Test-Path $_ } + +if ($ShutdownBuildServers -and $PSCmdlet.ShouldProcess("dotnet build servers", "Shut down")) { + dotnet build-server shutdown | Out-Null +} + +# Template content is raw dotnet new input and never contains build output; skip it explicitly. +$templateContent = Join-Path $repoRoot (Join-Path "src" (Join-Path "CoreEx.Template" "content")) + +$targets = foreach ($root in $searchRoots) { + Get-ChildItem -LiteralPath $root -Directory -Recurse -ErrorAction SilentlyContinue | + Where-Object { $_.Name -in @("bin", "obj") -and -not $_.FullName.StartsWith($templateContent, [StringComparison]::OrdinalIgnoreCase) } | + ForEach-Object { + foreach ($config in $Configuration) { + $path = Join-Path $_.FullName $config + if (Test-Path -LiteralPath $path) { Get-Item -LiteralPath $path } + } + } +} + +if (-not $targets) { + Write-Output "Nothing to clean for configuration(s): $($Configuration -join ', ')." + return +} + +$totalBytes = 0L +$removed = 0 +$failed = @() + +foreach ($target in $targets) { + $bytes = (Get-ChildItem -LiteralPath $target.FullName -Recurse -File -Force -ErrorAction SilentlyContinue | Measure-Object Length -Sum).Sum + $relative = $target.FullName.Substring($repoRoot.Length).TrimStart('\', '/') + + if ($PSCmdlet.ShouldProcess($relative, "Remove ($([math]::Round($bytes / 1MB, 1)) MB)")) { + try { + Remove-Item -LiteralPath $target.FullName -Recurse -Force + $totalBytes += $bytes + $removed++ + } + catch { + $failed += $relative + Write-Warning "Could not remove '$relative' (likely locked by a running process): $($_.Exception.Message)" + } + } +} + +if ($WhatIfPreference) { return } + +Write-Output "Removed $removed director$(if ($removed -eq 1) { 'y' } else { 'ies' }); recovered $([math]::Round($totalBytes / 1GB, 2)) GB." +if ($failed) { + Write-Warning "$($failed.Count) director$(if ($failed.Count -eq 1) { 'y' } else { 'ies' }) could not be removed. Close Visual Studio/running hosts, or rerun with -ShutdownBuildServers." + exit 1 +} diff --git a/tools/validate-template-pack.ps1 b/tools/validate-template-pack.ps1 index c5541f98c..395598b29 100644 --- a/tools/validate-template-pack.ps1 +++ b/tools/validate-template-pack.ps1 @@ -9,7 +9,7 @@ This script validates the CoreEx.Template pack by: 2. Scaffolding each template+parameter combination into a temp directory. 3. Asserting expected files are present/absent and file content matches. 4. Running dotnet build on code-emitting templates to confirm compilation. -5. Cleaning up temporary test directories (unless -SkipCleanup). +5. Cleaning successful scenarios' build outputs immediately and temporary test directories at completion (unless -SkipCleanup). Runs on Windows, Linux, and macOS via pwsh (cross-platform PowerShell). @@ -857,6 +857,9 @@ function Invoke-Assertion { } try { + # Packages must contain every target framework; ignore any local inner-loop narrowing (see Directory.Build.targets). + $env:CoreExLocalTargetFramework = $null + # Step 1: Build and pack CoreEx.Template Write-Header "Building CoreEx.Template package" if (-not $NoRebuild) { @@ -883,10 +886,6 @@ try { New-Item -ItemType Directory -Path $localFeedPath -Force | Out-Null Write-Header "Building local NuGet feed" - # Build the solution so binaries and assets exist before packing individually. - dotnet build (Join-Path $repoRoot "CoreEx.sln") -c Release --nologo -m:1 - if ($LASTEXITCODE -ne 0) { throw "Failed to build solution" } - $srcRoot = Join-Path $repoRoot "src" # Only look one level deep — each package lives in its own direct subdirectory. # Recursing would pick up template content files inside CoreEx.Template/content/. @@ -895,6 +894,12 @@ try { ForEach-Object { Get-ChildItem -Path $_.FullName -Filter "*.csproj" | Select-Object -First 1 } | Where-Object { $_ -ne $null } + # Build only the packable libraries (not samples/tests) so binaries and assets exist before packing individually. + foreach ($proj in $libraryProjects) { + dotnet build $proj.FullName -c Release --nologo -m:1 -v q + if ($LASTEXITCODE -ne 0) { throw "Failed to build $($proj.Directory.Name)" } + } + foreach ($proj in $libraryProjects) { Write-Verbose "Packing $($proj.Directory.Name)" $packOutput = dotnet pack $proj.FullName -c Release --nologo --no-build --no-restore -o $localFeedPath 2>&1 @@ -1072,6 +1077,14 @@ try { if ($scenarioFailures.Count -eq 0) { Write-Pass "$($scenario.Name) PASSED" + if (-not $SkipCleanup) { + $buildOutputDirs = Get-ChildItem -LiteralPath $testDir -Directory -Recurse | + Where-Object { $_.Name -in @("bin", "obj") } | + Sort-Object { $_.FullName.Length } -Descending + foreach ($buildOutputDir in $buildOutputDirs) { + Remove-Item -LiteralPath $buildOutputDir.FullName -Recurse -Force + } + } } else { $scenarioFailures | ForEach-Object { Write-Verbose " Error: $_" } $failedScenarios += $scenario.Name From 90a2f217a662894b8ebe1e6b2116b064f1bfdb78 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Thu, 15 Oct 2026 10:42:14 -0700 Subject: [PATCH 09/25] Add repeatable product search relevance evaluation Capture reviewed catalog judgments and a real-model baseline, isolate evaluation data setup, and add manual/monthly reporting with retained artifacts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .../workflows/product-search-relevance.yml | 114 ++ .../product-search-relevance-baseline.json | 1226 +++++++++++++++++ samples/docs/search.md | 55 +- .../ProductSearchRelevanceSetupTests.cs | 33 + tools/product-search-relevance.dataset.json | 392 ++++++ tools/product-search-relevance.ps1 | 439 ++++++ tools/product-search-relevance.seed.yaml | 45 + tools/test-product-search-relevance.ps1 | 140 ++ 8 files changed, 2440 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/product-search-relevance.yml create mode 100644 samples/docs/product-search-relevance-baseline.json create mode 100644 samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs create mode 100644 tools/product-search-relevance.dataset.json create mode 100644 tools/product-search-relevance.ps1 create mode 100644 tools/product-search-relevance.seed.yaml create mode 100644 tools/test-product-search-relevance.ps1 diff --git a/.github/workflows/product-search-relevance.yml b/.github/workflows/product-search-relevance.yml new file mode 100644 index 000000000..cae6b04a7 --- /dev/null +++ b/.github/workflows/product-search-relevance.yml @@ -0,0 +1,114 @@ +name: Product search relevance + +on: + schedule: + - cron: '0 14 1 * *' + workflow_dispatch: + +concurrency: + group: product-search-relevance-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + evaluate: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + PRODUCTS_DATABASE_CONNECTION_STRING: Server=127.0.0.1;Database=contoso_search_eval;Username=postgres;Password=yourStrong#!Password + Aspire__Npgsql__ConnectionString: Server=127.0.0.1;Database=contoso_search_eval;Username=postgres;Password=yourStrong#!Password + Aspire__StackExchange__Redis__ConnectionString: localhost:6379 + Ollama__BaseAddress: http://localhost:11434/ + ASPNETCORE_ENVIRONMENT: Development + + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.x + + - name: Cache NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-search-relevance-${{ hashFiles('**/*.csproj', '**/Directory.Packages.props') }} + restore-keys: nuget-${{ runner.os }}- + + - name: Start isolated evaluation services + run: docker compose --profile search up -d db-postgres redis-cache ollama + + - name: Wait for PostgreSQL and create the isolated evaluation database + run: | + for attempt in $(seq 1 60); do + if docker compose exec -T db-postgres pg_isready -U postgres; then break; fi + sleep 2 + done + docker compose exec -T db-postgres createdb -U postgres contoso_search_eval + + - name: Install pinned embedding model + timeout-minutes: 10 + run: | + docker compose exec -T ollama ollama pull nomic-embed-text:v1.5 + docker compose exec -T ollama ollama list + + - name: Restore evaluation projects + run: | + dotnet restore samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --verbosity minimal + dotnet restore samples/tests/Contoso.Products.Test.Api/Contoso.Products.Test.Api.csproj --verbosity minimal + + - name: Build evaluation projects + run: | + dotnet build samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-restore --configuration Release --framework net10.0 + dotnet build samples/tests/Contoso.Products.Test.Api/Contoso.Products.Test.Api.csproj --no-restore --configuration Release --framework net10.0 + + - name: Apply schema and reference data to the isolated database + run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- all --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" + + - name: Reset only the isolated Products schema + run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- reset --accept-prompts --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" + + - name: Reload reference data + run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- data --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" + + - name: Load the dedicated evaluation catalog + run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- execute "$GITHUB_WORKSPACE/tools/product-search-relevance.seed.yaml" --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" + + - name: Prepare and verify the real-model index + run: dotnet test samples/tests/Contoso.Products.Test.Api/Contoso.Products.Test.Api.csproj --no-build --configuration Release --framework net10.0 --filter "FullyQualifiedName~ProductSearchRelevanceSetupTests" --logger "console;verbosity=normal" + + - name: Evaluate the live Products REST API + shell: bash + env: + ASPNETCORE_URLS: http://localhost:5207 + run: | + mkdir -p artifacts + dotnet run --project samples/src/Contoso.Products.Api/Contoso.Products.Api.csproj --no-build --configuration Release --framework net10.0 --no-launch-profile > artifacts/products-api.log 2>&1 & + api_pid=$! + trap 'kill "$api_pid" 2>/dev/null || true' EXIT + for attempt in $(seq 1 60); do + if curl --fail --silent http://localhost:5207/api/products/%24query > /dev/null; then break; fi + if ! kill -0 "$api_pid" 2>/dev/null; then + cat artifacts/products-api.log + exit 1 + fi + sleep 2 + done + curl --fail --silent http://localhost:5207/api/products/%24query > /dev/null + pwsh -NoProfile -File tools/product-search-relevance.ps1 -BaseAddress http://localhost:5207/ -OllamaAddress http://localhost:11434/ -OutputPath artifacts/product-search-relevance-report.json + + - name: Upload evaluation report and diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: product-search-relevance-${{ github.run_number }} + path: artifacts/ + if-no-files-found: warn + retention-days: 90 + + - name: Stop isolated evaluation services + if: always() + run: docker compose --profile search down diff --git a/samples/docs/product-search-relevance-baseline.json b/samples/docs/product-search-relevance-baseline.json new file mode 100644 index 000000000..04b281a42 --- /dev/null +++ b/samples/docs/product-search-relevance-baseline.json @@ -0,0 +1,1226 @@ +{ + "evaluationVersion": "product-search-relevance-runner-v1", + "completedAtUtc": "2026-10-15T17:01:00.2457191Z", + "status": "completed", + "datasetVersion": "product-search-relevance-v1", + "model": "nomic-embed-text:v1.5", + "modelDigest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f", + "profile": "ollama:nomic-embed-text:v1.5:768:retrieval-prefix-v1", + "dimensions": 768, + "documentRecipe": "product-document-v2", + "documentInstruction": "search_document: ", + "queryInstruction": "search_query: ", + "datasetSha256": "E118A3F4B3B851A535FD3EBDBE8BB5C1C1F7B3F7EE2F21AD3D737B85E9D16C9B", + "implementationIdentity": { + "clientSha256": "7A4EF9679E9621D1271F524277EFD79C0D910BFCD2E3F5735FC401DA3C7BEF77", + "documentBuilderSha256": "F237F42D07A6269C8B686A11949FE49809756286CBCDA09F9F3B4CD51BD9DFED" + }, + "catalogIdentity": { + "seedFile": "tools/product-search-relevance.seed.yaml", + "seedFileSha256": "F18614A4520837867BF9187D812ECC5BAF0559D0E652FC1175CDBECFC74ADF44", + "eligibleCatalogCount": 29, + "reviewedCatalogSha256": "EE8FBC8AADA96142D82F5C97B3B7509AD1E21B67DB26A69C941DB5DEDF9EA3C5", + "liveApiCatalogSha256": "EE8FBC8AADA96142D82F5C97B3B7509AD1E21B67DB26A69C941DB5DEDF9EA3C5", + "eligibility": "Default Products API catalog: exclude inactive and logically deleted products; non-stocked service rows remain eligible." + }, + "metrics": { + "relevanceThresholdForPrecisionAt5": 2, + "ndcgAt5Gain": "2^grade - 1; logarithmic rank discount; judged grades 0-3", + "precisionAt5": "Relevant means grade >= 2; denominator is always 5.", + "noRelevantQueries": "Reported separately and excluded from relevance-metric means.", + "exactSkuTop1": "Reported only for exact-SKU queries.", + "constraints": "Returned price-filtered results are validated against the declared hard price condition; no threshold is imposed on relevance scores." + }, + "splits": { + "tuning": { + "queryCount": 18, + "relevantQueryCount": 17, + "noRelevantQueryCount": 1, + "noRelevantQueryIds": [ + "tune-no-current-bike-under-1000" + ], + "outOfCatalogQueryCount": 0, + "meanNdcgAt5": 0.8377447396384872, + "meanPrecisionAt5": 0.37647058823529417, + "exactSkuQueryCount": 3, + "exactSkuTop1Accuracy": 0.6666666666666666, + "constrainedQueryCount": 3, + "constrainedResultCount": 15, + "constraintViolationCount": 0, + "constraintViolationRate": 0.0 + }, + "held-out": { + "queryCount": 6, + "relevantQueryCount": 3, + "noRelevantQueryCount": 3, + "noRelevantQueryIds": [ + "holdout-no-electric-bike-charger", + "holdout-no-night-riding-lights", + "holdout-no-city-bike-child-seat" + ], + "outOfCatalogQueryCount": 3, + "meanNdcgAt5": 0.8769765845238191, + "meanPrecisionAt5": 0.26666666666666666, + "exactSkuQueryCount": 1, + "exactSkuTop1Accuracy": 0.0, + "constrainedQueryCount": 1, + "constrainedResultCount": 5, + "constraintViolationCount": 0, + "constraintViolationRate": 0.0 + } + }, + "queries": [ + { + "id": "tune-exact-yeti-asr", + "split": "tuning", + "kind": "exact-sku", + "query": "YETI-ASR-C2-2025", + "intent": "Exact SKU lookup for the Yeti ASR C2 2025.", + "expectedTop1Sku": "YETI-ASR-C2-2025", + "actualTop1Sku": "YETI-ASR-C2-2025", + "exactSkuTop1": true, + "relevantJudgedCount": 1, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "YETI-ASR-C2-2025", + "text": "Yeti ASR C2", + "price": 5800.0, + "grade": 3 + }, + { + "sku": "ONEUP-V3-DROPPER-180", + "text": "OneUp Dropper Post V3 180mm", + "price": 269.0, + "grade": 0 + }, + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "text": "SRAM X0 Transmission Rear Derailleur", + "price": 400.0, + "grade": 0 + }, + { + "sku": "SPEC-STATUS-140", + "text": "Specialized Status 140", + "price": 2499.0, + "grade": 0 + }, + { + "sku": "GIANT-ANTHEM-ADV29-1", + "text": "Giant Anthem Advanced 29 1", + "price": 7800.0, + "grade": 0 + } + ] + }, + { + "id": "tune-exact-shimano-derailleur", + "split": "tuning", + "kind": "exact-sku", + "query": "SHIMANO-XT-M8100-RD", + "intent": "Exact SKU lookup for the Shimano XT rear derailleur.", + "expectedTop1Sku": "SHIMANO-XT-M8100-RD", + "actualTop1Sku": "MUCOFF-NANO-CLEANER", + "exactSkuTop1": false, + "relevantJudgedCount": 1, + "noRelevant": false, + "ndcgAt5": 0.0, + "precisionAt5": 0.0, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "MUCOFF-NANO-CLEANER", + "text": "Muc-Off Nano Tech Bike Cleaner", + "price": 16.99, + "grade": 0 + }, + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "text": "SRAM X0 Transmission Rear Derailleur", + "price": 400.0, + "grade": 0 + }, + { + "sku": "SRAM-XX1-EAGLE-CASSETTE", + "text": "SRAM XX1 Eagle Cassette", + "price": 449.0, + "grade": 0 + }, + { + "sku": "MUCOFF-TUBELESS-SEALANT", + "text": "Muc-Off MTB Tubeless Sealant", + "price": 8.0, + "grade": 0 + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 0 + } + ] + }, + { + "id": "tune-exact-mucoff-cleaner", + "split": "tuning", + "kind": "exact-sku", + "query": "MUCOFF-NANO-CLEANER", + "intent": "Exact SKU lookup for the Nano Tech bicycle cleaner.", + "expectedTop1Sku": "MUCOFF-NANO-CLEANER", + "actualTop1Sku": "MUCOFF-NANO-CLEANER", + "exactSkuTop1": true, + "relevantJudgedCount": 1, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "MUCOFF-NANO-CLEANER", + "text": "Muc-Off Nano Tech Bike Cleaner", + "price": 16.99, + "grade": 3 + }, + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "text": "SRAM X0 Transmission Rear Derailleur", + "price": 400.0, + "grade": 0 + }, + { + "sku": "SRAM-XX1-EAGLE-CASSETTE", + "text": "SRAM XX1 Eagle Cassette", + "price": 449.0, + "grade": 0 + }, + { + "sku": "MUCOFF-TUBELESS-SEALANT", + "text": "Muc-Off MTB Tubeless Sealant", + "price": 8.0, + "grade": 0 + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 0 + } + ] + }, + { + "id": "tune-paraphrase-muddy-bike-cleaner", + "split": "tuning", + "kind": "paraphrase", + "query": "something to clean mud off my bike", + "intent": "A user describes the cleaning task rather than the product name.", + "expectedTop1Sku": null, + "actualTop1Sku": "MUCOFF-NANO-CLEANER", + "exactSkuTop1": null, + "relevantJudgedCount": 2, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.4, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "MUCOFF-NANO-CLEANER", + "text": "Muc-Off Nano Tech Bike Cleaner", + "price": 16.99, + "grade": 3 + }, + { + "sku": "MUCOFF-DRIVETRAIN-CLEANER", + "text": "Muc-Off Drivetrain Cleaner", + "price": 25.0, + "grade": 2 + }, + { + "sku": "MUCOFF-TUBELESS-SEALANT", + "text": "Muc-Off MTB Tubeless Sealant", + "price": 8.0, + "grade": 0 + }, + { + "sku": "ONEUP-CARBON-HANDLEBAR", + "text": "OneUp Carbon Handlebar", + "price": 159.0, + "grade": 0 + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 0 + } + ] + }, + { + "id": "tune-terminology-changing-gears", + "split": "tuning", + "kind": "terminology", + "query": "a component for changing gears", + "intent": "Broad functional terminology for drivetrain components.", + "expectedTop1Sku": null, + "actualTop1Sku": "SHIMANO-XT-M8100-RD", + "exactSkuTop1": null, + "relevantJudgedCount": 6, + "noRelevant": false, + "ndcgAt5": 0.8875825932815664, + "precisionAt5": 0.8, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "SHIMANO-XT-M8100-RD", + "text": "Shimano Deore XT M8100 Rear Derailleur", + "price": 129.0, + "grade": 3 + }, + { + "sku": "SRAM-GX-EAGLE-RD", + "text": "SRAM GX Eagle Rear Derailleur", + "price": 135.0, + "grade": 3 + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 0 + }, + { + "sku": "SHIMANO-SLX-M7100-CASSETTE", + "text": "Shimano SLX M7100 12-speed Cassette", + "price": 99.0, + "grade": 2 + }, + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "text": "SRAM X0 Transmission Rear Derailleur", + "price": 400.0, + "grade": 3 + } + ] + }, + { + "id": "tune-paraphrase-climbing-gears", + "split": "tuning", + "kind": "paraphrase", + "query": "a wide range of low gears for climbing", + "intent": "A climbing use case described without naming a cassette.", + "expectedTop1Sku": null, + "actualTop1Sku": "SRAM-XX1-EAGLE-CASSETTE", + "exactSkuTop1": null, + "relevantJudgedCount": 3, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.4, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "SRAM-XX1-EAGLE-CASSETTE", + "text": "SRAM XX1 Eagle Cassette", + "price": 449.0, + "grade": 3 + }, + { + "sku": "SHIMANO-SLX-M7100-CASSETTE", + "text": "Shimano SLX M7100 12-speed Cassette", + "price": 99.0, + "grade": 3 + }, + { + "sku": "SHIMANO-XT-M8100-RD", + "text": "Shimano Deore XT M8100 Rear Derailleur", + "price": 129.0, + "grade": 1 + }, + { + "sku": "GIANT-TRANCE-X-ADVPRO29", + "text": "Giant Trance X Advanced Pro 29", + "price": 6200.0, + "grade": 0 + }, + { + "sku": "ONEUP-V3-DROPPER-180", + "text": "OneUp Dropper Post V3 180mm", + "price": 269.0, + "grade": 0 + } + ] + }, + { + "id": "tune-paraphrase-tubeless-punctures", + "split": "tuning", + "kind": "paraphrase", + "query": "liquid to seal small punctures in tubeless tires", + "intent": "Describes the purpose of a tubeless tire sealant without its product name.", + "expectedTop1Sku": null, + "actualTop1Sku": "MUCOFF-TUBELESS-SEALANT", + "exactSkuTop1": null, + "relevantJudgedCount": 1, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "MUCOFF-TUBELESS-SEALANT", + "text": "Muc-Off MTB Tubeless Sealant", + "price": 8.0, + "grade": 3 + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 0 + }, + { + "sku": "MUCOFF-NANO-CLEANER", + "text": "Muc-Off Nano Tech Bike Cleaner", + "price": 16.99, + "grade": 0 + }, + { + "sku": "MUCOFF-DRIVETRAIN-CLEANER", + "text": "Muc-Off Drivetrain Cleaner", + "price": 25.0, + "grade": 0 + }, + { + "sku": "GIANT-ANTHEM-ADV29-1", + "text": "Giant Anthem Advanced 29 1", + "price": 7800.0, + "grade": 0 + } + ] + }, + { + "id": "tune-paraphrase-lower-saddle", + "split": "tuning", + "kind": "paraphrase", + "query": "a seat post that lowers the saddle for descents", + "intent": "Describes dropper-post behavior in everyday language.", + "expectedTop1Sku": null, + "actualTop1Sku": "ONEUP-V3-DROPPER-180", + "exactSkuTop1": null, + "relevantJudgedCount": 2, + "noRelevant": false, + "ndcgAt5": 0.9173194127129571, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "ONEUP-V3-DROPPER-180", + "text": "OneUp Dropper Post V3 180mm", + "price": 269.0, + "grade": 3 + }, + { + "sku": "SPEC-STUMPJUMPER-15-COMP", + "text": "Specialized Stumpjumper 15 Comp", + "price": 5500.0, + "grade": 0 + }, + { + "sku": "GIANT-TRANCE-X-ADVPRO29", + "text": "Giant Trance X Advanced Pro 29", + "price": 6200.0, + "grade": 0 + }, + { + "sku": "ONEUP-CARBON-HANDLEBAR", + "text": "OneUp Carbon Handlebar", + "price": 159.0, + "grade": 0 + }, + { + "sku": "CANYON-NEURON-CF7", + "text": "Canyon Neuron CF 7", + "price": 3499.0, + "grade": 0 + } + ] + }, + { + "id": "tune-terminology-reduce-hand-vibration", + "split": "tuning", + "kind": "terminology", + "query": "handlebars that reduce hand fatigue on rough trails", + "intent": "Matches rider language to a product's vibration and fatigue benefit.", + "expectedTop1Sku": null, + "actualTop1Sku": "ONEUP-CARBON-HANDLEBAR", + "exactSkuTop1": null, + "relevantJudgedCount": 1, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "ONEUP-CARBON-HANDLEBAR", + "text": "OneUp Carbon Handlebar", + "price": 159.0, + "grade": 3 + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 0 + }, + { + "sku": "SPEC-STATUS-140", + "text": "Specialized Status 140", + "price": 2499.0, + "grade": 0 + }, + { + "sku": "CANYON-NEURON-CF7", + "text": "Canyon Neuron CF 7", + "price": 3499.0, + "grade": 0 + }, + { + "sku": "YETI-SB120-C2", + "text": "Yeti SB120 C2", + "price": 6200.0, + "grade": 0 + } + ] + }, + { + "id": "tune-paraphrase-trailside-repair", + "split": "tuning", + "kind": "paraphrase", + "query": "a compact tool for fixing my bike on the trail", + "intent": "A use-case paraphrase for a portable repair multi-tool.", + "expectedTop1Sku": null, + "actualTop1Sku": "ONEUP-EDC-TOOL-V2", + "exactSkuTop1": null, + "relevantJudgedCount": 1, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 3 + }, + { + "sku": "GIANT-TRANCE-X-ADVPRO29", + "text": "Giant Trance X Advanced Pro 29", + "price": 6200.0, + "grade": 0 + }, + { + "sku": "SPEC-STUMPJUMPER-15-COMP", + "text": "Specialized Stumpjumper 15 Comp", + "price": 5500.0, + "grade": 0 + }, + { + "sku": "SHIMANO-XTR-M9100-CRANK", + "text": "Shimano XTR M9100 Crankset", + "price": 499.0, + "grade": 0 + }, + { + "sku": "SHIMANO-XT-M8100-RD", + "text": "Shimano Deore XT M8100 Rear Derailleur", + "price": 129.0, + "grade": 0 + } + ] + }, + { + "id": "tune-no-current-bike-under-1000", + "split": "tuning", + "kind": "price-constraint", + "query": "a mountain bike under $1000", + "intent": "No currently eligible seeded mountain bike is priced below $1000; the deleted Giant Talon is not eligible.", + "expectedTop1Sku": null, + "actualTop1Sku": "ONEUP-V3-DROPPER-180", + "exactSkuTop1": null, + "relevantJudgedCount": 0, + "noRelevant": true, + "ndcgAt5": null, + "precisionAt5": null, + "eligibleCandidateCount": 13, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "ONEUP-V3-DROPPER-180", + "text": "OneUp Dropper Post V3 180mm", + "price": 269.0, + "grade": 0 + }, + { + "sku": "SRAM-XX1-EAGLE-CASSETTE", + "text": "SRAM XX1 Eagle Cassette", + "price": 449.0, + "grade": 0 + }, + { + "sku": "SHIMANO-XT-M8100-RD", + "text": "Shimano Deore XT M8100 Rear Derailleur", + "price": 129.0, + "grade": 0 + }, + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "text": "SRAM X0 Transmission Rear Derailleur", + "price": 400.0, + "grade": 0 + }, + { + "sku": "SRAM-GX-EAGLE-RD", + "text": "SRAM GX Eagle Rear Derailleur", + "price": 135.0, + "grade": 0 + } + ] + }, + { + "id": "tune-terminology-cross-country-race", + "split": "tuning", + "kind": "terminology", + "query": "a fast cross-country bike for racing", + "intent": "Cross-country race terminology matched to seeded bike descriptions.", + "expectedTop1Sku": null, + "actualTop1Sku": "SPEC-CHISEL-COMP", + "exactSkuTop1": null, + "relevantJudgedCount": 4, + "noRelevant": false, + "ndcgAt5": 0.8592395637645646, + "precisionAt5": 0.8, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "SPEC-CHISEL-COMP", + "text": "Specialized Chisel Comp", + "price": 3400.0, + "grade": 2 + }, + { + "sku": "GIANT-ANTHEM-ADV29-1", + "text": "Giant Anthem Advanced 29 1", + "price": 7800.0, + "grade": 3 + }, + { + "sku": "YETI-ASR-C2-2025", + "text": "Yeti ASR C2", + "price": 5800.0, + "grade": 3 + }, + { + "sku": "CANYON-LUXTRAIL-CF6", + "text": "Canyon Lux Trail CF 6", + "price": 3899.0, + "grade": 2 + }, + { + "sku": "SPEC-ENDURO-PRO", + "text": "Specialized Enduro Pro", + "price": 9500.0, + "grade": 0 + } + ] + }, + { + "id": "tune-broad-steep-bike-park", + "split": "tuning", + "kind": "broad", + "query": "a bike for steep rough descents and bike-park riding", + "intent": "Broad descending intent with several suitable bike categories.", + "expectedTop1Sku": null, + "actualTop1Sku": "CANYON-TORQUE-MULLET-CF8", + "exactSkuTop1": null, + "relevantJudgedCount": 6, + "noRelevant": false, + "ndcgAt5": 0.8688881839864373, + "precisionAt5": 1.0, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "CANYON-TORQUE-MULLET-CF8", + "text": "Canyon Torque Mullet CF 8", + "price": 3599.0, + "grade": 2 + }, + { + "sku": "CANYON-SENDER-CFR", + "text": "Canyon Sender CFR", + "price": 7799.0, + "grade": 3 + }, + { + "sku": "YETI-SB165-C3", + "text": "Yeti SB165 C3", + "price": 7200.0, + "grade": 3 + }, + { + "sku": "SPEC-ENDURO-PRO", + "text": "Specialized Enduro Pro", + "price": 9500.0, + "grade": 3 + }, + { + "sku": "YETI-SB140-LR-C2", + "text": "Yeti SB140 LR C2", + "price": 6600.0, + "grade": 2 + } + ] + }, + { + "id": "tune-paraphrase-drivetrain-degreaser", + "split": "tuning", + "kind": "paraphrase", + "query": "degreaser for a greasy chain and cassette", + "intent": "Names drivetrain parts and soil rather than the cleaner product name.", + "expectedTop1Sku": null, + "actualTop1Sku": "MUCOFF-DRIVETRAIN-CLEANER", + "exactSkuTop1": null, + "relevantJudgedCount": 2, + "noRelevant": false, + "ndcgAt5": 0.9173194127129571, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "MUCOFF-DRIVETRAIN-CLEANER", + "text": "Muc-Off Drivetrain Cleaner", + "price": 25.0, + "grade": 3 + }, + { + "sku": "SHIMANO-XT-M8100-RD", + "text": "Shimano Deore XT M8100 Rear Derailleur", + "price": 129.0, + "grade": 0 + }, + { + "sku": "SRAM-XX1-EAGLE-CASSETTE", + "text": "SRAM XX1 Eagle Cassette", + "price": 449.0, + "grade": 0 + }, + { + "sku": "SRAM-GX-EAGLE-RD", + "text": "SRAM GX Eagle Rear Derailleur", + "price": 135.0, + "grade": 0 + }, + { + "sku": "SHIMANO-SLX-M7100-CASSETTE", + "text": "Shimano SLX M7100 12-speed Cassette", + "price": 99.0, + "grade": 0 + } + ] + }, + { + "id": "tune-price-derailleur-under-150", + "split": "tuning", + "kind": "price-constraint", + "query": "a rear derailleur for a 12-speed mountain bike under $150", + "intent": "Hard price ceiling and product type; two seeded derailleurs satisfy both.", + "expectedTop1Sku": null, + "actualTop1Sku": "SHIMANO-XT-M8100-RD", + "exactSkuTop1": null, + "relevantJudgedCount": 2, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.4, + "eligibleCandidateCount": 8, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "SHIMANO-XT-M8100-RD", + "text": "Shimano Deore XT M8100 Rear Derailleur", + "price": 129.0, + "grade": 3 + }, + { + "sku": "SRAM-GX-EAGLE-RD", + "text": "SRAM GX Eagle Rear Derailleur", + "price": 135.0, + "grade": 3 + }, + { + "sku": "SHIMANO-SLX-M7100-CASSETTE", + "text": "Shimano SLX M7100 12-speed Cassette", + "price": 99.0, + "grade": 0 + }, + { + "sku": "MUCOFF-DRIVETRAIN-CLEANER", + "text": "Muc-Off Drivetrain Cleaner", + "price": 25.0, + "grade": 0 + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 0 + } + ] + }, + { + "id": "tune-price-trail-bike-under-5000", + "split": "tuning", + "kind": "price-constraint", + "query": "a trail bike for technical descents under $5000", + "intent": "Filters to affordable seeded bikes while preferring trail models suitable for technical descents.", + "expectedTop1Sku": null, + "actualTop1Sku": "CANYON-NEURON-CF7", + "exactSkuTop1": null, + "relevantJudgedCount": 4, + "noRelevant": false, + "ndcgAt5": 0.883484406764202, + "precisionAt5": 0.6, + "eligibleCandidateCount": 19, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "CANYON-NEURON-CF7", + "text": "Canyon Neuron CF 7", + "price": 3499.0, + "grade": 3 + }, + { + "sku": "SPEC-STATUS-140", + "text": "Specialized Status 140", + "price": 2499.0, + "grade": 0 + }, + { + "sku": "CANYON-SPECTRAL-CF8", + "text": "Canyon Spectral CF 8", + "price": 4999.0, + "grade": 3 + }, + { + "sku": "CANYON-LUXTRAIL-CF6", + "text": "Canyon Lux Trail CF 6", + "price": 3899.0, + "grade": 2 + }, + { + "sku": "SPEC-CHISEL-COMP", + "text": "Specialized Chisel Comp", + "price": 3400.0, + "grade": 0 + } + ] + }, + { + "id": "tune-broad-everyday-singletrack", + "split": "tuning", + "kind": "broad", + "query": "a versatile mountain bike for after-work singletrack", + "intent": "Ambiguous everyday trail request without a specific price, discipline or component.", + "expectedTop1Sku": null, + "actualTop1Sku": "CANYON-SPECTRAL-CF8", + "exactSkuTop1": null, + "relevantJudgedCount": 7, + "noRelevant": false, + "ndcgAt5": 0.6534081967245269, + "precisionAt5": 0.4, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "CANYON-SPECTRAL-CF8", + "text": "Canyon Spectral CF 8", + "price": 4999.0, + "grade": 2 + }, + { + "sku": "YETI-SB120-C2", + "text": "Yeti SB120 C2", + "price": 6200.0, + "grade": 2 + }, + { + "sku": "SPEC-CHISEL-COMP", + "text": "Specialized Chisel Comp", + "price": 3400.0, + "grade": 1 + }, + { + "sku": "CANYON-LUXTRAIL-CF6", + "text": "Canyon Lux Trail CF 6", + "price": 3899.0, + "grade": 0 + }, + { + "sku": "YETI-ASR-C2-2025", + "text": "Yeti ASR C2", + "price": 5800.0, + "grade": 1 + } + ] + }, + { + "id": "tune-broad-weekend-ride-kit", + "split": "tuning", + "kind": "broad", + "query": "useful gear for a weekend ride on rough trails", + "intent": "Intentionally broad request that could mean a trail accessory or repair item.", + "expectedTop1Sku": null, + "actualTop1Sku": "CANYON-NEURON-CF7", + "exactSkuTop1": null, + "relevantJudgedCount": 4, + "noRelevant": false, + "ndcgAt5": 0.25441880390707017, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "CANYON-NEURON-CF7", + "text": "Canyon Neuron CF 7", + "price": 3499.0, + "grade": 0 + }, + { + "sku": "CANYON-SPECTRAL-CF8", + "text": "Canyon Spectral CF 8", + "price": 4999.0, + "grade": 0 + }, + { + "sku": "SHIMANO-XT-M8100-RD", + "text": "Shimano Deore XT M8100 Rear Derailleur", + "price": 129.0, + "grade": 0 + }, + { + "sku": "YETI-SB140-LR-C2", + "text": "Yeti SB140 LR C2", + "price": 6600.0, + "grade": 0 + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 2 + } + ] + }, + { + "id": "holdout-exact-sram-x0-derailleur", + "split": "held-out", + "kind": "exact-sku", + "query": "SRAM-X0-TRANSMISSION-RD", + "intent": "Held-out exact SKU lookup for the SRAM X0 electronic rear derailleur.", + "expectedTop1Sku": "SRAM-X0-TRANSMISSION-RD", + "actualTop1Sku": "MUCOFF-NANO-CLEANER", + "exactSkuTop1": false, + "relevantJudgedCount": 1, + "noRelevant": false, + "ndcgAt5": 0.6309297535714574, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "MUCOFF-NANO-CLEANER", + "text": "Muc-Off Nano Tech Bike Cleaner", + "price": 16.99, + "grade": 0 + }, + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "text": "SRAM X0 Transmission Rear Derailleur", + "price": 400.0, + "grade": 3 + }, + { + "sku": "SRAM-XX1-EAGLE-CASSETTE", + "text": "SRAM XX1 Eagle Cassette", + "price": 449.0, + "grade": 0 + }, + { + "sku": "MUCOFF-TUBELESS-SEALANT", + "text": "Muc-Off MTB Tubeless Sealant", + "price": 8.0, + "grade": 0 + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59.0, + "grade": 0 + } + ] + }, + { + "id": "holdout-terminology-workshop-labour", + "split": "held-out", + "kind": "terminology", + "query": "an hourly workshop charge for a bicycle tune-up", + "intent": "Held-out service request for workshop labour rather than a stocked physical product.", + "expectedTop1Sku": null, + "actualTop1Sku": "LABOR", + "exactSkuTop1": null, + "relevantJudgedCount": 1, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.2, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "LABOR", + "text": "Labor", + "price": 80.0, + "grade": 3 + }, + { + "sku": "SPEC-STATUS-140", + "text": "Specialized Status 140", + "price": 2499.0, + "grade": 0 + }, + { + "sku": "GIANT-ANTHEM-ADV29-1", + "text": "Giant Anthem Advanced 29 1", + "price": 7800.0, + "grade": 0 + }, + { + "sku": "MUCOFF-DRIVETRAIN-CLEANER", + "text": "Muc-Off Drivetrain Cleaner", + "price": 25.0, + "grade": 0 + }, + { + "sku": "GIANT-TRANCE-X-ADVPRO29", + "text": "Giant Trance X Advanced Pro 29", + "price": 6200.0, + "grade": 0 + } + ] + }, + { + "id": "holdout-price-carbon-bike-under-4000", + "split": "held-out", + "kind": "price-constraint", + "query": "a carbon mountain bike for long rides under $4000", + "intent": "Held-out structured price ceiling with a long-ride carbon-bike intent.", + "expectedTop1Sku": null, + "actualTop1Sku": "CANYON-NEURON-CF7", + "exactSkuTop1": null, + "relevantJudgedCount": 3, + "noRelevant": false, + "ndcgAt5": 1.0, + "precisionAt5": 0.4, + "eligibleCandidateCount": 18, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "CANYON-NEURON-CF7", + "text": "Canyon Neuron CF 7", + "price": 3499.0, + "grade": 3 + }, + { + "sku": "CANYON-LUXTRAIL-CF6", + "text": "Canyon Lux Trail CF 6", + "price": 3899.0, + "grade": 3 + }, + { + "sku": "CANYON-TORQUE-MULLET-CF8", + "text": "Canyon Torque Mullet CF 8", + "price": 3599.0, + "grade": 1 + }, + { + "sku": "ONEUP-CARBON-HANDLEBAR", + "text": "OneUp Carbon Handlebar", + "price": 159.0, + "grade": 0 + }, + { + "sku": "SPEC-STATUS-140", + "text": "Specialized Status 140", + "price": 2499.0, + "grade": 0 + } + ] + }, + { + "id": "holdout-no-electric-bike-charger", + "split": "held-out", + "kind": "out-of-catalog", + "query": "a charger for the battery on an electric mountain bike", + "intent": "The catalog includes an electric mountain bike but no battery charger accessory.", + "expectedTop1Sku": null, + "actualTop1Sku": "YETI-MTE-C2", + "exactSkuTop1": null, + "relevantJudgedCount": 0, + "noRelevant": true, + "ndcgAt5": null, + "precisionAt5": null, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "YETI-MTE-C2", + "text": "Yeti MTe C2 eMTB", + "price": 9850.0, + "grade": 0 + }, + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "text": "SRAM X0 Transmission Rear Derailleur", + "price": 400.0, + "grade": 0 + }, + { + "sku": "SHIMANO-XTR-M9100-CRANK", + "text": "Shimano XTR M9100 Crankset", + "price": 499.0, + "grade": 0 + }, + { + "sku": "ONEUP-V3-DROPPER-180", + "text": "OneUp Dropper Post V3 180mm", + "price": 269.0, + "grade": 0 + }, + { + "sku": "SHIMANO-XT-M8100-RD", + "text": "Shimano Deore XT M8100 Rear Derailleur", + "price": 129.0, + "grade": 0 + } + ] + }, + { + "id": "holdout-no-night-riding-lights", + "split": "held-out", + "kind": "out-of-catalog", + "query": "USB rechargeable lights for night riding", + "intent": "The reviewed eligible catalog contains no lights or night-riding electronics.", + "expectedTop1Sku": null, + "actualTop1Sku": "SRAM-X0-TRANSMISSION-RD", + "exactSkuTop1": null, + "relevantJudgedCount": 0, + "noRelevant": true, + "ndcgAt5": null, + "precisionAt5": null, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "text": "SRAM X0 Transmission Rear Derailleur", + "price": 400.0, + "grade": 0 + }, + { + "sku": "SPEC-STATUS-140", + "text": "Specialized Status 140", + "price": 2499.0, + "grade": 0 + }, + { + "sku": "YETI-MTE-C2", + "text": "Yeti MTe C2 eMTB", + "price": 9850.0, + "grade": 0 + }, + { + "sku": "MUCOFF-TUBELESS-SEALANT", + "text": "Muc-Off MTB Tubeless Sealant", + "price": 8.0, + "grade": 0 + }, + { + "sku": "CANYON-LUXTRAIL-CF6", + "text": "Canyon Lux Trail CF 6", + "price": 3899.0, + "grade": 0 + } + ] + }, + { + "id": "holdout-no-city-bike-child-seat", + "split": "held-out", + "kind": "out-of-catalog", + "query": "a child seat for a city bicycle", + "intent": "The reviewed eligible catalog contains no child seats or city-bike accessories.", + "expectedTop1Sku": null, + "actualTop1Sku": "ONEUP-V3-DROPPER-180", + "exactSkuTop1": null, + "relevantJudgedCount": 0, + "noRelevant": true, + "ndcgAt5": null, + "precisionAt5": null, + "eligibleCandidateCount": 29, + "constraintViolationCount": 0, + "topFive": [ + { + "sku": "ONEUP-V3-DROPPER-180", + "text": "OneUp Dropper Post V3 180mm", + "price": 269.0, + "grade": 0 + }, + { + "sku": "GIANT-ANTHEM-ADV29-1", + "text": "Giant Anthem Advanced 29 1", + "price": 7800.0, + "grade": 0 + }, + { + "sku": "LABOR", + "text": "Labor", + "price": 80.0, + "grade": 0 + }, + { + "sku": "MUCOFF-NANO-CLEANER", + "text": "Muc-Off Nano Tech Bike Cleaner", + "price": 16.99, + "grade": 0 + }, + { + "sku": "GIANT-TRANCE-X-ADVPRO29", + "text": "Giant Trance X Advanced Pro 29", + "price": 6200.0, + "grade": 0 + } + ] + } + ] +} diff --git a/samples/docs/search.md b/samples/docs/search.md index 845586239..e2f927a0e 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -20,7 +20,8 @@ The goal is not merely to store some numbers and label the result "AI". It is to | Separate search projection and pgvector semantic retrieval | Implemented | PostgreSQL stores 768-dimensional embeddings and computes exact, filtered cosine-distance rankings. | | PostgreSQL full-text retrieval and hybrid ranking | Proposed | The current sample does not perform full-text or hybrid search. | | Explicit backfill / asynchronous search indexing | Implemented | Test setup publishes an atomic initial backfill; one globbed product-event subscriber reconciles subsequent changes, including logical deletion. | -| Real-model relevance evaluation and HNSW benchmark | Initial probes implemented; broader evaluation proposed | Two illustrative semantic expectations pass after descriptive enrichment. Full-catalog relevance and index performance remain unproven. | +| Real-model relevance evaluation | Baseline measured; monthly/manual workflow configured | A version-controlled, 24-query judged set, isolated-catalog setup, opt-in REST/Ollama runner, and real report are available. Scores are observations, not quality gates. | +| HNSW benchmark | Proposed | Exact full-catalog retrieval is still the comparison baseline; index performance has not been measured. | The implemented baseline is PostgreSQL as the search store, one embedding per product/profile/recipe, a rebuildable search projection, and indexing outside product-write transactions. It uses local Ollama with Nomic v1.5 and 768 dimensions. Exact retrieval ranks eligible **indexed** products; count means the number of those candidates, not the number judged relevant or the complete catalog size. Full-text/hybrid retrieval windows remain future work. @@ -643,7 +644,7 @@ For the original cleaning query, adding the description increased the cleaner's Additional controls used an independent Python standard-library HTTP client and cosine calculation, reproducing the original PowerShell baseline rankings. Repeated query embeddings and single-versus-batched requests had cosine similarity **1.0** in this run. The cleaning query was closer to the functional cleaning description (**0.7803**) than to unrelated cake (**0.3661**) or database-backup (**0.4082**) descriptions. -This is evidence that document content materially affects this configuration, and that the model can separate these functional descriptions. It is **not** proof that sparse text is the only cause, that the model is optimal, or that an eight-query/three-product sample predicts full-catalog quality. The whole-bike-versus-parts query had a particularly small winning margin, despite passing. There was no held-out evaluation set, and descriptions were designed with the task in mind. +This is evidence that document content materially affects this configuration, and that the model can separate these functional descriptions. It is **not** proof that sparse text is the only cause, that the model is optimal, or that an eight-query/three-product sample predicts full-catalog quality. The whole-bike-versus-parts query had a particularly small winning margin, despite passing. This initial experiment had no held-out evaluation set, and its diagnostic descriptions were designed with the task in mind. The experiment did not modify catalog data or change the original smoke expectations. The subsequent implementation adds researched product descriptions and includes them in `product-document-v2`; broader evaluation still matters. pgvector cannot supply meaning absent from its input vectors. @@ -664,6 +665,44 @@ The live MCP HTTP transport was also exercised against this API: `search_product The lower ranks expose the remaining quality problem: sealant is not a cleaner, and a repair multi-tool ranks above some drivetrain components. Exact search does not fix that; it faithfully returns the model's nearest candidates. These examples motivate hybrid retrieval and a curated evaluation set, not an invented confidence threshold or a claim that search is finished. +### Curated real-model relevance evaluation + +The opt-in evaluation lives in [`tools/product-search-relevance.dataset.json`](../../tools/product-search-relevance.dataset.json). It contains 24 realistic requests split into 18 tuning and 6 held-out queries: exact SKUs, product terminology, paraphrases, broad/ambiguous intent, hard price filters, and out-of-catalog requests. Every query attests that all 29 default-eligible seeded products were reviewed; listed judgments carry a grade and rationale, while each omitted SKU explicitly means grade 0. The eligible snapshot and the exact seed-file SHA-256 are pinned so changed seed data or live API catalog fields stop the run rather than silently changing the benchmark. + +Run the evaluator only when the Products REST API, its PostgreSQL database, and Ollama are already available and the configured profile's backfill is ready: + +```powershell +.\tools\test-product-search-relevance.ps1 +.\tools\product-search-relevance.ps1 -OutputPath artifacts\product-search-relevance-report.json +``` + +The first command exercises metric math, invalid judgments, catalog drift, incomplete backfill, and mocked HTTP failure without external services. The second makes real HTTP requests; it never rebuilds the index or substitutes synthetic embeddings. It verifies the pinned Nomic model digest, semantic/ranked API mode, exact 29-product live catalog signature, and candidate count for every query. A missing API, model/profile mismatch, seed/catalog drift, or incomplete/unready index fails explicitly. Reports identify the model, profile, document recipe, seed and catalog hashes, and include per-query results plus separate tuning/held-out aggregates. Do not edit held-out judgments or tune against held-out scores. + +The dedicated [`tools/product-search-relevance.seed.yaml`](../../tools/product-search-relevance.seed.yaml) is an exact copy of the reviewed test catalog, but is loaded explicitly through DbEx. The safe sequence is: create/migrate the isolated database (`all`), reset the `products` schema (`reset --accept-prompts`), restore its reference rows (`data`), then load the evaluation catalog (`execute `). The `Reset` filter is scoped to the `products` schema, and in practice clears its reference and transactional tables, search projection, and readiness marker. `Data` restores embedded reference data; `Execute` then inserts the product, inventory, and movement rows. Execute is **not** a reset or upsert: repeating it against populated tables fails with a duplicate-primary-key error and leaves existing rows unchanged. Do not rerun Execute without first doing the explicit Reset and reference-data reload. + +From the repository root, first set `$connectionString` to a connection whose database is exactly `contoso_search_eval`, then run: + +```powershell +$databaseProject = 'samples\src\Contoso.Products.Database\Contoso.Products.Database.csproj' +dotnet run --no-launch-profile --project $databaseProject --no-build --framework net10.0 -- all --connection-string $connectionString +dotnet run --no-launch-profile --project $databaseProject --no-build --framework net10.0 -- reset --accept-prompts --connection-string $connectionString +dotnet run --no-launch-profile --project $databaseProject --no-build --framework net10.0 -- data --connection-string $connectionString +$seedPath = (Resolve-Path 'tools\product-search-relevance.seed.yaml').Path +dotnet run --no-launch-profile --project $databaseProject --no-build --framework net10.0 -- execute $seedPath --connection-string $connectionString +``` + +The NUnit fixture below is the next step: it builds the vector projection, not the catalog data. The REST evaluator itself never resets, loads, or reindexes anything. + +The isolated preparation fixture, [`ProductSearchRelevanceSetupTests.cs`](../tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs), refuses to run unless the configured database is exactly `contoso_search_eval`; it does not seed or reset data. It uses the real indexing path to rebuild embeddings from the already-loaded catalog and verifies profile readiness. The separate [monthly/manual GitHub Actions workflow](../../.github/workflows/product-search-relevance.yml) provisions PostgreSQL, Redis, and Ollama, installs the pinned model, creates that dedicated database, runs the explicit DbEx sequence, the fixture, and the live API evaluator, then uploads the JSON report and API log as a 90-day artifact. Setup failures fail the workflow, and cleanup runs regardless. It is not part of default CI. Run it manually with `workflow_dispatch`, or download the artifact from the monthly run. + +The captured [`baseline report`](product-search-relevance-baseline.json) is one measured run, not a target: tuning mean nDCG@5 is **0.8377** over 17 relevant queries (one no-relevant query is reported separately), and tuning mean Precision@5 is **0.3765**. Exact-SKU top-1 is **2/3**; the Shimano XT derailleur query returned Muc-Off cleaner. Held-out mean nDCG@5 is **0.8770** over only **3** relevant queries, mean Precision@5 is **0.2667**, and exact-SKU top-1 is **0/1** (the SRAM X0 query returned Muc-Off cleaner). The held-out split also has three no-relevant/out-of-catalog queries; its tiny relevant denominator makes these descriptive results highly uncertain. The report records zero price-filter violations in 15 tuning and 5 held-out constrained results. No score threshold is imposed, and these results do not justify a claim that search is generally good or bad. + +Routine reports belong in the git-ignored `artifacts/` directory, not in the checked-in baseline. Replacing `samples/docs/product-search-relevance-baseline.json` is an explicit review step: compare the new report's per-query results, aggregate metrics, and model/dataset/catalog/implementation identities with the existing baseline; explain why the replacement is appropriate in the change description; then deliberately copy the reviewed report over the baseline and update the documented findings. Git history preserves the previous baseline. Do not automatically promote scheduled reports or replace the baseline merely because a score decreased. + +Rerun and review the benchmark when the embedding model/profile, document recipe or text fields, ranking/retrieval implementation, judged catalog/seed data, or judgment set changes. Preserve the prior report for comparison and record the changed identity; do not silently replace labels. When PostgreSQL full-text/RRF is implemented, run the same pinned dataset/model and compare per-query rankings and both splits against this semantic baseline; treat held-out as a final comparison, not a tuning target. Any new catalog revision requires reviewing all eligible products for every query before updating the pinned catalog identity. + +Judgments are graded 0 (irrelevant), 1 (marginal), 2 (useful secondary result), or 3 (direct fit). The report uses gain `2^grade - 1` for nDCG@5; Precision@5 counts grade 2 or 3 and always divides by five. Exact-SKU top-1 accuracy is shown only for exact-SKU queries. Queries with no grade-above-zero product are listed separately and excluded from nDCG/precision means rather than counted as successful empty results. Price-filter violations are counted and reported; no similarity/confidence cutoff or quality pass threshold is invented. The checked-in query set is not itself a measured baseline: capture and review a real report before making claims about full-catalog relevance. + Measure document and query embedding separately. Caching repeated query embeddings can help, but keys must include the profile and relevant preprocessing; user text may be sensitive, so retention/logging needs care. Trace embedding time, database time, indexing lag, retries, and failures. Avoid logging raw vectors, credentials, or full user queries by default. @@ -676,8 +715,9 @@ Trace embedding time, database time, indexing lag, retries, and failures. Avoid | 2. Exact pgvector retrieval (implemented) | Understand distance ordering with real filters. | Real database tests for ordering, eligibility, profile isolation, paging, freshness and atomic rebuild failure. | | 3. Full-text and RRF | Compare exact terminology with semantic intent. | Inspectable branch ranks and curated hybrid comparisons. | | 4. Indexing and backfill (implemented; operational recovery remains) | Maintain rebuildable derived state. | Duplicate/out-of-order handling, stale-write protection, updates/logical deletes and explicit readiness. Resumable reconciliation remains future work. | -| 5. HNSW benchmark | Measure speed versus recall at meaningful scale. | Exact baseline, larger corpus, real query plans, latency and recall results. | -| 6. MCP demonstration | Reuse the domain capability from an AI client. | Same API behavior, accurate tool descriptions, cross-host demonstration. | +| 5. Curated relevance evaluation (implemented; baseline captured) | Assess real-model rankings against reviewed judgments. | Pinned full-catalog labels, isolated held-out queries, metric definitions and a reproducible live report. | +| 6. HNSW benchmark | Measure speed versus recall at meaningful scale. | Exact baseline, larger corpus, real query plans, latency and recall results. | +| 7. MCP demonstration | Reuse the domain capability from an AI client. | Same API behavior, accurate tool descriptions, cross-host demonstration. | These are teaching milestones, not instructions to ship ranked search without lifecycle support. Completion requires the corresponding operational behavior, not only the read query. @@ -713,6 +753,13 @@ These files own the implemented behavior and the foundations for later milestone | [Ollama client](../src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs) | Task prefixes, model profile, explicit truncation policy, and response validation. | | [Embedding client tests](../tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs) | Mocked success/error responses and exact request construction. | | [Live embedding smoke check](../../tools/test-product-embeddings.ps1) | Explicit local-model probe and retained relevance expectations. | +| [Curated relevance judgments](../../tools/product-search-relevance.dataset.json) | Versioned graded labels, reviewed catalog snapshot, tuning/held-out split, and pinned model/seed identity. | +| [Relevance evaluator](../../tools/product-search-relevance.ps1) | Opt-in real REST/Ollama evaluation, drift/backfill checks, per-query metrics, and split aggregates. | +| [Relevance evaluator tests](../../tools/test-product-search-relevance.ps1) | Local metric, dataset validation, drift, backfill, and mocked-HTTP failure tests. | +| [Dedicated evaluation seed](../../tools/product-search-relevance.seed.yaml) | Exact reviewed catalog copy loaded using DbEx Execute after isolated reset/reference-data reload. | +| [Isolated setup fixture](../tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs) | Explicitly guarded real backfill and readiness verification for `contoso_search_eval` only. | +| [Monthly/manual workflow](../../.github/workflows/product-search-relevance.yml) | Separate scheduled run with isolated services and retained JSON/API-log artifact. | +| [Measured baseline report](product-search-relevance-baseline.json) | Captured model/profile/catalog identities, per-query results, and split metrics; no quality threshold. | | [Product query configuration](../src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs) | Server-selected semantic mode and existing filter/order fields. | | [Product repository](../src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs) | Existing filters, joins, ordering, mapping, and paging to preserve. | | [Product read service](../src/Contoso.Products.Application/ProductReadService.cs) | Current query delegation through the Application boundary. | diff --git a/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs b/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs new file mode 100644 index 000000000..6f6e35b9d --- /dev/null +++ b/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs @@ -0,0 +1,33 @@ +namespace Contoso.Products.Test.Api; + +[TestFixture, Explicit("Builds and verifies the real Ollama search profile in the isolated contoso_search_eval database.")] +public class ProductSearchRelevanceSetupTests : WithApiTester +{ + [OneTimeSetUp] + public async Task OneTimeSetUpAsync() + { + var connectionString = Environment.GetEnvironmentVariable("Aspire__Npgsql__ConnectionString"); + var database = string.IsNullOrWhiteSpace(connectionString) ? null : new Npgsql.NpgsqlConnectionStringBuilder(connectionString).Database; + if (!string.Equals(database, "contoso_search_eval", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException("This fixture only runs against the isolated contoso_search_eval database. Run the explicit DbEx Reset, Data, and Execute seed sequence first."); + + await Test.ClearFusionCacheAsync().ConfigureAwait(false); + + Test.Scoped(test => test.Run(async _ => + { + var count = await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false); + count.Should().Be(31); + }).AssertSuccess()); + } + + [Test] + public void Product_Search_RealModelProfileIsReady() + { + var response = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=something to clean mud off my bike&$take=1&$count=true") + .AssertOK(); + + response.Response.Headers.Should().ContainKey("X-Paging-Total-Count") + .WhoseValue.Should().ContainSingle().Which.Should().Be("29"); + } +} diff --git a/tools/product-search-relevance.dataset.json b/tools/product-search-relevance.dataset.json new file mode 100644 index 000000000..9b6cb4e6c --- /dev/null +++ b/tools/product-search-relevance.dataset.json @@ -0,0 +1,392 @@ +{ + "datasetVersion": "product-search-relevance-v1", + "profile": "ollama:nomic-embed-text:v1.5:768:retrieval-prefix-v1", + "model": "nomic-embed-text:v1.5", + "modelDigest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f", + "dimensions": 768, + "documentRecipe": "product-document-v2", + "documentInstruction": "search_document: ", + "queryInstruction": "search_query: ", + "seedFile": "tools/product-search-relevance.seed.yaml", + "seedFileSha256": "F18614A4520837867BF9187D812ECC5BAF0559D0E652FC1175CDBECFC74ADF44", + "catalogEligibility": "Default Products API catalog: exclude inactive and logically deleted products; non-stocked service rows remain eligible.", + "catalogReviewed": true, + "catalog": [ + { "sku": "YETI-ASR-C2-2025", "text": "Yeti ASR C2", "price": 5800, "description": "A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack." }, + { "sku": "YETI-SB120-C2", "text": "Yeti SB120 C2", "price": 6200, "description": "A carbon full-suspension trail mountain bike designed for efficient climbs, responsive handling and flowing singletrack descents." }, + { "sku": "YETI-SB140-LR-C2", "text": "Yeti SB140 LR C2", "price": 6600, "description": "A carbon full-suspension trail mountain bike in the Lunch Ride configuration, built for technical climbs and rough, fast descents." }, + { "sku": "YETI-SB165-C3", "text": "Yeti SB165 C3", "price": 7200, "description": "A carbon full-suspension mountain bike for gravity-focused riding, steep descents and demanding bike-park terrain." }, + { "sku": "YETI-MTE-C2", "text": "Yeti MTe C2 eMTB", "price": 9850, "description": "An electric-assist mountain bike with a carbon full-suspension frame, designed for technical trail riding and assisted climbs." }, + { "sku": "CANYON-LUXTRAIL-CF6", "text": "Canyon Lux Trail CF 6", "price": 3899, "description": "A carbon full-suspension mountain bike combining cross-country pedalling efficiency with trail capability for long rides and technical singletrack." }, + { "sku": "CANYON-SPECTRAL-CF8", "text": "Canyon Spectral CF 8", "price": 4999, "description": "A carbon full-suspension trail mountain bike for varied singletrack, balancing climbing capability with control on rough, technical descents." }, + { "sku": "CANYON-NEURON-CF7", "text": "Canyon Neuron CF 7", "price": 3499, "description": "A carbon full-suspension trail mountain bike for long rides over varied terrain, combining efficient climbing with stable descending." }, + { "sku": "CANYON-TORQUE-MULLET-CF8", "text": "Canyon Torque Mullet CF 8", "price": 3599, "description": "A carbon mixed-wheel mountain bike designed for freeride and bike-park use, including jumps, drops and steep descents." }, + { "sku": "CANYON-SENDER-CFR", "text": "Canyon Sender CFR", "price": 7799, "description": "A carbon downhill mountain bike designed for racing and fast descending on steep, demanding tracks and bike-park terrain." }, + { "sku": "SPEC-STUMPJUMPER-15-COMP", "text": "Specialized Stumpjumper 15 Comp", "price": 5500, "description": "A full-suspension trail mountain bike with adjustable geometry, designed to balance climbing efficiency and control on technical descents." }, + { "sku": "SPEC-ENDURO-PRO", "text": "Specialized Enduro Pro", "price": 9500, "description": "A carbon full-suspension enduro mountain bike built for steep, rough descents and enduro racing while retaining the ability to pedal uphill." }, + { "sku": "SPEC-CHISEL-COMP", "text": "Specialized Chisel Comp", "price": 3400, "description": "An aluminium cross-country mountain bike designed for efficient pedalling, racing and fast singletrack riding." }, + { "sku": "SPEC-STATUS-140", "text": "Specialized Status 140", "price": 2499, "description": "An aluminium mixed-wheel full-suspension mountain bike designed for playful trail riding, jumps and technical descents." }, + { "sku": "GIANT-TRANCE-X-ADVPRO29", "text": "Giant Trance X Advanced Pro 29", "price": 6200, "description": "A carbon full-suspension trail mountain bike with 29-inch wheels and adjustable geometry, designed for climbing and technical descending." }, + { "sku": "GIANT-ANTHEM-ADV29-1", "text": "Giant Anthem Advanced 29 1", "price": 7800, "description": "A carbon full-suspension cross-country mountain bike with 29-inch wheels, designed for efficient pedalling and fast riding on demanding race courses." }, + { "sku": "SHIMANO-XT-M8100-RD", "text": "Shimano Deore XT M8100 Rear Derailleur", "price": 129, "description": "A 12-speed mountain bike rear derailleur that moves the chain between cassette sprockets to change gears. Its clutch helps control chain movement on rough trails." }, + { "sku": "SHIMANO-SLX-M7100-CASSETTE", "text": "Shimano SLX M7100 12-speed Cassette", "price": 99, "description": "A 12-speed mountain bike cassette that provides a wide range of gears for climbing and descending. It uses Shimano Micro Spline mounting and Hyperglide Plus shifting." }, + { "sku": "SHIMANO-XTR-M9100-CRANK", "text": "Shimano XTR M9100 Crankset", "price": 499, "description": "A mountain bike crankset that transfers pedalling power to the chain through a direct-mount chainring, designed for lightweight cross-country and trail use." }, + { "sku": "SRAM-GX-EAGLE-RD", "text": "SRAM GX Eagle Rear Derailleur", "price": 135, "description": "A cable-operated 12-speed mountain bike rear derailleur for SRAM Eagle drivetrains. It shifts the chain between rear sprockets and uses a clutch to control chain movement." }, + { "sku": "SRAM-X0-TRANSMISSION-RD", "text": "SRAM X0 Transmission Rear Derailleur", "price": 400, "description": "A wireless electronic mountain bike rear derailleur for SRAM Eagle Transmission drivetrains. Its hangerless Full Mount design attaches around the rear axle on compatible frames." }, + { "sku": "SRAM-XX1-EAGLE-CASSETTE", "text": "SRAM XX1 Eagle Cassette", "price": 449, "description": "A lightweight 12-speed mountain bike cassette with a wide gear range for climbing and descending, designed for SRAM Eagle drivetrains and XD freehub drivers." }, + { "sku": "ONEUP-V3-DROPPER-180", "text": "OneUp Dropper Post V3 180mm", "price": 269, "description": "A mountain bike dropper seatpost with 180 mm of travel. Riders can lower the saddle for descending and raise it for pedalling using a compatible remote." }, + { "sku": "ONEUP-CARBON-HANDLEBAR", "text": "OneUp Carbon Handlebar", "price": 159, "description": "A carbon mountain bike handlebar with a shaped profile designed to absorb vibration and reduce hand fatigue while maintaining steering control." }, + { "sku": "ONEUP-EDC-TOOL-V2", "text": "OneUp EDC Tool V2", "price": 59, "description": "A compact trailside repair multi-tool with hex keys, a Torx key, a chain breaker and a tyre lever, designed for storage in compatible OneUp steerer or pump systems." }, + { "sku": "MUCOFF-NANO-CLEANER", "text": "Muc-Off Nano Tech Bike Cleaner", "price": 16.99, "description": "A spray-on bicycle cleaner that breaks down dirt and grime on frames and components before rinsing. Suitable for routine washing after muddy rides." }, + { "sku": "MUCOFF-TUBELESS-SEALANT", "text": "Muc-Off MTB Tubeless Sealant", "price": 8.00, "description": "A tubeless mountain bike tyre sealant that helps seal punctures and maintain an airtight tyre system. Its UV-reactive dye helps locate leaks." }, + { "sku": "MUCOFF-DRIVETRAIN-CLEANER", "text": "Muc-Off Drivetrain Cleaner", "price": 25.00, "description": "A bicycle drivetrain cleaner that removes grease, oil and grime from chains, cassettes, chainrings and derailleurs. It can be sprayed on or used with a chain-cleaning device." }, + { "sku": "LABOR", "text": "Labor", "price": 80.00, "description": "Workshop labour charged by the hour for bicycle maintenance and repair. This is a service rather than a stocked physical product." } + ], + "queries": [ + { + "id": "tune-exact-yeti-asr", + "split": "tuning", + "kind": "exact-sku", + "query": "YETI-ASR-C2-2025", + "expectedTop1Sku": "YETI-ASR-C2-2025", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Exact SKU lookup for the Yeti ASR C2 2025.", + "judgments": [ + { "sku": "YETI-ASR-C2-2025", "grade": 3, "rationale": "Exact identifier match." } + ] + }, + { + "id": "tune-exact-shimano-derailleur", + "split": "tuning", + "kind": "exact-sku", + "query": "SHIMANO-XT-M8100-RD", + "expectedTop1Sku": "SHIMANO-XT-M8100-RD", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Exact SKU lookup for the Shimano XT rear derailleur.", + "judgments": [ + { "sku": "SHIMANO-XT-M8100-RD", "grade": 3, "rationale": "Exact identifier match." } + ] + }, + { + "id": "tune-exact-mucoff-cleaner", + "split": "tuning", + "kind": "exact-sku", + "query": "MUCOFF-NANO-CLEANER", + "expectedTop1Sku": "MUCOFF-NANO-CLEANER", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Exact SKU lookup for the Nano Tech bicycle cleaner.", + "judgments": [ + { "sku": "MUCOFF-NANO-CLEANER", "grade": 3, "rationale": "Exact identifier match." } + ] + }, + { + "id": "tune-paraphrase-muddy-bike-cleaner", + "split": "tuning", + "kind": "paraphrase", + "query": "something to clean mud off my bike", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "A user describes the cleaning task rather than the product name.", + "judgments": [ + { "sku": "MUCOFF-NANO-CLEANER", "grade": 3, "rationale": "Specifically cleans bicycle dirt and grime after muddy rides." }, + { "sku": "MUCOFF-DRIVETRAIN-CLEANER", "grade": 2, "rationale": "Cleans bicycle grime, but is specialized for drivetrain parts rather than the whole bike." } + ] + }, + { + "id": "tune-terminology-changing-gears", + "split": "tuning", + "kind": "terminology", + "query": "a component for changing gears", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Broad functional terminology for drivetrain components.", + "judgments": [ + { "sku": "SHIMANO-XT-M8100-RD", "grade": 3, "rationale": "A rear derailleur directly moves the chain to change gears." }, + { "sku": "SRAM-GX-EAGLE-RD", "grade": 3, "rationale": "A rear derailleur directly moves the chain to change gears." }, + { "sku": "SRAM-X0-TRANSMISSION-RD", "grade": 3, "rationale": "An electronic rear derailleur directly changes gears." }, + { "sku": "SHIMANO-SLX-M7100-CASSETTE", "grade": 2, "rationale": "The cassette supplies the selectable sprockets used when changing gears." }, + { "sku": "SRAM-XX1-EAGLE-CASSETTE", "grade": 2, "rationale": "The cassette supplies the selectable sprockets used when changing gears." }, + { "sku": "SHIMANO-XTR-M9100-CRANK", "grade": 1, "rationale": "The crankset is a drivetrain component but does not itself select gears." } + ] + }, + { + "id": "tune-paraphrase-climbing-gears", + "split": "tuning", + "kind": "paraphrase", + "query": "a wide range of low gears for climbing", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "A climbing use case described without naming a cassette.", + "judgments": [ + { "sku": "SHIMANO-SLX-M7100-CASSETTE", "grade": 3, "rationale": "Explicitly offers a wide range of gears for climbing." }, + { "sku": "SRAM-XX1-EAGLE-CASSETTE", "grade": 3, "rationale": "Explicitly offers a wide gear range for climbing." }, + { "sku": "SHIMANO-XT-M8100-RD", "grade": 1, "rationale": "A derailleur is part of a drivetrain, but its description does not establish a low climbing range." } + ] + }, + { + "id": "tune-paraphrase-tubeless-punctures", + "split": "tuning", + "kind": "paraphrase", + "query": "liquid to seal small punctures in tubeless tires", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Describes the purpose of a tubeless tire sealant without its product name.", + "judgments": [ + { "sku": "MUCOFF-TUBELESS-SEALANT", "grade": 3, "rationale": "The product is a tubeless tyre sealant that helps seal punctures." } + ] + }, + { + "id": "tune-paraphrase-lower-saddle", + "split": "tuning", + "kind": "paraphrase", + "query": "a seat post that lowers the saddle for descents", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Describes dropper-post behavior in everyday language.", + "judgments": [ + { "sku": "ONEUP-V3-DROPPER-180", "grade": 3, "rationale": "The dropper post lowers the saddle for descending." }, + { "sku": "YETI-SB165-C3", "grade": 1, "rationale": "A downhill-oriented bike relates to the use case but is not the requested seat post." } + ] + }, + { + "id": "tune-terminology-reduce-hand-vibration", + "split": "tuning", + "kind": "terminology", + "query": "handlebars that reduce hand fatigue on rough trails", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Matches rider language to a product's vibration and fatigue benefit.", + "judgments": [ + { "sku": "ONEUP-CARBON-HANDLEBAR", "grade": 3, "rationale": "Its shaped carbon profile is designed to absorb vibration and reduce hand fatigue." } + ] + }, + { + "id": "tune-paraphrase-trailside-repair", + "split": "tuning", + "kind": "paraphrase", + "query": "a compact tool for fixing my bike on the trail", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "A use-case paraphrase for a portable repair multi-tool.", + "judgments": [ + { "sku": "ONEUP-EDC-TOOL-V2", "grade": 3, "rationale": "A compact trailside repair tool with a chain breaker and tyre lever." } + ] + }, + { + "id": "tune-no-current-bike-under-1000", + "split": "tuning", + "kind": "price-constraint", + "query": "a mountain bike under $1000", + "constraints": { "price": { "operator": "lt", "value": 1000 } }, + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "No currently eligible seeded mountain bike is priced below $1000; the deleted Giant Talon is not eligible.", + "judgments": [] + }, + { + "id": "tune-terminology-cross-country-race", + "split": "tuning", + "kind": "terminology", + "query": "a fast cross-country bike for racing", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Cross-country race terminology matched to seeded bike descriptions.", + "judgments": [ + { "sku": "YETI-ASR-C2-2025", "grade": 3, "rationale": "Built for efficient climbing and fast technical singletrack." }, + { "sku": "GIANT-ANTHEM-ADV29-1", "grade": 3, "rationale": "Designed for efficient pedalling and demanding race courses." }, + { "sku": "SPEC-CHISEL-COMP", "grade": 2, "rationale": "An efficient cross-country race bike, though aluminium rather than carbon." }, + { "sku": "CANYON-LUXTRAIL-CF6", "grade": 2, "rationale": "Combines cross-country efficiency with trail capability, but is framed for long rides rather than racing." } + ] + }, + { + "id": "tune-broad-steep-bike-park", + "split": "tuning", + "kind": "broad", + "query": "a bike for steep rough descents and bike-park riding", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Broad descending intent with several suitable bike categories.", + "judgments": [ + { "sku": "YETI-SB165-C3", "grade": 3, "rationale": "Designed for gravity riding, steep descents and bike-park terrain." }, + { "sku": "CANYON-SENDER-CFR", "grade": 3, "rationale": "A downhill bike for steep tracks and bike-park terrain." }, + { "sku": "SPEC-ENDURO-PRO", "grade": 3, "rationale": "Built for steep, rough descents and enduro racing." }, + { "sku": "CANYON-TORQUE-MULLET-CF8", "grade": 2, "rationale": "Designed for freeride and bike-park use, but not specifically downhill racing." }, + { "sku": "SPEC-STATUS-140", "grade": 2, "rationale": "Designed for jumps and technical descents, but is a more playful trail bike." }, + { "sku": "YETI-SB140-LR-C2", "grade": 2, "rationale": "Built for rough, fast descents, though its stated use is trail riding." } + ] + }, + { + "id": "tune-paraphrase-drivetrain-degreaser", + "split": "tuning", + "kind": "paraphrase", + "query": "degreaser for a greasy chain and cassette", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Names drivetrain parts and soil rather than the cleaner product name.", + "judgments": [ + { "sku": "MUCOFF-DRIVETRAIN-CLEANER", "grade": 3, "rationale": "Removes grease and grime from chains, cassettes and other drivetrain parts." }, + { "sku": "MUCOFF-NANO-CLEANER", "grade": 1, "rationale": "A general bicycle cleaner, but not specifically a drivetrain degreaser." } + ] + }, + { + "id": "tune-price-derailleur-under-150", + "split": "tuning", + "kind": "price-constraint", + "query": "a rear derailleur for a 12-speed mountain bike under $150", + "constraints": { "price": { "operator": "lt", "value": 150 } }, + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Hard price ceiling and product type; two seeded derailleurs satisfy both.", + "judgments": [ + { "sku": "SHIMANO-XT-M8100-RD", "grade": 3, "rationale": "A 12-speed mountain bike rear derailleur priced at $129." }, + { "sku": "SRAM-GX-EAGLE-RD", "grade": 3, "rationale": "A 12-speed mountain bike rear derailleur priced at $135." } + ] + }, + { + "id": "tune-price-trail-bike-under-5000", + "split": "tuning", + "kind": "price-constraint", + "query": "a trail bike for technical descents under $5000", + "constraints": { "price": { "operator": "lt", "value": 5000 } }, + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Filters to affordable seeded bikes while preferring trail models suitable for technical descents.", + "judgments": [ + { "sku": "CANYON-SPECTRAL-CF8", "grade": 3, "rationale": "A trail bike balancing climbing with control on rough technical descents, priced at $4999." }, + { "sku": "CANYON-NEURON-CF7", "grade": 3, "rationale": "A trail bike with stable descending over varied terrain, priced at $3499." }, + { "sku": "CANYON-LUXTRAIL-CF6", "grade": 2, "rationale": "A trail-capable cross-country bike for technical singletrack, priced at $3899." }, + { "sku": "CANYON-TORQUE-MULLET-CF8", "grade": 1, "rationale": "Capable on steep descents but oriented to freeride and bike parks rather than trail riding." } + ] + }, + { + "id": "tune-broad-everyday-singletrack", + "split": "tuning", + "kind": "broad", + "query": "a versatile mountain bike for after-work singletrack", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Ambiguous everyday trail request without a specific price, discipline or component.", + "judgments": [ + { "sku": "YETI-SB120-C2", "grade": 2, "rationale": "A trail bike for efficient climbs and flowing singletrack descents." }, + { "sku": "CANYON-SPECTRAL-CF8", "grade": 2, "rationale": "A versatile trail bike for varied singletrack." }, + { "sku": "CANYON-NEURON-CF7", "grade": 2, "rationale": "A trail bike for long rides over varied terrain." }, + { "sku": "SPEC-STUMPJUMPER-15-COMP", "grade": 2, "rationale": "Balances climbing efficiency and technical trail control." }, + { "sku": "GIANT-TRANCE-X-ADVPRO29", "grade": 2, "rationale": "A trail bike designed for climbing and technical descending." }, + { "sku": "SPEC-CHISEL-COMP", "grade": 1, "rationale": "A cross-country bike suitable for fast singletrack but less versatile on technical trails." }, + { "sku": "YETI-ASR-C2-2025", "grade": 1, "rationale": "An efficient cross-country bike, narrower in purpose than the broad trail request." } + ] + }, + { + "id": "tune-broad-weekend-ride-kit", + "split": "tuning", + "kind": "broad", + "query": "useful gear for a weekend ride on rough trails", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Intentionally broad request that could mean a trail accessory or repair item.", + "judgments": [ + { "sku": "ONEUP-EDC-TOOL-V2", "grade": 2, "rationale": "A compact trailside tool is useful for repairs during a ride." }, + { "sku": "ONEUP-V3-DROPPER-180", "grade": 1, "rationale": "A dropper supports descending, but the request does not specifically ask for a bike component." }, + { "sku": "ONEUP-CARBON-HANDLEBAR", "grade": 1, "rationale": "A trail riding component, but not an obvious general-purpose ride kit item." }, + { "sku": "MUCOFF-TUBELESS-SEALANT", "grade": 1, "rationale": "A repair-related consumable, though it is installed in a tyre rather than carried as general kit." } + ] + }, + { + "id": "holdout-exact-sram-x0-derailleur", + "split": "held-out", + "kind": "exact-sku", + "query": "SRAM-X0-TRANSMISSION-RD", + "expectedTop1Sku": "SRAM-X0-TRANSMISSION-RD", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Held-out exact SKU lookup for the SRAM X0 electronic rear derailleur.", + "judgments": [ + { "sku": "SRAM-X0-TRANSMISSION-RD", "grade": 3, "rationale": "Exact identifier match." } + ] + }, + { + "id": "holdout-terminology-workshop-labour", + "split": "held-out", + "kind": "terminology", + "query": "an hourly workshop charge for a bicycle tune-up", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Held-out service request for workshop labour rather than a stocked physical product.", + "judgments": [ + { "sku": "LABOR", "grade": 3, "rationale": "The catalog entry is hourly workshop labour for bicycle maintenance and repair." } + ] + }, + { + "id": "holdout-price-carbon-bike-under-4000", + "split": "held-out", + "kind": "price-constraint", + "query": "a carbon mountain bike for long rides under $4000", + "constraints": { "price": { "operator": "lt", "value": 4000 } }, + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "Held-out structured price ceiling with a long-ride carbon-bike intent.", + "judgments": [ + { "sku": "CANYON-LUXTRAIL-CF6", "grade": 3, "rationale": "A carbon full-suspension bike explicitly described for long rides, priced at $3899." }, + { "sku": "CANYON-NEURON-CF7", "grade": 3, "rationale": "A carbon full-suspension trail bike explicitly designed for long rides, priced at $3499." }, + { "sku": "CANYON-TORQUE-MULLET-CF8", "grade": 1, "rationale": "A carbon bike under the price limit, but its freeride and bike-park focus is a poor long-ride fit." } + ] + }, + { + "id": "holdout-no-electric-bike-charger", + "split": "held-out", + "kind": "out-of-catalog", + "query": "a charger for the battery on an electric mountain bike", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "The catalog includes an electric mountain bike but no battery charger accessory.", + "judgments": [] + }, + { + "id": "holdout-no-night-riding-lights", + "split": "held-out", + "kind": "out-of-catalog", + "query": "USB rechargeable lights for night riding", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "The reviewed eligible catalog contains no lights or night-riding electronics.", + "judgments": [] + }, + { + "id": "holdout-no-city-bike-child-seat", + "split": "held-out", + "kind": "out-of-catalog", + "query": "a child seat for a city bicycle", + "catalogReviewed": true, + "judgedCatalogCount": 29, + "omittedSkuGrade": 0, + "intent": "The reviewed eligible catalog contains no child seats or city-bike accessories.", + "judgments": [] + } + ] +} diff --git a/tools/product-search-relevance.ps1 b/tools/product-search-relevance.ps1 new file mode 100644 index 000000000..69b3c97ff --- /dev/null +++ b/tools/product-search-relevance.ps1 @@ -0,0 +1,439 @@ +[CmdletBinding()] +param( + [uri] $BaseAddress = 'http://localhost:5207/', + [uri] $OllamaAddress = 'http://localhost:11434/', + [string] $DatasetPath = "$PSScriptRoot\product-search-relevance.dataset.json", + [string] $OutputPath +) + +$ErrorActionPreference = 'Stop' + +function Get-ProductSearchDataset { + param([Parameter(Mandatory)][string] $Path) + if (!(Test-Path -LiteralPath $Path -PathType Leaf)) { throw "Relevance dataset not found: $Path" } + try { return Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json } + catch { throw "Could not parse relevance dataset '$Path': $($_.Exception.Message)" } +} + +function Test-ProductSearchDataset { + param([Parameter(Mandatory)][object] $Dataset) + + if ($Dataset.datasetVersion -ne 'product-search-relevance-v1') { throw 'Unsupported product-search relevance dataset version.' } + if ($Dataset.profile -ne 'ollama:nomic-embed-text:v1.5:768:retrieval-prefix-v1' -or + $Dataset.model -ne 'nomic-embed-text:v1.5' -or + $Dataset.modelDigest -ne '0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f' -or + [int]$Dataset.dimensions -ne 768 -or + $Dataset.documentRecipe -ne 'product-document-v2' -or + $Dataset.documentInstruction -ne 'search_document: ' -or + $Dataset.queryInstruction -ne 'search_query: ') { + throw 'Dataset model, profile or document/query recipe does not match the pinned Products implementation.' + } + if ($Dataset.catalogReviewed -ne $true -or [string]::IsNullOrWhiteSpace($Dataset.catalogEligibility)) { + throw 'The eligible seeded catalog must be explicitly reviewed and described.' + } + if (@($Dataset.queries).Count -lt 20 -or @($Dataset.queries).Count -gt 30) { + throw 'The curated evaluation must contain 20 to 30 queries.' + } + if (@($Dataset.catalog).Count -eq 0) { throw 'The reviewed catalog snapshot is empty.' } + + $catalogSkus = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($product in $Dataset.catalog) { + if ([string]::IsNullOrWhiteSpace($product.sku) -or + [string]::IsNullOrWhiteSpace($product.text) -or + $null -eq $product.price -or + [string]::IsNullOrWhiteSpace($product.description)) { + throw 'Every catalog snapshot entry requires a SKU, text, price and description.' + } + if (!$catalogSkus.Add([string]$product.sku)) { throw "Duplicate catalog SKU '$($product.sku)'." } + } + + $queryIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($query in $Dataset.queries) { + if ([string]::IsNullOrWhiteSpace($query.id) -or !$queryIds.Add([string]$query.id)) { throw "Missing or duplicate query ID '$($query.id)'." } + if ([string]::IsNullOrWhiteSpace($query.query) -or [string]::IsNullOrWhiteSpace($query.intent)) { throw "Query '$($query.id)' requires query text and intent." } + if ($query.split -notin @('tuning', 'held-out')) { throw "Query '$($query.id)' has an invalid split." } + if ($query.kind -notin @('exact-sku', 'terminology', 'paraphrase', 'broad', 'price-constraint', 'out-of-catalog')) { + throw "Query '$($query.id)' has an unsupported query kind." + } + if ($query.catalogReviewed -ne $true -or [int]$query.judgedCatalogCount -ne $catalogSkus.Count -or [int]$query.omittedSkuGrade -ne 0) { + throw "Query '$($query.id)' must attest that all catalog entries were judged and omitted SKUs mean grade zero." + } + if ($query.kind -eq 'exact-sku') { + if ([string]::IsNullOrWhiteSpace($query.expectedTop1Sku) -or !$catalogSkus.Contains([string]$query.expectedTop1Sku)) { + throw "Exact-SKU query '$($query.id)' must name a known expected top-1 SKU." + } + } + elseif ($null -ne $query.expectedTop1Sku) { throw "Only exact-SKU queries may specify expectedTop1Sku ('$($query.id)')." } + + if ($null -ne $query.constraints) { + if ($null -eq $query.constraints.price -or $query.constraints.price.operator -notin @('eq', 'lt', 'le', 'gt', 'ge') -or $null -eq $query.constraints.price.value) { + throw "Query '$($query.id)' has an unsupported structured constraint." + } + [decimal]$constraintValue = 0 + if (![decimal]::TryParse([string]$query.constraints.price.value, [Globalization.NumberStyles]::Number, [Globalization.CultureInfo]::InvariantCulture, [ref]$constraintValue)) { + throw "Query '$($query.id)' has an invalid price constraint value." + } + } + + $judgedSkus = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($judgment in $query.judgments) { + if (!$catalogSkus.Contains([string]$judgment.sku)) { throw "Query '$($query.id)' judges unknown SKU '$($judgment.sku)'." } + if (!$judgedSkus.Add([string]$judgment.sku)) { throw "Query '$($query.id)' judges SKU '$($judgment.sku)' more than once." } + if ([int]$judgment.grade -lt 1 -or [int]$judgment.grade -gt 3 -or [string]::IsNullOrWhiteSpace($judgment.rationale)) { + throw "Query '$($query.id)' has an invalid grade or missing rationale for '$($judgment.sku)'." + } + $product = $Dataset.catalog | Where-Object sku -EQ $judgment.sku | Select-Object -First 1 + if (!(Test-ProductSearchConstraint -Product $product -Query $query)) { + throw "Query '$($query.id)' judges SKU '$($judgment.sku)' even though it violates the query constraints." + } + } + if ($query.kind -eq 'exact-sku' -and + !(@($query.judgments | Where-Object { $_.sku -eq $query.expectedTop1Sku -and [int]$_.grade -eq 3 }).Count -eq 1)) { + throw "Exact-SKU query '$($query.id)' must grade its expected product a direct fit." + } + } + if (!(@($Dataset.queries | Where-Object split -EQ 'tuning').Count -gt 0) -or + !(@($Dataset.queries | Where-Object split -EQ 'held-out').Count -gt 0)) { + throw 'Both tuning and held-out queries are required.' + } + return $true +} + +function Test-ProductSearchImplementationIdentity { + param([Parameter(Mandatory)][object] $Dataset) + $clientPath = Join-Path $PSScriptRoot '..\samples\src\Contoso.Products.Infrastructure\Clients\Ollama\OllamaHttpClient.cs' + $documentPath = Join-Path $PSScriptRoot '..\samples\src\Contoso.Products.Application\Search\ProductSearchDocument.cs' + if (!(Test-Path -LiteralPath $clientPath -PathType Leaf) -or !(Test-Path -LiteralPath $documentPath -PathType Leaf)) { + throw 'Could not verify the checked-in Products embedding and document implementation.' + } + $client = Get-Content -LiteralPath $clientPath -Raw + $document = Get-Content -LiteralPath $documentPath -Raw + if ($client -notmatch 'ModelName\s*=\s*"([^"]+)"' -or $Matches[1] -ne $Dataset.model) { + throw 'Dataset model does not match the checked-in Ollama client.' + } + if ($client -notmatch 'Dimensions\s*=\s*(\d+)' -or [int]$Matches[1] -ne [int]$Dataset.dimensions) { + throw 'Dataset dimensions do not match the checked-in Ollama client.' + } + if ($client -notmatch 'Profile\s*=\s*"([^"]+)"' -or $Matches[1] -ne $Dataset.profile) { + throw 'Dataset profile does not match the checked-in Ollama client.' + } + if ($document -notmatch 'TemplateVersion\s*=\s*"([^"]+)"' -or $Matches[1] -ne $Dataset.documentRecipe) { + throw 'Dataset document recipe does not match the checked-in canonical document.' + } + if (!$client.Contains("EmbedDocumentAsync(string text, CancellationToken ct = default) => EmbedAsync(text, `"$($Dataset.documentInstruction)`"") -or + !$client.Contains("EmbedQueryAsync(string text, CancellationToken ct = default) => EmbedAsync(text, `"$($Dataset.queryInstruction)`"")) { + throw 'Dataset query/document instructions do not match the checked-in Ollama client.' + } + return [pscustomobject]@{ + clientSha256 = (Get-FileHash -LiteralPath $clientPath -Algorithm SHA256).Hash + documentBuilderSha256 = (Get-FileHash -LiteralPath $documentPath -Algorithm SHA256).Hash + } +} + +function Get-ProductSearchCatalogSignature { + param([Parameter(Mandatory)][object[]] $Products) + $lines = foreach ($product in $Products) { + $sku = ([string]$product.sku).Trim() + $text = ([string]$product.text).Trim() + $description = ([string]$product.description).Trim() + $price = ([decimal]$product.price).ToString('0.############################', [Globalization.CultureInfo]::InvariantCulture) + "$sku|$text|$price|$description" + } + [string[]]$ordered = @($lines) + [Array]::Sort($ordered, [StringComparer]::Ordinal) + $bytes = [Text.Encoding]::UTF8.GetBytes([string]::Join("`n", $ordered)) + return [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData($bytes)) +} + +function Get-ProductSearchEligibleCatalog { + param([Parameter(Mandatory)][object] $Dataset, [Parameter(Mandatory)][object] $Query) + $products = @($Dataset.catalog) + if ($null -eq $Query.constraints -or $null -eq $Query.constraints.price) { return ,$products } + $constraint = $Query.constraints.price + [decimal]$limit = $constraint.value + return ,@($products | Where-Object { + [decimal]$price = $_.price + switch ($constraint.operator) { + 'eq' { $price -eq $limit } + 'lt' { $price -lt $limit } + 'le' { $price -le $limit } + 'gt' { $price -gt $limit } + 'ge' { $price -ge $limit } + } + }) +} + +function Get-ProductSearchFilter { + param([Parameter(Mandatory)][object] $Query) + if ($null -eq $Query.constraints -or $null -eq $Query.constraints.price) { return $null } + $constraint = $Query.constraints.price + $value = ([decimal]$constraint.value).ToString('0.############################', [Globalization.CultureInfo]::InvariantCulture) + return "price $($constraint.operator) $value" +} + +function Get-ProductSearchMetrics { + param([int[]] $RankedGrades, [int[]] $IdealGrades) + $relevant = @($IdealGrades | Where-Object { $_ -gt 0 }).Count + if ($relevant -eq 0) { + return [pscustomobject]@{ HasRelevant = $false; NdcgAt5 = $null; PrecisionAt5 = $null } + } + + $dcg = 0.0 + for ($i = 0; $i -lt [Math]::Min(5, $RankedGrades.Count); $i++) { + $dcg += ([Math]::Pow(2, $RankedGrades[$i]) - 1) / [Math]::Log($i + 2, 2) + } + $idealDcg = 0.0 + $idealTop = @($IdealGrades | Sort-Object -Descending | Select-Object -First 5) + for ($i = 0; $i -lt $idealTop.Count; $i++) { + $idealDcg += ([Math]::Pow(2, $idealTop[$i]) - 1) / [Math]::Log($i + 2, 2) + } + $precision = @($RankedGrades | Select-Object -First 5 | Where-Object { $_ -ge 2 }).Count / 5.0 + return [pscustomobject]@{ HasRelevant = $true; NdcgAt5 = $dcg / $idealDcg; PrecisionAt5 = $precision } +} + +function Get-ProductSearchResponse { + param([Parameter(Mandatory)][scriptblock] $Request, [Parameter(Mandatory)][uri] $Uri) + try { $response = & $Request $Uri } + catch { throw "HTTP request failed for '$Uri': $($_.Exception.Message)" } + if ($null -eq $response -or [int]$response.StatusCode -lt 200 -or [int]$response.StatusCode -ge 300) { + $status = if ($null -eq $response) { 'no response' } else { [string]$response.StatusCode } + throw "HTTP request to '$Uri' returned $status." + } + try { $data = ConvertFrom-Json -InputObject ([string]$response.Content) -NoEnumerate } + catch { throw "Invalid JSON returned by '$Uri': $($_.Exception.Message)" } + return [pscustomobject]@{ Data = $data; Headers = $response.Headers } +} + +function Get-ProductSearchCountHeader { + param([Parameter(Mandatory)][object] $Headers) + foreach ($key in $Headers.Keys) { + if ([string]$key -ieq 'X-Paging-Total-Count') { + [int]$count = 0 + if (![int]::TryParse([string]$Headers[$key], [ref]$count)) { throw 'Products API returned an invalid X-Paging-Total-Count header.' } + return $count + } + } + foreach ($key in $Headers.AllKeys) { + if ([string]$key -ieq 'X-Paging-Total-Count') { + [int]$count = 0 + if (![int]::TryParse([string]$Headers[$key], [ref]$count)) { throw 'Products API returned an invalid X-Paging-Total-Count header.' } + return $count + } + } + throw 'Products API did not return X-Paging-Total-Count; cannot prove catalog and index completeness.' +} + +function Test-ProductSearchConstraint { + param([Parameter(Mandatory)][object] $Product, [Parameter(Mandatory)][object] $Query) + if ($null -eq $Query.constraints -or $null -eq $Query.constraints.price) { return $true } + if ($null -eq $Product.price) { return $false } + [decimal]$price = $Product.price + [decimal]$limit = $Query.constraints.price.value + switch ($Query.constraints.price.operator) { + 'eq' { return $price -eq $limit } + 'lt' { return $price -lt $limit } + 'le' { return $price -le $limit } + 'gt' { return $price -gt $limit } + 'ge' { return $price -ge $limit } + } + return $false +} + +function Invoke-ProductSearchRelevanceEvaluation { + param( + [Parameter(Mandatory)][object] $Dataset, + [Parameter(Mandatory)][uri] $BaseAddress, + [Parameter(Mandatory)][uri] $OllamaAddress, + [Parameter(Mandatory)][scriptblock] $Request, + [string] $DatasetSha256 + ) + $implementationIdentity = Test-ProductSearchImplementationIdentity $Dataset + $catalogSignature = Get-ProductSearchCatalogSignature -Products @($Dataset.catalog) + $seedPath = Join-Path $PSScriptRoot ("..\{0}" -f ($Dataset.seedFile -replace '/', '\')) + if (!(Test-Path -LiteralPath $seedPath -PathType Leaf)) { throw "Pinned seed fixture not found: $seedPath" } + $actualSeedHash = (Get-FileHash -LiteralPath $seedPath -Algorithm SHA256).Hash + if ($actualSeedHash -ne $Dataset.seedFileSha256) { + throw "Seed catalog drift: expected SHA-256 $($Dataset.seedFileSha256), got $actualSeedHash. Review the catalog and update the dataset intentionally." + } + if ([string]::IsNullOrWhiteSpace($DatasetSha256)) { + $DatasetSha256 = (Get-FileHash -LiteralPath (Join-Path $PSScriptRoot 'product-search-relevance.dataset.json') -Algorithm SHA256).Hash + } + + $models = (Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($OllamaAddress, 'api/tags'))).Data.models + $model = @($models | Where-Object { $_.name -eq $Dataset.model -and $_.digest -eq $Dataset.modelDigest }) + if ($model.Count -ne 1) { throw "Ollama model drift: expected $($Dataset.model) with digest $($Dataset.modelDigest)." } + + $schema = (Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($BaseAddress, 'api/products/$query'))).Data + if ($schema.search.mode -ne 'semantic' -or $schema.search.ranked -ne $true) { + throw 'Products API drift: this evaluation requires semantic ranked search.' + } + + $catalogResponse = Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($BaseAddress, 'api/products?$take=100&$count=true')) + $liveCatalog = @($catalogResponse.Data) + $liveCatalogCount = Get-ProductSearchCountHeader $catalogResponse.Headers + if ($liveCatalogCount -ne @($Dataset.catalog).Count -or $liveCatalog.Count -ne @($Dataset.catalog).Count) { + $expectedSkus = @($Dataset.catalog | ForEach-Object sku) + $actualSkus = @($liveCatalog | ForEach-Object sku) + $extraSkus = @($actualSkus | Where-Object { $_ -notin $expectedSkus }) + $missingSkus = @($expectedSkus | Where-Object { $_ -notin $actualSkus }) + throw "Catalog drift: expected $(@($Dataset.catalog).Count) eligible seeded products; API returned $liveCatalogCount total and $($liveCatalog.Count) rows. Extra SKUs: $($extraSkus -join ', '); missing SKUs: $($missingSkus -join ', ')." + } + $liveCatalogSignature = Get-ProductSearchCatalogSignature -Products $liveCatalog + if ($liveCatalogSignature -ne $catalogSignature) { + throw "Catalog drift: live Products API catalog signature $liveCatalogSignature does not match reviewed dataset signature $catalogSignature." + } + + $firstQuery = $Dataset.queries[0] + $firstUri = [uri]::new($BaseAddress, "api/products?`$search=$([uri]::EscapeDataString([string]$firstQuery.query))&`$take=5&`$count=true") + $firstResponse = Get-ProductSearchResponse -Request $Request -Uri $firstUri + $firstExpectedCount = @((Get-ProductSearchEligibleCatalog -Dataset $Dataset -Query $firstQuery)).Count + $firstActualCount = Get-ProductSearchCountHeader $firstResponse.Headers + if ($firstActualCount -ne $firstExpectedCount) { + throw "Search backfill is not ready or is incomplete for profile '$($Dataset.profile)' and recipe '$($Dataset.documentRecipe)': expected $firstExpectedCount indexed candidates, got $firstActualCount." + } + + $reports = [Collections.Generic.List[object]]::new() + foreach ($query in $Dataset.queries) { + if ($query.id -eq $firstQuery.id) { + $response = $firstResponse + } + else { + $relative = "api/products?`$search=$([uri]::EscapeDataString([string]$query.query))&`$take=5&`$count=true" + $filter = Get-ProductSearchFilter $query + if ($filter) { $relative += "&`$filter=$([uri]::EscapeDataString($filter))" } + $response = Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($BaseAddress, $relative)) + } + + $candidateCount = Get-ProductSearchCountHeader $response.Headers + $expectedCandidateCount = @((Get-ProductSearchEligibleCatalog -Dataset $Dataset -Query $query)).Count + if ($candidateCount -ne $expectedCandidateCount) { + throw "Search backfill is incomplete or eligibility drifted for query '$($query.id)' (profile '$($Dataset.profile)', recipe '$($Dataset.documentRecipe)'): expected $expectedCandidateCount candidates, got $candidateCount." + } + + $judgments = @{} + foreach ($judgment in $query.judgments) { $judgments[[string]$judgment.sku] = [int]$judgment.grade } + $ranked = @($response.Data) + $expectedPageCount = [Math]::Min(5, $expectedCandidateCount) + if ($ranked.Count -ne $expectedPageCount) { + throw "Incomplete result page for query '$($query.id)': expected $expectedPageCount rows, got $($ranked.Count)." + } + $knownSkus = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($product in $ranked) { + $sku = [string]$product.sku + if (!(@($Dataset.catalog | Where-Object sku -EQ $sku).Count)) { throw "Catalog drift: search query '$($query.id)' returned unreviewed SKU '$sku'." } + if (!$knownSkus.Add($sku)) { throw "Products API returned duplicate SKU '$sku' for query '$($query.id)'." } + } + $constraintViolations = @($ranked | Where-Object { !(Test-ProductSearchConstraint -Product $_ -Query $query) }) + $grades = @($ranked | ForEach-Object { if ($judgments.ContainsKey([string]$_.sku)) { $judgments[[string]$_.sku] } else { [int]$query.omittedSkuGrade } }) + $idealGrades = @($query.judgments | ForEach-Object { [int]$_.grade }) + $metrics = Get-ProductSearchMetrics -RankedGrades $grades -IdealGrades $idealGrades + $topFive = @($ranked | ForEach-Object { + [pscustomobject]@{ + sku = [string]$_.sku + text = [string]$_.text + price = [decimal]$_.price + grade = if ($judgments.ContainsKey([string]$_.sku)) { $judgments[[string]$_.sku] } else { [int]$query.omittedSkuGrade } + } + }) + $reports.Add([pscustomobject]@{ + id = [string]$query.id + split = [string]$query.split + kind = [string]$query.kind + query = [string]$query.query + intent = [string]$query.intent + expectedTop1Sku = $query.expectedTop1Sku + actualTop1Sku = if ($ranked.Count) { [string]$ranked[0].sku } else { $null } + exactSkuTop1 = if ($query.kind -eq 'exact-sku') { ($ranked.Count -gt 0 -and $ranked[0].sku -eq $query.expectedTop1Sku) } else { $null } + relevantJudgedCount = @($idealGrades | Where-Object { $_ -gt 0 }).Count + noRelevant = !$metrics.HasRelevant + ndcgAt5 = $metrics.NdcgAt5 + precisionAt5 = $metrics.PrecisionAt5 + eligibleCandidateCount = $expectedCandidateCount + constraintViolationCount = $constraintViolations.Count + topFive = $topFive + }) + } + + $splitReports = @{} + foreach ($split in @('tuning', 'held-out')) { + $splitQueries = @($reports | Where-Object split -EQ $split) + $relevantQueries = @($splitQueries | Where-Object { !$_.noRelevant }) + $exactQueries = @($splitQueries | Where-Object { $null -ne $_.exactSkuTop1 }) + $constraintQueries = @($splitQueries | Where-Object { $null -ne $_.eligibleCandidateCount -and $null -ne $_.kind -and $_.kind -eq 'price-constraint' }) + $constraintResultCount = 0 + foreach ($item in $constraintQueries) { $constraintResultCount += $item.topFive.Count } + $violations = ($constraintQueries | Measure-Object -Property constraintViolationCount -Sum).Sum + if ($null -eq $violations) { $violations = 0 } + $splitReports[$split] = [pscustomobject]@{ + queryCount = $splitQueries.Count + relevantQueryCount = $relevantQueries.Count + noRelevantQueryCount = @($splitQueries | Where-Object noRelevant).Count + noRelevantQueryIds = @($splitQueries | Where-Object noRelevant | ForEach-Object id) + outOfCatalogQueryCount = @($splitQueries | Where-Object kind -EQ 'out-of-catalog').Count + meanNdcgAt5 = if ($relevantQueries.Count) { ($relevantQueries | Measure-Object -Property ndcgAt5 -Average).Average } else { $null } + meanPrecisionAt5 = if ($relevantQueries.Count) { ($relevantQueries | Measure-Object -Property precisionAt5 -Average).Average } else { $null } + exactSkuQueryCount = $exactQueries.Count + exactSkuTop1Accuracy = if ($exactQueries.Count) { @($exactQueries | Where-Object exactSkuTop1).Count / [double]$exactQueries.Count } else { $null } + constrainedQueryCount = $constraintQueries.Count + constrainedResultCount = $constraintResultCount + constraintViolationCount = [int]$violations + constraintViolationRate = if ($constraintResultCount) { [int]$violations / [double]$constraintResultCount } else { $null } + } + } + + return [pscustomobject]@{ + evaluationVersion = 'product-search-relevance-runner-v1' + completedAtUtc = [DateTime]::UtcNow.ToString('o') + status = 'completed' + datasetVersion = [string]$Dataset.datasetVersion + model = [string]$Dataset.model + modelDigest = [string]$Dataset.modelDigest + profile = [string]$Dataset.profile + dimensions = [int]$Dataset.dimensions + documentRecipe = [string]$Dataset.documentRecipe + documentInstruction = [string]$Dataset.documentInstruction + queryInstruction = [string]$Dataset.queryInstruction + datasetSha256 = $DatasetSha256 + implementationIdentity = $implementationIdentity + catalogIdentity = [pscustomobject]@{ + seedFile = [string]$Dataset.seedFile + seedFileSha256 = $actualSeedHash + eligibleCatalogCount = $liveCatalogCount + reviewedCatalogSha256 = $catalogSignature + liveApiCatalogSha256 = $liveCatalogSignature + eligibility = [string]$Dataset.catalogEligibility + } + metrics = [pscustomobject]@{ + relevanceThresholdForPrecisionAt5 = 2 + ndcgAt5Gain = '2^grade - 1; logarithmic rank discount; judged grades 0-3' + precisionAt5 = 'Relevant means grade >= 2; denominator is always 5.' + noRelevantQueries = 'Reported separately and excluded from relevance-metric means.' + exactSkuTop1 = 'Reported only for exact-SKU queries.' + constraints = 'Returned price-filtered results are validated against the declared hard price condition; no threshold is imposed on relevance scores.' + } + splits = $splitReports + queries = @($reports) + } +} + +if ($MyInvocation.InvocationName -ne '.') { + Test-ProductSearchDataset (Get-ProductSearchDataset $DatasetPath) | Out-Null + $dataset = Get-ProductSearchDataset $DatasetPath + $datasetHash = (Get-FileHash -LiteralPath $DatasetPath -Algorithm SHA256).Hash + $rootPath = Join-Path $PSScriptRoot ("..\{0}" -f ($dataset.seedFile -replace '/', '\')) + if (!(Test-Path -LiteralPath $rootPath -PathType Leaf)) { throw "Pinned seed fixture not found: $rootPath" } + $actualSeedHash = (Get-FileHash -LiteralPath $rootPath -Algorithm SHA256).Hash + if ($actualSeedHash -ne $dataset.seedFileSha256) { + throw "Seed catalog drift: expected SHA-256 $($dataset.seedFileSha256), got $actualSeedHash. Review the catalog and update the dataset intentionally." + } + + $request = { param([uri] $Uri) Invoke-WebRequest -Uri $Uri -TimeoutSec 120 } + $report = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress $BaseAddress -OllamaAddress $OllamaAddress -Request $request -DatasetSha256 $datasetHash + $json = ConvertTo-Json -InputObject $report -Depth 30 + if ($OutputPath) { + $outputFullPath = [IO.Path]::GetFullPath($OutputPath) + [IO.File]::WriteAllText($outputFullPath, $json + [Environment]::NewLine, [Text.UTF8Encoding]::new($false)) + Write-Host "Wrote real-model product-search report to $outputFullPath" + } + else { Write-Output $json } +} diff --git a/tools/product-search-relevance.seed.yaml b/tools/product-search-relevance.seed.yaml new file mode 100644 index 000000000..0edc5a1da --- /dev/null +++ b/tools/product-search-relevance.seed.yaml @@ -0,0 +1,45 @@ +products: + - product: + - { product_id: ^1, sku: YETI-ASR-C2-2025, text: Yeti ASR C2, sub_category_code: XC, unit_of_measure_code: EA, price: 5800, brand_code: YETI, tags_json: '["cross-country","full-suspension"]', description: 'A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack.' } + - { product_id: ^2, sku: YETI-SB120-C2, text: Yeti SB120 C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: YETI, description: 'A carbon full-suspension trail mountain bike designed for efficient climbs, responsive handling and flowing singletrack descents.' } + - { product_id: ^3, sku: YETI-SB140-LR-C2, text: Yeti SB140 LR C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6600, brand_code: YETI, description: 'A carbon full-suspension trail mountain bike in the Lunch Ride configuration, built for technical climbs and rough, fast descents.' } + - { product_id: ^4, sku: YETI-SB165-C3, text: Yeti SB165 C3, sub_category_code: EN, unit_of_measure_code: EA, price: 7200, brand_code: YETI, description: 'A carbon full-suspension mountain bike for gravity-focused riding, steep descents and demanding bike-park terrain.' } + - { product_id: ^5, sku: YETI-MTE-C2, text: Yeti MTe C2 eMTB, sub_category_code: EM, unit_of_measure_code: EA, price: 9850, brand_code: YETI, description: 'An electric-assist mountain bike with a carbon full-suspension frame, designed for technical trail riding and assisted climbs.' } + - { product_id: ^6, sku: CANYON-LUXTRAIL-CF6, text: Canyon Lux Trail CF 6, sub_category_code: XC, unit_of_measure_code: EA, price: 3899, brand_code: CANYON, description: 'A carbon full-suspension mountain bike combining cross-country pedalling efficiency with trail capability for long rides and technical singletrack.' } + - { product_id: ^7, sku: CANYON-SPECTRAL-CF8, text: Canyon Spectral CF 8, sub_category_code: TR, unit_of_measure_code: EA, price: 4999, brand_code: CANYON, description: 'A carbon full-suspension trail mountain bike for varied singletrack, balancing climbing capability with control on rough, technical descents.' } + - { product_id: ^8, sku: CANYON-NEURON-CF7, text: Canyon Neuron CF 7, sub_category_code: TR, unit_of_measure_code: EA, price: 3499, brand_code: CANYON, description: 'A carbon full-suspension trail mountain bike for long rides over varied terrain, combining efficient climbing with stable descending.' } + - { product_id: ^9, sku: CANYON-TORQUE-MULLET-CF8, text: Canyon Torque Mullet CF 8, sub_category_code: EN, unit_of_measure_code: EA, price: 3599, brand_code: CANYON, description: 'A carbon mixed-wheel mountain bike designed for freeride and bike-park use, including jumps, drops and steep descents.' } + - { product_id: ^10, sku: CANYON-SENDER-CFR, text: Canyon Sender CFR, sub_category_code: DH, unit_of_measure_code: EA, price: 7799, brand_code: CANYON, description: 'A carbon downhill mountain bike designed for racing and fast descending on steep, demanding tracks and bike-park terrain.' } + - { product_id: ^11, sku: SPEC-STUMPJUMPER-15-COMP, text: Specialized Stumpjumper 15 Comp, sub_category_code: TR, unit_of_measure_code: EA, price: 5500, brand_code: SPEC, description: 'A full-suspension trail mountain bike with adjustable geometry, designed to balance climbing efficiency and control on technical descents.' } + - { product_id: ^12, sku: SPEC-ENDURO-PRO, text: Specialized Enduro Pro, sub_category_code: EN, unit_of_measure_code: EA, price: 9500, brand_code: SPEC, description: 'A carbon full-suspension enduro mountain bike built for steep, rough descents and enduro racing while retaining the ability to pedal uphill.' } + - { product_id: ^13, sku: SPEC-EPIC-8-PRO, text: Specialized Epic 8 Pro, sub_category_code: XC, unit_of_measure_code: EA, price: 9000, is_inactive: true, brand_code: SPEC, description: 'A carbon full-suspension cross-country mountain bike designed for efficient pedalling, competitive racing and technical singletrack.' } + - { product_id: ^14, sku: SPEC-CHISEL-COMP, text: Specialized Chisel Comp, sub_category_code: HT, unit_of_measure_code: EA, price: 3400, brand_code: SPEC, description: 'An aluminium cross-country mountain bike designed for efficient pedalling, racing and fast singletrack riding.' } + - { product_id: ^15, sku: SPEC-STATUS-140, text: Specialized Status 140, sub_category_code: FR, unit_of_measure_code: EA, price: 2499, brand_code: SPEC, description: 'An aluminium mixed-wheel full-suspension mountain bike designed for playful trail riding, jumps and technical descents.' } + - { product_id: ^16, sku: GIANT-TRANCE-X-ADVPRO29, text: Giant Trance X Advanced Pro 29, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: GIANT, description: 'A carbon full-suspension trail mountain bike with 29-inch wheels and adjustable geometry, designed for climbing and technical descending.' } + - { product_id: ^17, sku: GIANT-ANTHEM-ADV29-1, text: Giant Anthem Advanced 29 1, sub_category_code: XC, unit_of_measure_code: EA, price: 7800, brand_code: GIANT, description: 'A carbon full-suspension cross-country mountain bike with 29-inch wheels, designed for efficient pedalling and fast riding on demanding race courses.' } + - { product_id: ^18, sku: GIANT-TALON-2, text: Giant Talon 2, sub_category_code: HT, unit_of_measure_code: EA, price: 800, is_deleted: true, brand_code: GIANT, description: 'An aluminium hardtail mountain bike with a suspension fork, designed as an accessible introduction to off-road trails.' } + - { product_id: ^19, sku: SHIMANO-XT-M8100-RD, text: Shimano Deore XT M8100 Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 129, brand_code: SHIMANO, description: 'A 12-speed mountain bike rear derailleur that moves the chain between cassette sprockets to change gears. Its clutch helps control chain movement on rough trails.' } + - { product_id: ^20, sku: SHIMANO-SLX-M7100-CASSETTE, text: Shimano SLX M7100 12-speed Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 99, brand_code: SHIMANO, description: 'A 12-speed mountain bike cassette that provides a wide range of gears for climbing and descending. It uses Shimano Micro Spline mounting and Hyperglide Plus shifting.' } + - { product_id: ^21, sku: SHIMANO-XTR-M9100-CRANK, text: Shimano XTR M9100 Crankset, sub_category_code: CR, unit_of_measure_code: EA, price: 499, brand_code: SHIMANO, description: 'A mountain bike crankset that transfers pedalling power to the chain through a direct-mount chainring, designed for lightweight cross-country and trail use.' } + - { product_id: ^22, sku: SRAM-GX-EAGLE-RD, text: SRAM GX Eagle Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 135, brand_code: SRAM, description: 'A cable-operated 12-speed mountain bike rear derailleur for SRAM Eagle drivetrains. It shifts the chain between rear sprockets and uses a clutch to control chain movement.' } + - { product_id: ^23, sku: SRAM-X0-TRANSMISSION-RD, text: SRAM X0 Transmission Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 400, brand_code: SRAM, description: 'A wireless electronic mountain bike rear derailleur for SRAM Eagle Transmission drivetrains. Its hangerless Full Mount design attaches around the rear axle on compatible frames.' } + - { product_id: ^24, sku: SRAM-XX1-EAGLE-CASSETTE, text: SRAM XX1 Eagle Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 449, brand_code: SRAM, description: 'A lightweight 12-speed mountain bike cassette with a wide gear range for climbing and descending, designed for SRAM Eagle drivetrains and XD freehub drivers.' } + - { product_id: ^25, sku: ONEUP-V3-DROPPER-180, text: OneUp Dropper Post V3 180mm, sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP, description: 'A mountain bike dropper seatpost with 180 mm of travel. Riders can lower the saddle for descending and raise it for pedalling using a compatible remote.' } + - { product_id: ^26, sku: ONEUP-CARBON-HANDLEBAR, text: OneUp Carbon Handlebar, sub_category_code: HB, unit_of_measure_code: EA, price: 159, brand_code: ONEUP, description: 'A carbon mountain bike handlebar with a shaped profile designed to absorb vibration and reduce hand fatigue while maintaining steering control.' } + - { product_id: ^27, sku: ONEUP-COMPOSITE-PEDALS, text: OneUp Composite Pedals, sub_category_code: PE, unit_of_measure_code: PR, price: 49, is_inactive: true, brand_code: ONEUP, description: 'Flat mountain bike pedals with composite bodies, broad platforms and replaceable traction pins, designed to grip ordinary flat-soled riding shoes.' } + - { product_id: ^28, sku: ONEUP-EDC-TOOL-V2, text: OneUp EDC Tool V2, sub_category_code: TO, unit_of_measure_code: EA, price: 59, brand_code: ONEUP, description: 'A compact trailside repair multi-tool with hex keys, a Torx key, a chain breaker and a tyre lever, designed for storage in compatible OneUp steerer or pump systems.' } + - { product_id: ^29, sku: MUCOFF-NANO-CLEANER, text: Muc-Off Nano Tech Bike Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 16.99, brand_code: MUCOFF, description: 'A spray-on bicycle cleaner that breaks down dirt and grime on frames and components before rinsing. Suitable for routine washing after muddy rides.' } + - { product_id: ^30, sku: MUCOFF-TUBELESS-SEALANT, text: Muc-Off MTB Tubeless Sealant, sub_category_code: SE, unit_of_measure_code: EA, price: 8.00, brand_code: MUCOFF, description: 'A tubeless mountain bike tyre sealant that helps seal punctures and maintain an airtight tyre system. Its UV-reactive dye helps locate leaks.' } + - { product_id: ^31, sku: MUCOFF-DRIVETRAIN-CLEANER, text: Muc-Off Drivetrain Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 25.00, brand_code: MUCOFF, description: 'A bicycle drivetrain cleaner that removes grease, oil and grime from chains, cassettes, chainrings and derailleurs. It can be sprayed on or used with a chain-cleaning device.' } + - { product_id: ^32, sku: LABOR, text: Labor, sub_category_code: LA, unit_of_measure_code: HR, price: 80.00, brand_code: NONE, is_non_stocked: true, description: 'Workshop labour charged by the hour for bicycle maintenance and repair. This is a service rather than a stocked physical product.' } + - inventory: + - { inventory_id: ^1, qty_on_hand: 3 } + - { inventory_id: ^16, qty_on_hand: 1 } + - { inventory_id: ^24, qty_on_hand: 3 } + - { inventory_id: ^28, qty_on_hand: 5 } + - movement: + - { movement_id: ^1, product_id: ^1, quantity: -2, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } + - { movement_id: ^2, product_id: ^6, quantity: -1, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } + - { movement_id: ^3, product_id: ^31, quantity: -3, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } + - { movement_id: ^4, product_id: ^24, quantity: -2, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } + - { movement_id: ^5, product_id: ^28, quantity: -1, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } diff --git a/tools/test-product-search-relevance.ps1 b/tools/test-product-search-relevance.ps1 new file mode 100644 index 000000000..793692a23 --- /dev/null +++ b/tools/test-product-search-relevance.ps1 @@ -0,0 +1,140 @@ +$ErrorActionPreference = 'Stop' +. "$PSScriptRoot\product-search-relevance.ps1" + +function Assert-Equal { + param([object] $Actual, [object] $Expected, [string] $Message) + if ($Actual -ne $Expected) { throw "$Message Expected '$Expected', got '$Actual'." } +} + +function Assert-Throws { + param([scriptblock] $Action, [string] $Message) + try { & $Action } catch { return } + throw $Message +} + +$dataset = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" +Test-ProductSearchDataset $dataset | Out-Null + +$metrics = Get-ProductSearchMetrics -RankedGrades @(3, 2, 0) -IdealGrades @(3, 2, 1) +$expectedNdcg = (7 + 3 / [Math]::Log(3, 2)) / (7 + 3 / [Math]::Log(3, 2) + 1 / [Math]::Log(4, 2)) +if ([Math]::Abs($metrics.NdcgAt5 - $expectedNdcg) -gt 0.0000001) { throw 'nDCG@5 gain/discount math is incorrect.' } +Assert-Equal $metrics.PrecisionAt5 0.4 'Precision@5 must count grade 2 or 3 in five fixed slots.' +$emptyMetrics = Get-ProductSearchMetrics -RankedGrades @(0, 0) -IdealGrades @() +Assert-Equal $emptyMetrics.HasRelevant $false 'A query with no relevant catalog items must be identified separately.' +if ($null -ne $emptyMetrics.NdcgAt5 -or $null -ne $emptyMetrics.PrecisionAt5) { throw 'Metrics for a no-relevant query must be undefined, not zero-valued quality.' } +$marginalMetrics = Get-ProductSearchMetrics -RankedGrades @(1) -IdealGrades @(1) +Assert-Equal $marginalMetrics.HasRelevant $true 'A marginal grade remains useful graded relevance for nDCG.' +Assert-Equal $marginalMetrics.PrecisionAt5 0 'Precision@5 must retain the explicit grade-2 threshold.' + +$invalid = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" +$invalid.queries[0].judgments[0].sku = 'NOT-IN-CATALOG' +Assert-Throws { Test-ProductSearchDataset $invalid } 'Unknown judged SKUs must be rejected.' +$invalid = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" +$invalid.catalog[1].sku = $invalid.catalog[0].sku +Assert-Throws { Test-ProductSearchDataset $invalid } 'Duplicate catalog SKUs must be rejected.' +$invalid = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" +$invalid.queries[0].kind = 'unreviewed-kind' +Assert-Throws { Test-ProductSearchDataset $invalid } 'Unsupported query kinds must be rejected.' +$invalid = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" +$constrainedQuery = $invalid.queries | Where-Object id -EQ 'tune-price-derailleur-under-150' +$constrainedQuery.judgments[0].sku = 'SHIMANO-XTR-M9100-CRANK' +Assert-Throws { Test-ProductSearchDataset $invalid } 'Judgments violating hard query constraints must be rejected.' + +$validMock = { + param([uri] $Uri) + $path = $Uri.AbsolutePath + if ($path -eq '/api/tags') { + $content = @{ models = @(@{ name = $dataset.model; digest = $dataset.modelDigest }) } | ConvertTo-Json -Depth 8 -Compress + return [pscustomobject]@{ StatusCode = 200; Content = $content; Headers = @{} } + } + if ($path -eq '/api/products/$query') { + $content = @{ search = @{ mode = 'semantic'; ranked = $true } } | ConvertTo-Json -Depth 8 -Compress + return [pscustomobject]@{ StatusCode = 200; Content = $content; Headers = @{} } + } + if ($path -ne '/api/products') { throw "Unexpected mocked URI: $Uri" } + if (!$Uri.Query.Contains('$search=')) { + $items = @($dataset.catalog | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) + return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $items -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = "$($items.Count)" } } + } + $queryString = $Uri.Query.TrimStart([char]'?') + $searchPart = $queryString -split '&' | Where-Object { $_.StartsWith('$search=') } | Select-Object -First 1 + $queryText = [uri]::UnescapeDataString($searchPart.Substring(8)) + $query = $dataset.queries | Where-Object query -EQ $queryText | Select-Object -First 1 + $eligible = @($dataset.catalog) + if ($null -ne $query.constraints -and $null -ne $query.constraints.price) { + [decimal]$limit = $query.constraints.price.value + $eligible = @($eligible | Where-Object { + [decimal]$price = $_.price + switch ($query.constraints.price.operator) { + 'eq' { $price -eq $limit } + 'lt' { $price -lt $limit } + 'le' { $price -le $limit } + 'gt' { $price -gt $limit } + 'ge' { $price -ge $limit } + } + }) + } + $results = @($eligible | Select-Object -First 5 | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) + return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $results -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = "$($eligible.Count)" } } +}.GetNewClosure() + +$mockReport = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $validMock +Assert-Equal $mockReport.queries.Count 24 'The mocked end-to-end evaluation must exercise all curated queries.' +Assert-Equal $mockReport.splits.'held-out'.queryCount 6 'Tuning and held-out aggregates must remain split.' + +$shortPageMock = { + param([uri] $Uri) + if ($Uri.AbsolutePath -eq '/api/products' -and $Uri.Query.Contains('$search=')) { + $items = @($dataset.catalog | Select-Object -First 1 | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) + return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $items -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = "$($dataset.catalog.Count)" } } + } + & $validMock $Uri +}.GetNewClosure() +Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $shortPageMock } 'Incomplete search result pages must be rejected.' + +$driftMock = { + param([uri] $Uri) + if ($Uri.AbsolutePath -eq '/api/products' -and !$Uri.Query.Contains('$search=')) { + $items = @($dataset.catalog | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) + $items[0].text = 'Changed catalog item' + return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $items -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = "$($items.Count)" } } + } + & $validMock $Uri +}.GetNewClosure() +Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $driftMock } 'Catalog drift must stop evaluation.' + +$backfillMock = { + param([uri] $Uri) + if ($Uri.AbsolutePath -eq '/api/products' -and $Uri.Query.Contains('$search=')) { + return [pscustomobject]@{ StatusCode = 200; Content = '[]'; Headers = @{ 'X-Paging-Total-Count' = '28' } } + } + & $validMock $Uri +}.GetNewClosure() +Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $backfillMock } 'A ready flag with an incomplete search projection must fail the full-catalog count check.' + +$violationMock = { + param([uri] $Uri) + if ($Uri.AbsolutePath -eq '/api/products' -and $Uri.Query.Contains('$search=')) { + $queryString = $Uri.Query.TrimStart([char]'?') + $searchPart = $queryString -split '&' | Where-Object { $_.StartsWith('$search=') } | Select-Object -First 1 + $queryText = [uri]::UnescapeDataString($searchPart.Substring(8)) + if ($queryText -eq 'a rear derailleur for a 12-speed mountain bike under $150') { + $crank = $dataset.catalog | Where-Object sku -EQ 'SHIMANO-XTR-M9100-CRANK' + $eligible = @($dataset.catalog | Where-Object { [decimal]$_.price -lt 150 } | Select-Object -First 4) + $results = @(@($crank) + $eligible | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) + return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $results -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = '8' } } + } + } + & $validMock $Uri +}.GetNewClosure() +$violationReport = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $violationMock +$violationQuery = $violationReport.queries | Where-Object id -EQ 'tune-price-derailleur-under-150' +Assert-Equal $violationQuery.constraintViolationCount 1 'Hard price-filter violations must be measured rather than hidden.' +Assert-Equal $violationReport.splits.tuning.constraintViolationCount 1 'Constraint violations must be included in split aggregates.' + +$failedMock = { param([uri] $Uri) throw 'Mocked HTTP outage.' }.GetNewClosure() +Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $failedMock } 'HTTP failures must stop evaluation.' +$unavailableMock = { param([uri] $Uri) [pscustomobject]@{ StatusCode = 503; Content = '{"title":"Unavailable"}'; Headers = @{} } }.GetNewClosure() +Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $unavailableMock } 'Service-unavailable responses must stop evaluation.' + +Write-Host 'Product-search relevance tests passed: metric math, dataset validation, catalog drift, backfill completeness, constraint violations, and mocked HTTP failures.' From fe1830be472fe5e1ddfc6d3c7695351d9adf8459 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Thu, 15 Oct 2026 14:00:47 -0700 Subject: [PATCH 10/25] Add lexical product search and separate query repository Introduce independently maintained PostgreSQL full-text retrieval with exact-SKU priority and tuning-only relevance evaluation. Separate product query orchestration from CRUD and centralize server search-mode resolution. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- samples/docs/search.md | 129 ++++++++- .../src/Contoso.Mcp/Tools/CommerceTools.cs | 6 +- samples/src/Contoso.Products.Api/Program.cs | 2 +- .../Interfaces/IProductReadService.cs | 8 +- .../ProductReadService.cs | 14 +- .../Repositories/IProductQueryRepository.cs | 11 + .../Repositories/IProductRepository.cs | 16 +- ...15-175620-add-product-lexical-search.pgsql | 15 + .../Embeddings/ProductSearchSyncAdapter.cs | 7 +- .../Repositories/ProductLexicalSearchIndex.cs | 71 +++++ .../Repositories/ProductQueryArgsConfig.cs | 27 +- .../Repositories/ProductQueryRepository.cs | 110 +++++++ .../Repositories/ProductRepository.cs | 82 +----- .../Repositories/ProductSearchIndex.cs | 7 +- .../Sql/ProductLexicalSearchIsReady.pgsql | 1 + .../Sql/ProductLexicalSearchLock.pgsql | 2 + .../Sql/ProductLexicalSearchMarkReady.pgsql | 3 + .../Sql/ProductLexicalSearchMatches.pgsql | 6 + .../Sql/ProductLexicalSearchRefresh.pgsql | 14 + .../ProductLexicalSearchTests.cs | 269 ++++++++++++++++++ .../ProductSearchRelevanceSetupTests.cs | 19 +- .../ReadTests.ProductSemanticSearch.cs | 5 +- .../ReadTests/Product_Query_Schema.res.json | 1 + .../ProductLexicalSearchSubscriberTests.cs | 61 ++++ .../ProductSearchTests.cs | 4 + tools/product-search-relevance.ps1 | 106 +++++-- tools/test-product-search-relevance.ps1 | 25 ++ 27 files changed, 869 insertions(+), 152 deletions(-) create mode 100644 samples/src/Contoso.Products.Application/Repositories/IProductQueryRepository.cs create mode 100644 samples/src/Contoso.Products.Database/Migrations/20261015-175620-add-product-lexical-search.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/ProductLexicalSearchIndex.cs create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchIsReady.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchLock.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchMarkReady.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchMatches.pgsql create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchRefresh.pgsql create mode 100644 samples/tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs create mode 100644 samples/tests/Contoso.Products.Test.Subscribe/ProductLexicalSearchSubscriberTests.cs diff --git a/samples/docs/search.md b/samples/docs/search.md index e2f927a0e..32053a789 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -18,12 +18,13 @@ The goal is not merely to store some numbers and label the result "AI". It is to | Product descriptions and price filtering | Implemented | Optional descriptive content supports CRUD; decimal price constraints use existing REST and GraphQL-lite query configuration. | | Local embedding provider | Implemented | Ollama/Nomic document and query instructions, response validation, and an Application adapter boundary are available. | | Separate search projection and pgvector semantic retrieval | Implemented | PostgreSQL stores 768-dimensional embeddings and computes exact, filtered cosine-distance rankings. | -| PostgreSQL full-text retrieval and hybrid ranking | Proposed | The current sample does not perform full-text or hybrid search. | +| Independent PostgreSQL full-text retrieval | Implemented; opt-in server mode | Weighted English `tsvector`/GIN matching, safe websearch parsing, exact eligible SKU first, no embedding calls. | +| Hybrid ranking / RRF | Proposed | The two retrieval branches are not fused. | | Explicit backfill / asynchronous search indexing | Implemented | Test setup publishes an atomic initial backfill; one globbed product-event subscriber reconciles subsequent changes, including logical deletion. | | Real-model relevance evaluation | Baseline measured; monthly/manual workflow configured | A version-controlled, 24-query judged set, isolated-catalog setup, opt-in REST/Ollama runner, and real report are available. Scores are observations, not quality gates. | | HNSW benchmark | Proposed | Exact full-catalog retrieval is still the comparison baseline; index performance has not been measured. | -The implemented baseline is PostgreSQL as the search store, one embedding per product/profile/recipe, a rebuildable search projection, and indexing outside product-write transactions. It uses local Ollama with Nomic v1.5 and 768 dimensions. Exact retrieval ranks eligible **indexed** products; count means the number of those candidates, not the number judged relevant or the complete catalog size. Full-text/hybrid retrieval windows remain future work. +The default baseline is PostgreSQL as the search store, one embedding per product/profile/recipe, a rebuildable search projection, and indexing outside product-write transactions. It uses local Ollama with Nomic v1.5 and 768 dimensions. Exact semantic retrieval ranks eligible **indexed** products; count means the number of those candidates, not the number judged relevant or the complete catalog size. The independent full-text branch counts only lexical matches (including exact SKUs). Hybrid retrieval windows remain future work. ## Reading map @@ -96,7 +97,7 @@ This should retrieve the most relevant eligible Yeti products, not search the en Keyword mode allows ordinary ordering. Ranked modes own relevance ordering: CoreEx rejects an explicit `$orderby` and does not apply the configured default ordering. -The server configuration selects the mode; Products now selects semantic mode. There is no public `$searchMode` parameter. Semantic text is limited to 200 normalized characters, but the keyword-only eight-term limit no longer applies. +The server configuration selects the mode; Products defaults to semantic mode. Set `Products:Search:Mode` to `FullText` (environment variable `Products__Search__Mode=FullText`) to exercise lexical retrieval instead. The only supported values are `Semantic` and `FullText`; an unsupported value fails explicitly. There is no public `$searchMode` parameter or test-only endpoint. Both ranked modes limit text to 200 normalized characters; the keyword-only eight-term limit does not apply. REST query schema and GraphQL-lite use the same selected configuration and unchanged filter fields. Price, availability, and other hard constraints belong in structured filtering, not in assumptions about what an embedding understands. Products exposes a decimal `price` filter with `eq`, `ne`, `in`, `gt`, `ge`, `lt` and `le`; inventory filters are not currently exposed. @@ -345,7 +346,7 @@ This is the implemented semantic flow. Requests without `$search` do not call th **Products implementation responsibilities:** prepare documents, generate embeddings, execute provider-specific queries, enforce eligibility, maintain derived state, and evaluate relevance/performance. -The existing `ProductRepository.QueryAsync` joins product, subcategory, and inventory data. Ranked retrieval must preserve that behavior and all configured filters rather than quietly implementing a reduced API. +`ProductQueryRepository.QueryAsync` joins product, subcategory, and inventory data. Ranked retrieval must preserve that behavior and all configured filters rather than quietly implementing a reduced API. Query configuration, semantic and lexical retrieval, and snapshot-isolation checks stay together in this query repository; the CRUD/reservation `ProductRepository` depends only on `ProductsEfDb`. `ProductReadService` delegates `GetAsync` to the CRUD repository and query operations to `IProductQueryRepository`, so its by-id read still resolves both dependencies. Application orchestration depends on application-owned interfaces. `IProductEmbeddingAdapter` owns the model boundary; `IProductSearchSyncAdapter` exposes current-state reconciliation to the subscriber. Their implementations live in Infrastructure, where `ProductSearchIndex` handles derived storage and backfills. @@ -357,7 +358,7 @@ MCP (Model Context Protocol) lets an AI client discover and invoke tools such as The tool sends search text to Products and returns product data. It does not generate its own embeddings, query PostgreSQL directly, or maintain its own index. Non-MCP clients use the same capability. -The tool description now promises semantic candidate ordering, not literal keyword or exact-SKU matching. Price/brand filters remain available in REST/GraphQL-lite but are not yet arguments on the MCP tool. +The tool description defers to the Products server's configured ranked mode: semantic by default, or lexical matches when explicitly selected. It does not promise exact-SKU matching in semantic mode. Price/brand filters remain available in REST/GraphQL-lite but are not yet arguments on the MCP tool. Search does not resolve authentication or authorization. The sample's existing authentication gaps remain separate concerns. @@ -398,11 +399,10 @@ The rebuild indexes the current non-deleted catalog, including inactive products not seed products or download the model. It prepares documents and calls Ollama **before** starting a database transaction when invoked outside an existing transaction. Only publication takes a short framework-managed unit of work: lock the product table against writes, verify that its searchable documents still match the captured -catalog, replace the selected profile/recipe, and commit. A model failure leaves the previous index untouched; +catalog, replace the selected profile/recipe, and commit. A model failure leaves the previous vector index untouched; a concurrent searchable catalog edit rejects publication. Deleted-product entries disappear on rebuild, and -physical deletion also cascades through the foreign key. Other profiles remain separate. If a consumer calls -rebuild inside an existing unit of work, it inherits that transaction and publication remains subject to the -parent's eventual commit or rollback. +physical deletion also cascades through the foreign key. Other profiles remain separate. Rebuild and synchronization +reject a parent transaction: the lexical publication must commit independently before any model work. `products.product_search_readiness` records successful initial preparation for each profile/recipe. The readiness marker commits atomically with backfill publication, even for an empty catalog. Incremental events never mark a partially indexed catalog ready. A failed initial backfill leaves it unready; a failed later backfill preserves the previous published projection and readiness. @@ -427,6 +427,60 @@ API integration tests rebuild with controlled vectors, isolating distance orderi There is no relevance cutoff: every eligible compatible vector is a candidate. `$count=true` reports eligible indexed candidates, including poor semantic matches and excluding not-yet-indexed products. An unrelated query still returns neighbors; clients must not interpret a returned product as guaranteed suitable. +### Independent lexical projection + +`ProductLexicalSearchIndex` owns `products.product_lexical_search` and its separate +`product_lexical_search_readiness` table. Both are SQL-only derived state managed by the new DbEx migration, +not generated CRUD models. There is exactly one row per product. Recipe `product-lexical-english-v1` uses +explicit `pg_catalog.english` processing on both documents and queries: + +- Weight A: canonical SKU and product name. +- Weight B: brand/category/subcategory labels (code fallback) and existing JSON tags. +- Weight C: description. + +The document is a PostgreSQL `tsvector`; its GIN index supports lexical matching. A separate SKU B-tree +supports the exact-match alternative. `websearch_to_tsquery` receives bound text, not interpolated SQL or +raw `to_tsquery` syntax. English stemming and stopwords apply; punctuation is parsed by PostgreSQL. +Ordinary words are ANDed; quoted phrases, `OR`, and exclusions follow PostgreSQL websearch semantics. +This is not BM25, substring matching, fuzzy spelling correction, synonym expansion, or semantic retrieval. + +The entire normalized input is also compared with the stored SKU after trimming and the contract's existing +uppercase transformation. An exact eligible SKU is included even if tokenization removes it (e.g. a stopword +SKU). Ranking is exact-SKU first, then descending `ts_rank_cd` using PostgreSQL's default weight values and +no length normalization, then product ID. The OR predicate does not union duplicate rows. An exact but +inactive, deleted, or filtered-out SKU cannot bypass live eligibility. + +The repository composes these matches with the **same** product/subcategory/inventory joins and all existing +filters. It counts and pages the complete matching set in a `RepeatableRead` snapshot; compatible parent +snapshots enlist, weaker ones fail explicitly. An explicit `$orderby` is rejected. No-match, stopword-only, +and punctuation-only inputs return an empty set, not semantic neighbors or the whole catalog. An unprepared +lexical recipe returns HTTP 503. Cancellation propagates. Existing `QueryArgs` behavior is unchanged: +an empty/whitespace `$search` is treated as absent and performs an ordinary catalog query, not lexical matching. + +Lexical synchronization reads current authoritative state and publishes it in one short transaction. +It shares the product source-write lock, also locks reference-data sources during publication, and serializes +lexical publishers. Delete/reinsert is atomic to readers; duplicate or late events reconcile current state. +The complete lexical rebuild replaces all rows and publishes readiness in that **same** transaction, including +an empty catalog. Individual events cannot mark a partly indexed initial catalog ready. Cancellation/failure +rolls back publication and preserves any prior complete version. + +In default semantic mode, `ProductSearchIndex.SynchronizeAsync` and `RebuildAsync` first **commit** lexical +work and only then call the embedding provider. A semantic failure still retries the event, but cannot roll back +already-published lexical content/readiness. In explicitly configured FullText Subscribe mode, the application +adapter calls **only** the lexical index: an offline model does not cause event retries or prevent later lexical +updates. Configure API and Subscribe consistently. No background poller, automatic startup backfill, new service, +or new package is introduced. + +For a lexical-only initial backfill call `ProductLexicalSearchIndex.RebuildAsync` from a maintenance/test-owned +scope. The guarded evaluation fixture below selects that path when `Products__Search__Mode=FullText`. +Switching an installation that has run lexical-only maintenance back to Semantic requires a semantic rebuild: +FullText mode does not maintain vector rows. Changing the lexical document/configuration recipe also requires a +version bump and full rebuild. Readiness means a completed initial build, **not** a proof of continuous freshness. +Events can lag or be missed; live filters still exclude ineligible rows but term matching can temporarily reflect +older content. Reference-data-only edits have no automatic reindex trigger; rebuild explicitly. Lexical labels +come directly from database text, not localized request/cache values; English is the sole supported language. +The table locks/full rebuild are deliberately small-catalog sample choices, not a large-catalog online rebuild design. + ### Asynchronous maintenance Product CRUD already commits product mutations and outgoing events together using the transactional outbox. @@ -699,7 +753,57 @@ The captured [`baseline report`](product-search-relevance-baseline.json) is one Routine reports belong in the git-ignored `artifacts/` directory, not in the checked-in baseline. Replacing `samples/docs/product-search-relevance-baseline.json` is an explicit review step: compare the new report's per-query results, aggregate metrics, and model/dataset/catalog/implementation identities with the existing baseline; explain why the replacement is appropriate in the change description; then deliberately copy the reviewed report over the baseline and update the documented findings. Git history preserves the previous baseline. Do not automatically promote scheduled reports or replace the baseline merely because a score decreased. -Rerun and review the benchmark when the embedding model/profile, document recipe or text fields, ranking/retrieval implementation, judged catalog/seed data, or judgment set changes. Preserve the prior report for comparison and record the changed identity; do not silently replace labels. When PostgreSQL full-text/RRF is implemented, run the same pinned dataset/model and compare per-query rankings and both splits against this semantic baseline; treat held-out as a final comparison, not a tuning target. Any new catalog revision requires reviewing all eligible products for every query before updating the pinned catalog identity. +Rerun and review the benchmark when the embedding model/profile, document recipe or text fields, ranking/retrieval implementation, judged catalog/seed data, or judgment set changes. Preserve the prior report for comparison and record the changed identity; do not silently replace labels. Full-text tuning compares only the tuning split with this semantic baseline; held-out execution is deferred until an explicitly approved final comparison, not used as a tuning target. Any new catalog revision requires reviewing all eligible products for every query before updating the pinned catalog identity. + +### Lexical tuning-only comparison + +After the guarded `contoso_search_eval` Reset/Data/Execute sequence above, set the isolated connection and +FullText mode for the indexing fixture and the separately started Products API: + +```powershell +$env:Aspire__Npgsql__ConnectionString = $connectionString +$env:Products__Search__Mode = 'FullText' +dotnet test samples\tests\Contoso.Products.Test.Api --no-build --framework net10.0 --filter 'FullyQualifiedName~ProductSearchRelevanceSetupTests' +# Start the API separately with the same environment and your chosen BaseAddress. +.\tools\product-search-relevance.ps1 -BaseAddress http://localhost:5217/ -Approach fulltext -Split tuning ` + -BaselinePath samples\docs\product-search-relevance-baseline.json ` + -OutputPath artifacts\product-search-lexical-tuning.json +``` + +This evaluator makes **no Ollama requests** in full-text mode, identifies the lexical recipe and embedded SQL +hashes, and selects tuning queries **before issuing any search requests**. Full-text without `-Split tuning` +is rejected at this milestone. Semantic evaluation retains its original full-split default; `-Split tuning` +also supports a semantic tuning-only run. Baseline comparison validates dataset/catalog/seed identities and +tuning query IDs and emits aggregate deltas without replacing the original baseline. No judgments change. + +Lexical counts are matching-set counts, not semantic full-catalog candidate counts. The runner checks that +they are bounded by the live eligible catalog and that page size agrees with count. It cannot infer lexical +index completeness from a naturally sparse/no-match result; the guarded host fixture performs the actual full +backfill and verifies its 31 non-deleted source rows (29 active). No fake “every product must match” completeness +probe is sent to the lexical branch. + +Measured locally on **2026-10-15**, through the real FullText REST API over the isolated, dedicated seed: + +| Tuning metric | Original semantic baseline | Independent lexical v1 | +|---|---:|---:| +| Mean nDCG@5 (17 relevant queries) | 0.8377 | **0.2072** | +| Mean Precision@5 | 0.3765 | **0.0588** | +| Exact-SKU top-1 | 2/3 | **3/3** | +| Returned constrained results (3 price queries) | 15 | **0** | + +The lexical branch returned no matches for 14 of 18 tuning queries, including all three price-constrained +requests. Zero filter violations across zero constrained results is **not** evidence of useful constrained +retrieval; its violation rate is undefined. The exact-SKU improvement is real, but long natural-language +requests (often containing price words/numbers) are too restrictive under websearch's AND semantics, and +stemming does not supply absent synonyms or paraphrases. Only the three exact-SKU requests and the cross-country +race request returned results. No parsing policy, catalog content, or judgments were adjusted to improve these +scores. This is a useful independent lexical baseline, **not** an improvement to replace semantic defaults. +RRF/fusion, ANN, and reranking are not implemented. + +The ignored report `artifacts/product-search-lexical-tuning.json` contains all 18 per-query results, +matched counts, identities and baseline deltas; test/setup/API diagnostics are also under `artifacts/`. +The semantic baseline file remains unchanged. **No held-out requests or lexical held-out scores were run.** +The separate monthly/manual workflow remains semantic-only and has not been trialed remotely from this branch. Judgments are graded 0 (irrelevant), 1 (marginal), 2 (useful secondary result), or 3 (direct fit). The report uses gain `2^grade - 1` for nDCG@5; Precision@5 counts grade 2 or 3 and always divides by five. Exact-SKU top-1 accuracy is shown only for exact-SKU queries. Queries with no grade-above-zero product are listed separately and excluded from nDCG/precision means rather than counted as successful empty results. Price-filter violations are counted and reported; no similarity/confidence cutoff or quality pass threshold is invented. The checked-in query set is not itself a measured baseline: capture and review a real report before making claims about full-catalog relevance. @@ -713,7 +817,7 @@ Trace embedding time, database time, indexing lag, retries, and failures. Avoid |---|---|---| | 1. Documents and embeddings | See how verified catalog content becomes vectors. | Selected profile, reproducible builder, provider configuration, provenance. | | 2. Exact pgvector retrieval (implemented) | Understand distance ordering with real filters. | Real database tests for ordering, eligibility, profile isolation, paging, freshness and atomic rebuild failure. | -| 3. Full-text and RRF | Compare exact terminology with semantic intent. | Inspectable branch ranks and curated hybrid comparisons. | +| 3. Full-text (implemented); RRF (future) | Compare exact terminology with semantic intent. | Independent lexical matching, exact-SKU priority, model-free lifecycle and measured tuning-only comparison. Hybrid comparisons remain future work. | | 4. Indexing and backfill (implemented; operational recovery remains) | Maintain rebuildable derived state. | Duplicate/out-of-order handling, stale-write protection, updates/logical deletes and explicit readiness. Resumable reconciliation remains future work. | | 5. Curated relevance evaluation (implemented; baseline captured) | Assess real-model rankings against reviewed judgments. | Pinned full-catalog labels, isolated held-out queries, metric definitions and a reproducible live report. | | 6. HNSW benchmark | Measure speed versus recall at meaningful scale. | Exact baseline, larger corpus, real query plans, latency and recall results. | @@ -758,6 +862,9 @@ These files own the implemented behavior and the foundations for later milestone | [Relevance evaluator tests](../../tools/test-product-search-relevance.ps1) | Local metric, dataset validation, drift, backfill, and mocked-HTTP failure tests. | | [Dedicated evaluation seed](../../tools/product-search-relevance.seed.yaml) | Exact reviewed catalog copy loaded using DbEx Execute after isolated reset/reference-data reload. | | [Isolated setup fixture](../tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs) | Explicitly guarded real backfill and readiness verification for `contoso_search_eval` only. | +| [Lexical projection](../src/Contoso.Products.Infrastructure/Repositories/ProductLexicalSearchIndex.cs) | Model-free rebuild/synchronization, separate recipe/readiness, composable matches and weighted embedded SQL. | +| [Lexical API tests](../tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs) | English parsing, exact SKUs, live eligibility/filters, paging, weights/ties, lifecycle, readiness and cancellation. | +| [Lexical subscriber tests](../tests/Contoso.Products.Test.Subscribe/ProductLexicalSearchSubscriberTests.cs) | Real event dispatch and idempotent/deleted state with an unavailable embedding provider. | | [Monthly/manual workflow](../../.github/workflows/product-search-relevance.yml) | Separate scheduled run with isolated services and retained JSON/API-log artifact. | | [Measured baseline report](product-search-relevance-baseline.json) | Captured model/profile/catalog identities, per-query results, and split metrics; no quality threshold. | | [Product query configuration](../src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs) | Server-selected semantic mode and existing filter/order fields. | diff --git a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs index 07f5f370f..a101301e7 100644 --- a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs +++ b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs @@ -3,11 +3,11 @@ namespace Contoso.Mcp.Tools; [McpServerToolType] public static class CommerceTools { - /// Searches products by semantic relevance. - [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products by meaning using semantic search over names, descriptions and product classifications. Returns up to the requested number of product summaries, closest first; results are candidates, not guaranteed matches. Requires a current search index.")] + /// Searches products using the Products server's configured ranked retrieval mode. + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products using the Products server's ranked search mode (semantic by default; optionally English full-text). Returns up to the requested number of summaries, most relevant first. Results are candidates, not guaranteed matches; full-text can return none. Requires a prepared index.")] public static Task SearchProductsAsync( [FromServices] IHttpClientFactory clientFactory, - [Description("Describe the product or intended use to search for. Exact SKU matching is not guaranteed by semantic search.")] string searchTerm, + [Description("Product text, intended use, or a SKU. Semantic mode does not guarantee exact SKU matching; FullText mode prioritizes an exact eligible SKU and otherwise requires lexical matches.")] string searchTerm, [Description("Maximum number of products to return (1-25).")] int take = 10, CancellationToken cancellationToken = default) { diff --git a/samples/src/Contoso.Products.Api/Program.cs b/samples/src/Contoso.Products.Api/Program.cs index 2fbca5674..a1bc0e22d 100644 --- a/samples/src/Contoso.Products.Api/Program.cs +++ b/samples/src/Contoso.Products.Api/Program.cs @@ -71,7 +71,7 @@ private static async Task Main(string[] args) o.EnableIntrospection = true; // Defaults to false (secure-by-default); enabled here so client tooling (Postman, GraphiQL, etc.) can introspect the schema. o.AddReferenceDataQueries(sp, ReferenceDataQueryArgsConfig.Default) - .AddQuery("products", ProductQueryArgsConfig.Default, async (qa, pa, ct) => await CoreEx.ExecutionContext.GetRequiredService().QueryAsync(qa, pa, ct).ConfigureAwait(false)) + .AddQuery("products", ProductQueryArgsConfig.FromConfiguration(builder.Configuration), async (qa, pa, ct) => await CoreEx.ExecutionContext.GetRequiredService().QueryAsync(qa, pa, ct).ConfigureAwait(false)) .AddGet("product", (args, ct) => CoreEx.ExecutionContext.GetRequiredService().GetAsync(args.GetIdentifier(), ct)); }); diff --git a/samples/src/Contoso.Products.Application/Interfaces/IProductReadService.cs b/samples/src/Contoso.Products.Application/Interfaces/IProductReadService.cs index e5f4c1058..17dafe453 100644 --- a/samples/src/Contoso.Products.Application/Interfaces/IProductReadService.cs +++ b/samples/src/Contoso.Products.Application/Interfaces/IProductReadService.cs @@ -2,9 +2,9 @@ namespace Contoso.Products.Application.Interfaces; public interface IProductReadService { - Task GetAsync(string id, CancellationToken ct = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); - Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken ct = default); + Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken cancellationToken = default); - Task QuerySchemaAsync(CancellationToken ct = default); -} \ No newline at end of file + Task QuerySchemaAsync(CancellationToken cancellationToken = default); +} diff --git a/samples/src/Contoso.Products.Application/ProductReadService.cs b/samples/src/Contoso.Products.Application/ProductReadService.cs index 418dc05ff..7adb42861 100644 --- a/samples/src/Contoso.Products.Application/ProductReadService.cs +++ b/samples/src/Contoso.Products.Application/ProductReadService.cs @@ -1,13 +1,17 @@ namespace Contoso.Products.Application; [ScopedService] -public class ProductReadService(IProductRepository repository) : IProductReadService +public class ProductReadService(IProductRepository repository, IProductQueryRepository queryRepository) : IProductReadService { private readonly IProductRepository _repository = repository.ThrowIfNull(); + private readonly IProductQueryRepository _queryRepository = queryRepository.ThrowIfNull(); - public Task GetAsync(string id, CancellationToken ct = default) => _repository.GetAsync(id, ct); + /// + public Task GetAsync(string id, CancellationToken cancellationToken = default) => _repository.GetAsync(id, cancellationToken); - public Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken ct = default) => _repository.QueryAsync(query, paging, ct); + /// + public Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken cancellationToken = default) => _queryRepository.QueryAsync(query, paging, cancellationToken); - public Task QuerySchemaAsync(CancellationToken ct = default) => _repository.QuerySchemaAsync(ct); -} \ No newline at end of file + /// + public Task QuerySchemaAsync(CancellationToken cancellationToken = default) => _queryRepository.QuerySchemaAsync(cancellationToken); +} diff --git a/samples/src/Contoso.Products.Application/Repositories/IProductQueryRepository.cs b/samples/src/Contoso.Products.Application/Repositories/IProductQueryRepository.cs new file mode 100644 index 000000000..445978902 --- /dev/null +++ b/samples/src/Contoso.Products.Application/Repositories/IProductQueryRepository.cs @@ -0,0 +1,11 @@ +namespace Contoso.Products.Application.Repositories; + +/// Provides query and schema operations for the Products read model. +public interface IProductQueryRepository +{ + /// Gets the query schema supported by the Products read model. + Task QuerySchemaAsync(CancellationToken cancellationToken = default); + + /// Queries products using the supplied filters, search intent, and paging options. + Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken cancellationToken = default); +} diff --git a/samples/src/Contoso.Products.Application/Repositories/IProductRepository.cs b/samples/src/Contoso.Products.Application/Repositories/IProductRepository.cs index 2a734c388..8978f9735 100644 --- a/samples/src/Contoso.Products.Application/Repositories/IProductRepository.cs +++ b/samples/src/Contoso.Products.Application/Repositories/IProductRepository.cs @@ -2,17 +2,13 @@ namespace Contoso.Products.Application.Repositories; public interface IProductRepository { - Task GetAsync(string id, CancellationToken ct = default); + Task GetAsync(string id, CancellationToken cancellationToken = default); - Task> CreateAsync(Contracts.Product product, CancellationToken ct = default); + Task> CreateAsync(Contracts.Product product, CancellationToken cancellationToken = default); - Task> UpdateAsync(Contracts.Product product, CancellationToken ct = default); + Task> UpdateAsync(Contracts.Product product, CancellationToken cancellationToken = default); - Task DeleteAsync(string id, CancellationToken ct = default); + Task DeleteAsync(string id, CancellationToken cancellationToken = default); - Task QuerySchemaAsync(CancellationToken ct = default); - - Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken ct = default); - - Task> GetForReservationAsync(string[] ids, CancellationToken ct = default); -} \ No newline at end of file + Task> GetForReservationAsync(string[] ids, CancellationToken cancellationToken = default); +} diff --git a/samples/src/Contoso.Products.Database/Migrations/20261015-175620-add-product-lexical-search.pgsql b/samples/src/Contoso.Products.Database/Migrations/20261015-175620-add-product-lexical-search.pgsql new file mode 100644 index 000000000..2a307988d --- /dev/null +++ b/samples/src/Contoso.Products.Database/Migrations/20261015-175620-add-product-lexical-search.pgsql @@ -0,0 +1,15 @@ +-- Rebuildable derived state, deliberately excluded from DbEx CRUD model generation. +CREATE TABLE products.product_lexical_search ( + product_id VARCHAR(50) PRIMARY KEY REFERENCES products.product(product_id) ON DELETE CASCADE, + recipe VARCHAR(50) NOT NULL, + sku VARCHAR(50) NOT NULL, + document TSVECTOR NOT NULL +); + +CREATE INDEX ix_product_lexical_search_document ON products.product_lexical_search USING GIN (document); +CREATE INDEX ix_product_lexical_search_sku ON products.product_lexical_search (sku); + +CREATE TABLE products.product_lexical_search_readiness ( + recipe VARCHAR(50) PRIMARY KEY, + ready_on TIMESTAMPTZ NOT NULL +); diff --git a/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductSearchSyncAdapter.cs b/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductSearchSyncAdapter.cs index 7e9d777aa..4296b03c9 100644 --- a/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductSearchSyncAdapter.cs +++ b/samples/src/Contoso.Products.Infrastructure/Adapters/Embeddings/ProductSearchSyncAdapter.cs @@ -2,10 +2,13 @@ namespace Contoso.Products.Infrastructure.Adapters.Embeddings; /// Exposes product search reconciliation through the application-owned boundary. [ScopedService] -public sealed class ProductSearchSyncAdapter(Repositories.ProductSearchIndex index) : IProductSearchSyncAdapter +public sealed class ProductSearchSyncAdapter(Repositories.ProductSearchIndex index, Repositories.ProductLexicalSearchIndex lexicalIndex, IConfiguration configuration) : IProductSearchSyncAdapter { private readonly Repositories.ProductSearchIndex _index = index.ThrowIfNull(); + private readonly Repositories.ProductLexicalSearchIndex _lexicalIndex = lexicalIndex.ThrowIfNull(); + private readonly bool _isLexical = Repositories.ProductQueryArgsConfig.ResolveMode(configuration) == QuerySearchMode.FullText; /// - public Task SynchronizeAsync(string id, CancellationToken cancellationToken = default) => Result.GoAsync(() => _index.SynchronizeAsync(id, cancellationToken)); + public Task SynchronizeAsync(string id, CancellationToken cancellationToken = default) + => Result.GoAsync(() => _isLexical ? _lexicalIndex.SynchronizeAsync(id, cancellationToken) : _index.SynchronizeAsync(id, cancellationToken)); } diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductLexicalSearchIndex.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductLexicalSearchIndex.cs new file mode 100644 index 000000000..947470d76 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductLexicalSearchIndex.cs @@ -0,0 +1,71 @@ +namespace Contoso.Products.Infrastructure.Repositories; + +/// Maintains the model-independent, rebuildable English full-text projection. +[ScopedService] +public sealed class ProductLexicalSearchIndex(ProductsEfDb ef, PostgresUnitOfWork unitOfWork) +{ + /// The weighted English document and normalization recipe. + public const string Recipe = "product-lexical-english-v1"; + private static readonly SqlStatement _refresh = SqlStatement.FromResource("ProductLexicalSearchRefresh.pgsql"); + private static readonly SqlStatement _lock = SqlStatement.FromResource("ProductLexicalSearchLock.pgsql"); + private static readonly SqlStatement _isReady = SqlStatement.FromResource("ProductLexicalSearchIsReady.pgsql"); + private static readonly SqlStatement _markReady = SqlStatement.FromResource("ProductLexicalSearchMarkReady.pgsql"); + private static readonly SqlStatement _matches = SqlStatement.FromResource("ProductLexicalSearchMatches.pgsql"); + private readonly ProductsEfDb _ef = ef.ThrowIfNull(); + private readonly PostgresUnitOfWork _unitOfWork = unitOfWork.ThrowIfNull(); + + /// Returns one matching row per product, including exact canonical SKU matches independently of tokenization. + public IQueryable Matches(string text) + => _ef.SqlQuery(_matches, text.ThrowIfNullOrEmpty(), text.Trim().ToUpper(), Recipe); + + /// Rejects an unprepared recipe, including a partially synchronized initial index. + public async Task EnsureReadyAsync(CancellationToken cancellationToken = default) + { + if (!await _ef.Database.Statement(_isReady).Param("Recipe", Recipe, dbType: DbType.String).ScalarAsync(cancellationToken).ConfigureAwait(false)) + throw new TransientException("Product full-text search is not initialized for the configured lexical recipe. Complete a lexical backfill before searching."); + } + + /// Commits current text or deletion before inference; never publishes initial readiness. + public Task SynchronizeAsync(string id, CancellationToken cancellationToken = default) + => RefreshAsync(id.ThrowIfNullOrEmpty(), cancellationToken); + + /// Atomically replaces the full lexical projection and publishes its separate readiness marker. + public Task RebuildAsync(CancellationToken cancellationToken = default) => RefreshAsync(null, cancellationToken); + + /// Serializes publication with catalog writes and other lexical publishers without holding locks during inference. + private async Task RefreshAsync(string? id, CancellationToken cancellationToken) + { + if (_unitOfWork.Database.CurrentTransaction is not null) + throw new InvalidOperationException("Lexical search synchronization must run outside a parent transaction so publication commits independently of embedding work."); + + return await _unitOfWork.TransactionAsync(async token => + { + await _ef.Database.Statement(_lock).NonQueryAsync(token).ConfigureAwait(false); + await _ef.Database.Statement(_refresh) + .Param("ProductId", id, dbType: DbType.String) + .Param("Recipe", Recipe, dbType: DbType.String) + .NonQueryAsync(token).ConfigureAwait(false); + + if (id is not null) + return 0; + + await _ef.Database.Statement(_markReady) + .Param("Recipe", Recipe, dbType: DbType.String) + .Param("ReadyOn", Runtime.UtcNow, dbType: DbType.DateTimeOffset) + .NonQueryAsync(token).ConfigureAwait(false); + + return await _ef.Products.Model.Query().CountAsync(token).ConfigureAwait(false); + }, cancellationToken).ConfigureAwait(false); + } + + /// Represents a unique lexical candidate and its deterministic ranking inputs. + public sealed class SearchMatch + { + /// Gets or sets the product identifier. + public string Id { get; set; } = string.Empty; + /// Gets or sets whether the entire input matches the canonical SKU. + public bool ExactSku { get; set; } + /// Gets or sets the PostgreSQL cover-density score. + public double Rank { get; set; } + } +} diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs index 9eea70424..0ff6d2ac6 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs @@ -6,7 +6,25 @@ namespace Contoso.Products.Infrastructure.Repositories; /// Contoso.Products.Api's Program.cs passes directly into CoreEx.Data.GraphQL's AddQuery to drive GraphQL where/orderBy argument mapping. public class ProductQueryArgsConfig : QueryArgsConfig { - public ProductQueryArgsConfig() + /// Creates the default semantic query configuration. + public ProductQueryArgsConfig() : this(QuerySearchMode.Semantic) { } + + /// The server-selected full-text configuration; filter fields remain identical. + public static ProductQueryArgsConfig FullText { get; } = new(QuerySearchMode.FullText); + + /// Resolves the server-selected search mode without constructing a query configuration. + public static QuerySearchMode ResolveMode(IConfiguration configuration) + => configuration.ThrowIfNull()["Products:Search:Mode"] switch + { + null or "" or "Semantic" => QuerySearchMode.Semantic, + "FullText" => QuerySearchMode.FullText, + var mode => throw new InvalidOperationException($"Unsupported Products:Search:Mode '{mode}'. Expected Semantic or FullText.") + }; + + /// Resolves the explicit server setting without changing the public query arguments. + public static ProductQueryArgsConfig FromConfiguration(IConfiguration configuration) => ResolveMode(configuration) == QuerySearchMode.FullText ? FullText : Default; + + private ProductQueryArgsConfig(QuerySearchMode mode) { // Configure the query arguments for filtering products. WithFilter(filter => filter @@ -25,8 +43,11 @@ public ProductQueryArgsConfig() .AddField(nameof(Contracts.ProductBase.Text)) .AddField(nameof(Contracts.ProductBase.Brand))); - // Semantic search is executed by the repository; ordinary queries retain the configured SKU ordering. + // Ranked search is executed by the repository; ordinary queries retain the configured SKU ordering. WithSearch(search => search - .WithMode(QuerySearchMode.Semantic)); + .WithMode(mode) + .WithHelpText(mode == QuerySearchMode.FullText + ? "English full-text matching with websearch syntax, weighted SKU/name, category/brand, tags and description; exact canonical SKU first, then cover-density rank. No semantic expansion." + : "Semantic similarity using exact cosine distance over the configured product embedding profile.")); } } diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs new file mode 100644 index 000000000..5da8f76b8 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs @@ -0,0 +1,110 @@ +namespace Contoso.Products.Infrastructure.Repositories; + +/// Executes product queries, including semantic and lexical retrieval. +/// The Products database accessor. +/// The semantic search projection. +/// The PostgreSQL transaction coordinator. +/// The lexical search projection. +/// The server-side query and search configuration. +[ScopedService] +public class ProductQueryRepository(ProductsEfDb ef, ProductSearchIndex searchIndex, PostgresUnitOfWork unitOfWork, ProductLexicalSearchIndex lexicalIndex, IConfiguration configuration) : IProductQueryRepository +{ + private readonly ProductsEfDb _ef = ef.ThrowIfNull(); + private readonly ProductSearchIndex _searchIndex = searchIndex.ThrowIfNull(); + private readonly PostgresUnitOfWork _unitOfWork = unitOfWork.ThrowIfNull(); + private readonly ProductLexicalSearchIndex _lexicalIndex = lexicalIndex.ThrowIfNull(); + private readonly ProductQueryArgsConfig _queryConfig = ProductQueryArgsConfig.FromConfiguration(configuration.ThrowIfNull()); + + /// + public Task QuerySchemaAsync(CancellationToken cancellationToken = default) => Task.FromResult(_queryConfig.ToJsonSchema()); + + /// + public async Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken cancellationToken = default) + { + var parsed = _queryConfig.Parse(query).ThrowOnError(); + var isLexical = _queryConfig.SearchParser.Mode == QuerySearchMode.FullText; + var products = _ef.Products.Model.Query(); + if (query?.IsIncludeInactive is false) + products = products.Where(p => !p.IsInactive); + + var subcats = _ef.SubCategories.Query(); + var inventory = _ef.Inventory.Query(); + + var q = + from p in products + join sc in subcats on p.SubCategoryCode equals sc.Code into scg + from sc in scg.DefaultIfEmpty() + join i in inventory on p.Id equals i.Id into ig + from i in ig.DefaultIfEmpty() + select new + { + Product = p, + SubCategoryCode = sc.Code, + sc.CategoryCode, + QtyOnHand = i == null ? 0 : i.QtyOnHand + }; + + q = q.Where(parsed); + float[]? vector = null; + if (parsed.IsRankedSearch) + { + EnsureSnapshotIsolation(); + if (!isLexical) + { + await _searchIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + vector = await _searchIndex.EmbedQueryAsync((parsed.SearchResult?.Text).ThrowIfNull(), cancellationToken).ConfigureAwait(false); + } + } + + return !parsed.IsRankedSearch + ? await ReadAsync(cancellationToken).ConfigureAwait(false) + : await _unitOfWork.TransactionAsync(_unitOfWork.Database.DbArgs with { IsolationLevel = IsolationLevel.RepeatableRead }, ReadAsync, cancellationToken).ConfigureAwait(false); + + async Task> ReadAsync(CancellationToken cancellationToken) + { + if (parsed.IsRankedSearch && isLexical) + { + EnsureSnapshotIsolation(); + await _lexicalIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + q = from candidate in q + join match in _lexicalIndex.Matches((parsed.SearchResult?.Text).ThrowIfNull()) on candidate.Product.Id equals match.Id + orderby match.ExactSku descending, match.Rank descending, candidate.Product.Id + select candidate; + } + else if (vector is not null) + { + EnsureSnapshotIsolation(); + await _searchIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + q = (from candidate in q + join match in _searchIndex.Matches(vector) on candidate.Product.Id equals match.Id + orderby match.Distance, candidate.Product.Id + select candidate); + } + else + q = q.OrderBy(parsed); + + return await q.ToMappedItemsResultAsync(x => new Contracts.ProductLite + { + Id = x.Product.Id, + Sku = x.Product.Sku, + Text = x.Product.Text, + Description = x.Product.Description, + CategoryCode = x.CategoryCode, + SubCategoryCode = x.SubCategoryCode, + BrandCode = x.Product.BrandCode, + UnitOfMeasureCode = x.Product.UnitOfMeasureCode, + Price = x.Product.Price, + IsInactive = x.Product.IsInactive, + IsNonStocked = x.Product.IsNonStocked, + QtyOnHand = x.QtyOnHand + }, paging, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + /// Rejects a parent transaction that cannot provide a consistent PostgreSQL read snapshot. + private void EnsureSnapshotIsolation() + { + if (_unitOfWork.Database.CurrentTransaction is { } transaction && transaction.IsolationLevel is not (IsolationLevel.RepeatableRead or IsolationLevel.Serializable)) + throw new InvalidOperationException($"{(_queryConfig.SearchParser.Mode == QuerySearchMode.FullText ? "Full-text" : "Semantic")} product search requires a RepeatableRead or Serializable parent transaction."); + } +} diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs index 0d3ffc119..5115e190e 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs @@ -1,11 +1,9 @@ namespace Contoso.Products.Infrastructure.Repositories; [ScopedService] -public class ProductRepository(ProductsEfDb ef, ProductSearchIndex searchIndex, PostgresUnitOfWork unitOfWork) : IProductRepository +public class ProductRepository(ProductsEfDb ef) : IProductRepository { private readonly ProductsEfDb _ef = ef.ThrowIfNull(); - private readonly ProductSearchIndex _searchIndex = searchIndex.ThrowIfNull(); - private readonly PostgresUnitOfWork _unitOfWork = unitOfWork.ThrowIfNull(); public Task GetAsync(string id, CancellationToken cancellationToken = default) => _ef.Products.GetAsync(id, cancellationToken); @@ -15,84 +13,6 @@ public class ProductRepository(ProductsEfDb ef, ProductSearchIndex searchIndex, public Task DeleteAsync(string id, CancellationToken cancellationToken = default) => _ef.Products.DeleteAsync(id, cancellationToken); - public Task QuerySchemaAsync(CancellationToken cancellationToken = default) => Task.FromResult(ProductQueryArgsConfig.Default.ToJsonSchema()); - - public async Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken cancellationToken = default) - { - var parsed = ProductQueryArgsConfig.Default.Parse(query).ThrowOnError(); - var products = _ef.Products.Model.Query(); - if (query?.IsIncludeInactive is false) - products = products.Where(p => !p.IsInactive); - - var subcats = _ef.SubCategories.Query(); - var inventory = _ef.Inventory.Query(); - - var q = - from p in products - join sc in subcats on p.SubCategoryCode equals sc.Code into scg - from sc in scg.DefaultIfEmpty() - join i in inventory on p.Id equals i.Id into ig - from i in ig.DefaultIfEmpty() - select new - { - Product = p, - SubCategoryCode = sc.Code, - sc.CategoryCode, - QtyOnHand = i == null ? 0 : i.QtyOnHand - }; - - q = q.Where(parsed); - float[]? vector = null; - if (parsed.IsRankedSearch) - { - EnsureSnapshotIsolation(); - await _searchIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); - vector = await _searchIndex.EmbedQueryAsync((parsed.SearchResult?.Text).ThrowIfNull(), cancellationToken).ConfigureAwait(false); - } - - return vector is null - ? await ReadAsync(cancellationToken).ConfigureAwait(false) - : await _unitOfWork.TransactionAsync(_unitOfWork.Database.DbArgs with { IsolationLevel = IsolationLevel.RepeatableRead }, ReadAsync, cancellationToken).ConfigureAwait(false); - - async Task> ReadAsync(CancellationToken cancellationToken) - { - if (vector is not null) - { - EnsureSnapshotIsolation(); - await _searchIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); - q = (from candidate in q - join match in _searchIndex.Matches(vector) on candidate.Product.Id equals match.Id - orderby match.Distance, candidate.Product.Id - select candidate); - } - else - q = q.OrderBy(parsed); - - return await q.ToMappedItemsResultAsync(x => new Contracts.ProductLite - { - Id = x.Product.Id, - Sku = x.Product.Sku, - Text = x.Product.Text, - Description = x.Product.Description, - CategoryCode = x.CategoryCode, - SubCategoryCode = x.SubCategoryCode, - BrandCode = x.Product.BrandCode, - UnitOfMeasureCode = x.Product.UnitOfMeasureCode, - Price = x.Product.Price, - IsInactive = x.Product.IsInactive, - IsNonStocked = x.Product.IsNonStocked, - QtyOnHand = x.QtyOnHand - }, paging, cancellationToken: cancellationToken).ConfigureAwait(false); - } - } - - /// Rejects a parent transaction that cannot provide a consistent PostgreSQL read snapshot. - private void EnsureSnapshotIsolation() - { - if (_unitOfWork.Database.CurrentTransaction is { } transaction && transaction.IsolationLevel is not (IsolationLevel.RepeatableRead or IsolationLevel.Serializable)) - throw new InvalidOperationException("Semantic product search requires a RepeatableRead or Serializable parent transaction."); - } - public Task> GetForReservationAsync(string[] ids, CancellationToken cancellationToken = default) { var products = _ef.Products.Model.Query(); diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs index bf3d3d4d0..39da2d2d9 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs @@ -6,8 +6,9 @@ namespace Contoso.Products.Infrastructure.Repositories; /// The reference-data labels used by the canonical document. /// The shared PostgreSQL transaction coordinator. /// The indexing diagnostics logger. +/// The independently committed, model-free text projection. [ScopedService] -public sealed class ProductSearchIndex(ProductsEfDb ef, IProductEmbeddingAdapter embeddings, ReferenceDataOrchestrator referenceData, PostgresUnitOfWork unitOfWork, ILogger logger) +public sealed class ProductSearchIndex(ProductsEfDb ef, IProductEmbeddingAdapter embeddings, ReferenceDataOrchestrator referenceData, PostgresUnitOfWork unitOfWork, ILogger logger, ProductLexicalSearchIndex lexicalIndex) { private static readonly SqlStatement _isReady = SqlStatement.FromResource("ProductSearchIsReady.pgsql"); private static readonly SqlStatement _sourceHash = SqlStatement.FromResource("ProductSearchSourceHash.pgsql"); @@ -23,6 +24,7 @@ public sealed class ProductSearchIndex(ProductsEfDb ef, IProductEmbeddingAdapter private readonly ReferenceDataOrchestrator _referenceData = referenceData.ThrowIfNull(); private readonly PostgresUnitOfWork _unitOfWork = unitOfWork.ThrowIfNull(); private readonly ILogger _logger = logger.ThrowIfNull(); + private readonly ProductLexicalSearchIndex _lexicalIndex = lexicalIndex.ThrowIfNull(); /// Embeds search text before opening a database snapshot. public async Task EmbedQueryAsync(string text, CancellationToken cancellationToken = default) @@ -66,6 +68,7 @@ public async Task SynchronizeAsync(string id, CancellationToken cancellationToke if (_unitOfWork.Database.CurrentTransaction is not null) throw new InvalidOperationException("Search synchronization must run outside a parent transaction so inference does not hold database locks."); + await _lexicalIndex.SynchronizeAsync(id, cancellationToken).ConfigureAwait(false); await LoadLabelsAsync(cancellationToken).ConfigureAwait(false); var product = await _ef.Products.Model.Query().AsNoTracking().SingleOrDefaultAsync(p => p.Id == id, cancellationToken).ConfigureAwait(false); var document = product is null ? null : Document(product); @@ -119,6 +122,7 @@ await _unitOfWork.TransactionAsync(async token => /// A short publish-time lock rejects concurrent catalog edits instead of installing stale documents. public async Task RebuildAsync(CancellationToken cancellationToken = default) { + await _lexicalIndex.RebuildAsync(cancellationToken).ConfigureAwait(false); await LoadLabelsAsync(cancellationToken).ConfigureAwait(false); var products = await _ef.Products.Model.Query().AsNoTracking().OrderBy(p => p.Id).ToListAsync(cancellationToken).ConfigureAwait(false); var documents = products.Select(product => (Product: product, Document: Document(product))).ToArray(); @@ -149,6 +153,7 @@ await _ef.Database.Statement(_markReady) .Param("Recipe", ProductSearchDocument.TemplateVersion, dbType: DbType.String) .Param("ReadyOn", Runtime.UtcNow, dbType: DbType.DateTimeOffset) .NonQueryAsync(token).ConfigureAwait(false); + return documents.Length; }, cancellationToken).ConfigureAwait(false); } diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchIsReady.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchIsReady.pgsql new file mode 100644 index 000000000..9c418a249 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchIsReady.pgsql @@ -0,0 +1 @@ +SELECT EXISTS (SELECT 1 FROM products.product_lexical_search_readiness WHERE recipe = @Recipe); diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchLock.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchLock.pgsql new file mode 100644 index 000000000..bf35f4da2 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchLock.pgsql @@ -0,0 +1,2 @@ +LOCK TABLE products.product, products.brand, products.sub_category, products.category IN SHARE MODE; +LOCK TABLE products.product_lexical_search IN SHARE ROW EXCLUSIVE MODE; diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchMarkReady.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchMarkReady.pgsql new file mode 100644 index 000000000..14014c7ed --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchMarkReady.pgsql @@ -0,0 +1,3 @@ +INSERT INTO products.product_lexical_search_readiness (recipe, ready_on) +VALUES (@Recipe, @ReadyOn) +ON CONFLICT (recipe) DO UPDATE SET ready_on = EXCLUDED.ready_on; diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchMatches.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchMatches.pgsql new file mode 100644 index 000000000..0fec4e5b3 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchMatches.pgsql @@ -0,0 +1,6 @@ +SELECT s.product_id AS "Id", + s.sku = {1} AS "ExactSku", + ts_rank_cd(s.document, websearch_to_tsquery('pg_catalog.english', {0}))::double precision AS "Rank" +FROM products.product_lexical_search s +WHERE s.recipe = {2} + AND (s.document @@ websearch_to_tsquery('pg_catalog.english', {0}) OR s.sku = {1}) diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchRefresh.pgsql b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchRefresh.pgsql new file mode 100644 index 000000000..357f6a181 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/Sql/ProductLexicalSearchRefresh.pgsql @@ -0,0 +1,14 @@ +DELETE FROM products.product_lexical_search +WHERE CAST(@ProductId AS VARCHAR) IS NULL OR product_id = @ProductId; + +INSERT INTO products.product_lexical_search (product_id, recipe, sku, document) +SELECT p.product_id, @Recipe, p.sku, + setweight(to_tsvector('pg_catalog.english', coalesce(p.sku, '') || ' ' || coalesce(p.text, '')), 'A') || + setweight(to_tsvector('pg_catalog.english', coalesce(b.text, p.brand_code, '') || ' ' || coalesce(sc.text, p.sub_category_code, '') || ' ' || coalesce(c.text, sc.category_code, '')), 'B') || + setweight(to_tsvector('pg_catalog.english', coalesce(p.tags_json, '[]'::jsonb)), 'B') || + setweight(to_tsvector('pg_catalog.english', coalesce(p.description, '')), 'C') +FROM products.product p +LEFT JOIN products.brand b ON b.code = p.brand_code +LEFT JOIN products.sub_category sc ON sc.code = p.sub_category_code +LEFT JOIN products.category c ON c.code = sc.category_code +WHERE NOT p.is_deleted AND (CAST(@ProductId AS VARCHAR) IS NULL OR p.product_id = @ProductId); diff --git a/samples/tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs b/samples/tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs new file mode 100644 index 000000000..6b319e308 --- /dev/null +++ b/samples/tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs @@ -0,0 +1,269 @@ +namespace Contoso.Products.Test.Api; + +[NonParallelizable] +public class ProductLexicalSearchTests : WithApiTester +{ + private readonly TestEmbeddingAdapter _embeddings = new() { FailQuery = true, FailDocumentOnCall = 1 }; + + [OneTimeSetUp] + public async Task OneTimeSetUpAsync() + { + Test.UseAdditionalConfiguration("Products:Search:Mode", "FullText"); + Test.ReplaceScoped(_ => _embeddings); + await Test.MigratePostgresDataAsync(["read-data.seed.yaml"], DbMigration.ConfigureMigrationArgs).ConfigureAwait(false); + await Test.ClearFusionCacheAsync().ConfigureAwait(false); + Rebuild(); + } + + [TestCase(" shimano-xt-m8100-rd ", "SHIMANO-XT-M8100-RD")] + [TestCase("MUCOFF-NANO-CLEANER", "MUCOFF-NANO-CLEANER")] + public void ExactSku_IsFirstAndUnique(string query, string sku) + { + var result = Search(query); + result[0].Sku.Should().Be(sku); + result.Select(p => p.Id).Should().OnlyHaveUniqueItems(); + _embeddings.QueryCallCount.Should().Be(0); + _embeddings.DocumentCallCount.Should().Be(0); + } + + [TestCase("derailleurs", "SHIMANO-XT-M8100-RD")] + [TestCase("derailleur!!!", "SHIMANO-XT-M8100-RD")] + [TestCase("\"rear derailleur\"", "SHIMANO-XT-M8100-RD")] + [TestCase("derailleur OR cleaner", "MUCOFF-NANO-CLEANER")] + public void EnglishNormalizationAndWebsearch_Match(string query, string sku) + => Search(query).Should().Contain(p => p.Sku == sku); + + [TestCase("the and a")] + [TestCase("!!!")] + [TestCase("zzzxnomatch")] + [TestCase("'; DROP TABLE products.product; --")] + public void EmptyLexemesAndNoMatch_AreEmptyNotFallback(string query) => Search(query).Should().BeEmpty(); + + [Test] + public void FiltersAndMatchingSetPaging_AreConsistent() + { + var all = Search("derailleur"); + all.Should().HaveCount(4); + var page = Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=derailleur&$skip=1&$take=2&$count=true").AssertOK(); + page.Value!.Select(p => p.Id).Should().Equal(all.Skip(1).Take(2).Select(p => p.Id)); + page.Response.Headers.Should().ContainKey("X-Paging-Total-Count").WhoseValue.Should().ContainSingle().Which.Should().Be("4"); + Search("derailleur", "&$filter=brand eq 'SRAM' and price lt 160").Should().ContainSingle().Which.Sku.Should().Be("SRAM-GX-EAGLE-RD"); + Search("SHIMANO-XT-M8100-RD", "&$filter=price lt 0").Should().BeEmpty(); + Search("derailleur", "&$filter=category eq 'B'").Should().BeEmpty(); + Search("derailleur", "&$filter=subcategory eq 'DR'").Should().HaveCount(3); + Search("derailleur", "&$filter=startswith(sku,'SHIMANO')").Should().ContainSingle(); + } + + [Test] + public void Eligibility_IsLiveEvenWithRetainedProjection() + { + Search("SPEC-EPIC-8-PRO").Should().BeEmpty(); + Search("SPEC-EPIC-8-PRO", "&$inactive").First().Sku.Should().Be("SPEC-EPIC-8-PRO"); + Search("GIANT-TALON-2").Should().BeEmpty(); + try + { + Execute("UPDATE products.product SET is_deleted = true WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Search("MUCOFF-NANO-CLEANER").Should().BeEmpty(); + } + finally + { + Execute("UPDATE products.product SET is_deleted = false WHERE sku = 'MUCOFF-NANO-CLEANER'"); + } + } + + [Test] + public void Ties_UseStableProductIdentifier() + { + try + { + Execute("UPDATE products.product SET text = 'tieprobe', description = 'tieprobe' WHERE sku IN ('YETI-ASR-C2-2025', 'YETI-SB120-C2')"); + Rebuild(); + var results = Search("tieprobe"); + results.Select(p => p.Id).Should().Equal(1.ToGuid().ToString(), 2.ToGuid().ToString()); + Search("tieprobe").Select(p => p.Id).Should().Equal(results.Select(p => p.Id)); + } + finally + { + // Restore the authoritative seed rather than preserving altered descriptions. + Test.MigratePostgresDataAsync(["read-data.seed.yaml"], DbMigration.ConfigureMigrationArgs).GetAwaiter().GetResult(); + Rebuild(); + } + } + + [Test] + public void SchemaAndGraphQL_AdvertiseAndExecuteSelectedMode() + { + var schema = Test.Http().Run(HttpMethod.Get, "/api/products/$query").AssertOK().Value; + schema.GetProperty("search").GetProperty("mode").GetString().Should().Be("fulltext"); + schema.GetProperty("search").GetProperty("description").GetString().Should().Contain("English"); + Test.Http().Run(HttpMethod.Get, "/api/products?$search=derailleur&$orderby=sku").AssertBadRequest(); + var graphql = Test.Http().Run(HttpMethod.Post, "/api/query", + new { query = "{ products(search: \"MUCOFF-NANO-CLEANER\", first: 1) { edges { node { sku } } } }" }).AssertOK().Value; + graphql.TryGetProperty("errors", out _).Should().BeFalse(); + graphql.GetProperty("data").GetProperty("products").GetProperty("edges")[0].GetProperty("node").GetProperty("sku").GetString().Should().Be("MUCOFF-NANO-CLEANER"); + } + + [Test] + public void InvalidConfiguredMode_HasNoSilentFallback() + { + var configuration = Test.Configuration; + try + { + configuration["Products:Search:Mode"] = "Hybrid"; + Action resolve = () => ProductQueryArgsConfig.FromConfiguration(configuration); + resolve.Should().Throw().WithMessage("Unsupported Products:Search:Mode 'Hybrid'. Expected Semantic or FullText."); + } + finally + { + configuration["Products:Search:Mode"] = "FullText"; + } + } + + [TestCase("")] + [TestCase(" ")] + public void EmptySearch_PreservesOrdinaryQueryBehavior(string text) + => Test.Http().Run(HttpMethod.Get, $"/api/products?$search={Uri.EscapeDataString(text)}&$take=50").AssertOK().Value.Should().HaveCount(29); + + [Test] + public void SynchronizationAndRebuild_AreIndependentOfModelAndReadinessIsExplicit() + { + try + { + Execute("DELETE FROM products.product_lexical_search_readiness"); + Execute("UPDATE products.product SET description = 'lexicaloutageprobe' WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Synchronize(); + SearchUnavailable(); + Rebuild(); + Search("lexicaloutageprobe").Should().ContainSingle().Which.Sku.Should().Be("MUCOFF-NANO-CLEANER"); + Execute("UPDATE products.product SET description = 'replacementprobe' WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Synchronize(); + Search("lexicaloutageprobe").Should().BeEmpty(); + Search("replacementprobe").Should().ContainSingle(); + Execute("UPDATE products.product SET is_deleted = true WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Synchronize(); + Test.Scoped(test => test.Run(async _ => + { + var matches = test.Services.GetRequiredService().Matches("replacementprobe"); + (await matches.CountAsync().ConfigureAwait(false)).Should().Be(0); + }).AssertSuccess()); + _embeddings.DocumentCallCount.Should().Be(0); + _embeddings.QueryCallCount.Should().Be(0); + } + finally + { + Test.MigratePostgresDataAsync(["read-data.seed.yaml"], DbMigration.ConfigureMigrationArgs).GetAwaiter().GetResult(); + Rebuild(); + } + } + + [Test] + public void StopwordSku_StillMatchesExactly() + { + try + { + Execute("UPDATE products.product SET sku = 'THE' WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Synchronize(); + Search("the").Should().ContainSingle().Which.Sku.Should().Be("THE"); + } + finally + { + Execute("UPDATE products.product SET sku = 'MUCOFF-NANO-CLEANER' WHERE sku = 'THE'"); + Synchronize(); + } + } + + [Test] + public void TagsAndFieldWeights_AreSearchable() + { + try + { + Execute("UPDATE products.product SET text = 'weightprobe', description = NULL, tags_json = '[\"tagprobe\"]'::jsonb WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Execute("UPDATE products.product SET description = 'weightprobe' WHERE sku = 'YETI-ASR-C2-2025'"); + Rebuild(); + Search("weightprobe").First().Sku.Should().Be("MUCOFF-NANO-CLEANER"); + Search("tagprobe").Should().ContainSingle().Which.Sku.Should().Be("MUCOFF-NANO-CLEANER"); + Execute("UPDATE products.product SET sku = 'WEIGHTPROBE', text = 'opaque', description = NULL, tags_json = NULL WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Execute("UPDATE products.product SET text = 'weightprobe weightprobe weightprobe weightprobe', description = 'weightprobe' WHERE sku = 'YETI-ASR-C2-2025'"); + Rebuild(); + Test.Scoped(test => test.Run(async _ => + { + var matches = await test.Services.GetRequiredService().Matches("weightprobe").ToListAsync().ConfigureAwait(false); + matches.Single(m => m.ExactSku).Rank.Should().BeLessThan(matches.Single(m => !m.ExactSku).Rank); + }).AssertSuccess()); + Search("weightprobe").First().Sku.Should().Be("WEIGHTPROBE"); + } + finally + { + Test.MigratePostgresDataAsync(["read-data.seed.yaml"], DbMigration.ConfigureMigrationArgs).GetAwaiter().GetResult(); + Rebuild(); + } + } + + [Test] + public void EmptyBackfill_IsReadyAndCountIsZero() + { + try + { + Execute("UPDATE products.product SET is_deleted = true"); + Rebuild(); + var result = Test.Http().Run(HttpMethod.Get, "/api/products?$search=bike&$count=true").AssertOK(); + result.Value.Should().BeEmpty(); + result.Response.Headers.Should().ContainKey("X-Paging-Total-Count").WhoseValue.Should().ContainSingle().Which.Should().Be("0"); + } + finally + { + Test.MigratePostgresDataAsync(["read-data.seed.yaml"], DbMigration.ConfigureMigrationArgs).GetAwaiter().GetResult(); + Rebuild(); + } + } + + [TestCase(IsolationLevel.RepeatableRead)] + [TestCase(IsolationLevel.Serializable)] + public void CompatibleParent_PreservesSnapshot(IsolationLevel isolation) => Test.Scoped(test => test.Run(async _ => + { + var uow = test.Services.GetRequiredService(); + await uow.TransactionAsync(new PostgresDatabaseArgs { IsolationLevel = isolation }, async token => + { + var parent = uow.Database.CurrentTransaction; + var page = await test.Services.GetRequiredService().QueryAsync( + new QueryArgs { Search = "derailleur" }, PagingArgs.CreateWithCount(take: 1), token).ConfigureAwait(false); + page.Items.Should().ContainSingle(); + page.Paging!.TotalCount.Should().Be(4); + uow.Database.CurrentTransaction.Should().BeSameAs(parent); + }).ConfigureAwait(false); + }).AssertSuccess()); + + [Test] + public void CancellationAndParentTransaction_AreExplicit() => Test.Scoped(test => test.Run(async _ => + { + var index = test.Services.GetRequiredService(); + using var canceled = new CancellationTokenSource(); + canceled.Cancel(); + Func rebuild = () => index.RebuildAsync(canceled.Token); + await rebuild.Should().ThrowAsync().ConfigureAwait(false); + Func query = () => test.Services.GetRequiredService().QueryAsync(new QueryArgs { Search = "cleaner" }, null, canceled.Token); + await query.Should().ThrowAsync().ConfigureAwait(false); + await test.Services.GetRequiredService().TransactionAsync(async token => + { + Func sync = () => index.SynchronizeAsync(29.ToGuid().ToString(), token); + await sync.Should().ThrowAsync().ConfigureAwait(false); + Func search = () => test.Services.GetRequiredService().QueryAsync(new QueryArgs { Search = "cleaner" }, null, token); + await search.Should().ThrowAsync().ConfigureAwait(false); + }).ConfigureAwait(false); + }).AssertSuccess()); + + private ProductLite[] Search(string query, string suffix = "") + => Test.Http().Run(HttpMethod.Get, $"/api/products?$search={Uri.EscapeDataString(query)}&$take=50{suffix}").AssertOK().Value!; + + private void SearchUnavailable() => Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.ServiceUnavailable); + + private void Rebuild() => Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false)).AssertSuccess()); + + private void Synchronize() => Test.Scoped(test => test.Run(async _ => + (await test.Services.GetRequiredService().SynchronizeAsync(29.ToGuid().ToString()).ConfigureAwait(false)).ThrowOnError()).AssertSuccess()); + + private void Execute(string sql) => Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().Database.ExecuteSqlRawAsync(sql).ConfigureAwait(false)).AssertSuccess()); +} diff --git a/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs b/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs index 6f6e35b9d..90b82e5c7 100644 --- a/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs +++ b/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs @@ -1,6 +1,6 @@ namespace Contoso.Products.Test.Api; -[TestFixture, Explicit("Builds and verifies the real Ollama search profile in the isolated contoso_search_eval database.")] +[TestFixture, Explicit("Builds and verifies the selected real search branch in the isolated contoso_search_eval database.")] public class ProductSearchRelevanceSetupTests : WithApiTester { [OneTimeSetUp] @@ -15,7 +15,10 @@ public async Task OneTimeSetUpAsync() Test.Scoped(test => test.Run(async _ => { - var count = await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false); + var lexical = Environment.GetEnvironmentVariable("Products__Search__Mode") == "FullText"; + var count = lexical + ? await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false) + : await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false); count.Should().Be(31); }).AssertSuccess()); } @@ -23,11 +26,17 @@ public async Task OneTimeSetUpAsync() [Test] public void Product_Search_RealModelProfileIsReady() { + var lexical = Environment.GetEnvironmentVariable("Products__Search__Mode") == "FullText"; var response = Test.Http() - .Run(HttpMethod.Get, "/api/products?$search=something to clean mud off my bike&$take=1&$count=true") + .Run(HttpMethod.Get, lexical + ? "/api/products?$search=SHIMANO-XT-M8100-RD&$take=1&$count=true" + : "/api/products?$search=something to clean mud off my bike&$take=1&$count=true") .AssertOK(); - response.Response.Headers.Should().ContainKey("X-Paging-Total-Count") - .WhoseValue.Should().ContainSingle().Which.Should().Be("29"); + if (lexical) + response.Value.Should().ContainSingle().Which.Sku.Should().Be("SHIMANO-XT-M8100-RD"); + else + response.Response.Headers.Should().ContainKey("X-Paging-Total-Count") + .WhoseValue.Should().ContainSingle().Which.Should().Be("29"); } } diff --git a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs index 6be7ba201..6528f3ba2 100644 --- a/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs +++ b/samples/tests/Contoso.Products.Test.Api/ReadTests.ProductSemanticSearch.cs @@ -8,7 +8,7 @@ public void Product_Search_EnlistsInCompatibleParent(IsolationLevel isolation) = { var ef = test.Services.GetRequiredService(); var uow = test.Services.GetRequiredService(); - var repository = test.Services.GetRequiredService(); + var repository = test.Services.GetRequiredService(); await uow.TransactionAsync(new PostgresDatabaseArgs { IsolationLevel = isolation }, async token => { var parent = uow.Database.CurrentTransaction; @@ -28,7 +28,7 @@ public void Product_Search_RejectsWeakParentWithoutInference() => Test.Scoped(te { var ef = test.Services.GetRequiredService(); var uow = test.Services.GetRequiredService(); - var repository = test.Services.GetRequiredService(); + var repository = test.Services.GetRequiredService(); await uow.TransactionAsync(async token => { var parent = uow.Database.CurrentTransaction; @@ -215,6 +215,7 @@ public void Product_Search_FailedInitialBackfillDoesNotPublishReadiness() var index = test.Services.GetRequiredService(); Func rebuild = () => index.RebuildAsync(); await rebuild.Should().ThrowAsync().ConfigureAwait(false); + await test.Services.GetRequiredService().EnsureReadyAsync().ConfigureAwait(false); Func ready = () => index.EnsureReadyAsync(); await ready.Should().ThrowAsync().ConfigureAwait(false); }).AssertSuccess()); diff --git a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json index 2de327ac6..cf37236ec 100644 --- a/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json +++ b/samples/tests/Contoso.Products.Test.Api/Resources/ReadTests/Product_Query_Schema.res.json @@ -82,6 +82,7 @@ }, "search": { "mode": "semantic", + "description": "Semantic similarity using exact cosine distance over the configured product embedding profile.", "ranked": true, "maxLength": 200 } diff --git a/samples/tests/Contoso.Products.Test.Subscribe/ProductLexicalSearchSubscriberTests.cs b/samples/tests/Contoso.Products.Test.Subscribe/ProductLexicalSearchSubscriberTests.cs new file mode 100644 index 000000000..22aa95070 --- /dev/null +++ b/samples/tests/Contoso.Products.Test.Subscribe/ProductLexicalSearchSubscriberTests.cs @@ -0,0 +1,61 @@ +namespace Contoso.Products.Test.Subscribe; + +[NonParallelizable] +public class ProductLexicalSearchSubscriberTests : WithApiTester +{ + [OneTimeSetUp] + public void OneTimeSetUp() + { + Test.UseAdditionalConfiguration("Products:Search:Mode", "FullText"); + Test.ReplaceScoped(_ => new UnavailableEmbeddings()); + } + + [SetUp] + public async Task SetUpAsync() + { + await Test.MigratePostgresDataAsync(["mutate-data.seed.yaml"], DbMigration.ConfigureMigrationArgs).ConfigureAwait(false); + await Test.ClearFusionCacheAsync().ConfigureAwait(false); + } + + [TestCase("created")] + [TestCase("updated")] + [TestCase("activated")] + [TestCase("deactivated")] + public void Event_ReconcilesWithoutModelAndIsIdempotent(string action) + { + Receive(action); + Receive(action); + AssertMatches(1); + } + + [Test] + public void DeleteAndLateUpdate_CannotResurrect() + { + Receive("created"); + Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().Database.ExecuteSqlRawAsync( + "UPDATE products.product SET is_deleted = true WHERE sku = 'MUCOFF-NANO-CLEANER'").ConfigureAwait(false)).AssertSuccess()); + Receive("deleted"); + Receive("updated"); + AssertMatches(0); + } + + private void Receive(string action) => Test.Scoped(test => test.Run(async _ => + { + var message = Test.CreateCloudEventFrom(EventData.CreateEvent(action).WithKey(29.ToGuid().ToString())).ToServiceBusReceivedMessage(); + (await test.Services.GetRequiredKeyedService("subscriber-events").ReceiveAsync(message).ConfigureAwait(false)).IsSuccess.Should().BeTrue(); + }).AssertSuccess()); + + private void AssertMatches(int count) => Test.Scoped(test => test.Run(async _ => + { + (await test.Services.GetRequiredService().Matches("MUCOFF-NANO-CLEANER").CountAsync().ConfigureAwait(false)).Should().Be(count); + }).AssertSuccess()); + + private sealed class UnavailableEmbeddings : IProductEmbeddingAdapter + { + public string Profile => throw new InvalidOperationException("Full-text must not inspect the model profile."); + public int Dimensions => throw new InvalidOperationException("Full-text must not inspect model dimensions."); + public Task> EmbedDocumentAsync(string text, CancellationToken cancellationToken = default) => throw new HttpRequestException("Model is offline."); + public Task> EmbedQueryAsync(string text, CancellationToken cancellationToken = default) => throw new HttpRequestException("Model is offline."); + } +} diff --git a/samples/tests/Contoso.Products.Test.Subscribe/ProductSearchTests.cs b/samples/tests/Contoso.Products.Test.Subscribe/ProductSearchTests.cs index f64a09250..0c03cdd3d 100644 --- a/samples/tests/Contoso.Products.Test.Subscribe/ProductSearchTests.cs +++ b/samples/tests/Contoso.Products.Test.Subscribe/ProductSearchTests.cs @@ -95,6 +95,10 @@ public void ProviderFailure_RetriesAndPreservesPublishedVector() _embeddings.Fail = true; Receive("updated", retry: true); Count().Should().Be(1); + Test.Scoped(test => test.Run(async _ => + { + (await test.Services.GetRequiredService().Matches("Changed before provider failure").CountAsync().ConfigureAwait(false)).Should().BeGreaterThan(0); + }).AssertSuccess()); _embeddings.Fail = false; Receive("updated"); Count().Should().Be(1); diff --git a/tools/product-search-relevance.ps1 b/tools/product-search-relevance.ps1 index 69b3c97ff..9f9df8c77 100644 --- a/tools/product-search-relevance.ps1 +++ b/tools/product-search-relevance.ps1 @@ -3,7 +3,10 @@ param( [uri] $BaseAddress = 'http://localhost:5207/', [uri] $OllamaAddress = 'http://localhost:11434/', [string] $DatasetPath = "$PSScriptRoot\product-search-relevance.dataset.json", - [string] $OutputPath + [string] $OutputPath, + [ValidateSet('semantic', 'fulltext')][string] $Approach = 'semantic', + [ValidateSet('all', 'tuning', 'held-out')][string] $Split = 'all', + [string] $BaselinePath ) $ErrorActionPreference = 'Stop' @@ -245,8 +248,18 @@ function Invoke-ProductSearchRelevanceEvaluation { [Parameter(Mandatory)][uri] $BaseAddress, [Parameter(Mandatory)][uri] $OllamaAddress, [Parameter(Mandatory)][scriptblock] $Request, - [string] $DatasetSha256 + [string] $DatasetSha256, + [ValidateSet('semantic', 'fulltext')][string] $Approach = 'semantic', + [ValidateSet('all', 'tuning', 'held-out')][string] $Split = 'all', + [string] $BaselinePath ) + if ($Approach -eq 'fulltext' -and $Split -ne 'tuning') { + throw 'This lexical milestone is tuning-only. Pass -Split tuning; held-out queries must not execute during tuning.' + } + if ($BaselinePath -and $Split -ne 'tuning') { throw 'Baseline comparison is tuning-only.' } + $selectedQueries = @($Dataset.queries | Where-Object { $Split -eq 'all' -or $_.split -eq $Split }) + $requestedSplit = $Split + if (!$selectedQueries.Count) { throw "No queries selected for split '$Split'." } $implementationIdentity = Test-ProductSearchImplementationIdentity $Dataset $catalogSignature = Get-ProductSearchCatalogSignature -Products @($Dataset.catalog) $seedPath = Join-Path $PSScriptRoot ("..\{0}" -f ($Dataset.seedFile -replace '/', '\')) @@ -259,13 +272,30 @@ function Invoke-ProductSearchRelevanceEvaluation { $DatasetSha256 = (Get-FileHash -LiteralPath (Join-Path $PSScriptRoot 'product-search-relevance.dataset.json') -Algorithm SHA256).Hash } - $models = (Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($OllamaAddress, 'api/tags'))).Data.models - $model = @($models | Where-Object { $_.name -eq $Dataset.model -and $_.digest -eq $Dataset.modelDigest }) - if ($model.Count -ne 1) { throw "Ollama model drift: expected $($Dataset.model) with digest $($Dataset.modelDigest)." } + if ($Approach -eq 'semantic') { + $models = (Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($OllamaAddress, 'api/tags'))).Data.models + $model = @($models | Where-Object { $_.name -eq $Dataset.model -and $_.digest -eq $Dataset.modelDigest }) + if ($model.Count -ne 1) { throw "Ollama model drift: expected $($Dataset.model) with digest $($Dataset.modelDigest)." } + } + else { + $sqlPath = Join-Path $PSScriptRoot '..\samples\src\Contoso.Products.Infrastructure\Repositories\Sql' + $repositoryPath = Split-Path $sqlPath + $lexicalSource = Get-Content -LiteralPath (Join-Path $repositoryPath 'ProductLexicalSearchIndex.cs') -Raw + if ($lexicalSource -notmatch 'Recipe\s*=\s*"([^"]+)"') { throw 'Cannot identify the lexical recipe.' } + $lexicalRecipe = $Matches[1] + $implementationIdentity | Add-Member -NotePropertyName lexicalImplementationSha256 -NotePropertyValue @( + @('ProductLexicalSearchIndex.cs', 'ProductRepository.cs', 'ProductQueryArgsConfig.cs') | ForEach-Object { + [pscustomobject]@{ file = $_; sha256 = (Get-FileHash -LiteralPath (Join-Path $repositoryPath $_) -Algorithm SHA256).Hash } + }) + $implementationIdentity | Add-Member -NotePropertyName lexicalSqlSha256 -NotePropertyValue @( + Get-ChildItem -LiteralPath $sqlPath -Filter 'ProductLexicalSearch*.pgsql' | Sort-Object Name | ForEach-Object { + [pscustomobject]@{ file = $_.Name; sha256 = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash } + }) + } $schema = (Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($BaseAddress, 'api/products/$query'))).Data - if ($schema.search.mode -ne 'semantic' -or $schema.search.ranked -ne $true) { - throw 'Products API drift: this evaluation requires semantic ranked search.' + if ($schema.search.mode -ne $Approach -or $schema.search.ranked -ne $true) { + throw "Products API drift: this evaluation requires $Approach ranked search." } $catalogResponse = Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($BaseAddress, 'api/products?$take=100&$count=true')) @@ -283,17 +313,19 @@ function Invoke-ProductSearchRelevanceEvaluation { throw "Catalog drift: live Products API catalog signature $liveCatalogSignature does not match reviewed dataset signature $catalogSignature." } - $firstQuery = $Dataset.queries[0] + $firstQuery = $selectedQueries[0] $firstUri = [uri]::new($BaseAddress, "api/products?`$search=$([uri]::EscapeDataString([string]$firstQuery.query))&`$take=5&`$count=true") + $firstFilter = Get-ProductSearchFilter $firstQuery + if ($firstFilter) { $firstUri = [uri]($firstUri.AbsoluteUri + "&`$filter=$([uri]::EscapeDataString($firstFilter))") } $firstResponse = Get-ProductSearchResponse -Request $Request -Uri $firstUri $firstExpectedCount = @((Get-ProductSearchEligibleCatalog -Dataset $Dataset -Query $firstQuery)).Count $firstActualCount = Get-ProductSearchCountHeader $firstResponse.Headers - if ($firstActualCount -ne $firstExpectedCount) { + if ($Approach -eq 'semantic' -and $firstActualCount -ne $firstExpectedCount) { throw "Search backfill is not ready or is incomplete for profile '$($Dataset.profile)' and recipe '$($Dataset.documentRecipe)': expected $firstExpectedCount indexed candidates, got $firstActualCount." } $reports = [Collections.Generic.List[object]]::new() - foreach ($query in $Dataset.queries) { + foreach ($query in $selectedQueries) { if ($query.id -eq $firstQuery.id) { $response = $firstResponse } @@ -306,14 +338,15 @@ function Invoke-ProductSearchRelevanceEvaluation { $candidateCount = Get-ProductSearchCountHeader $response.Headers $expectedCandidateCount = @((Get-ProductSearchEligibleCatalog -Dataset $Dataset -Query $query)).Count - if ($candidateCount -ne $expectedCandidateCount) { - throw "Search backfill is incomplete or eligibility drifted for query '$($query.id)' (profile '$($Dataset.profile)', recipe '$($Dataset.documentRecipe)'): expected $expectedCandidateCount candidates, got $candidateCount." + if (($Approach -eq 'semantic' -and $candidateCount -ne $expectedCandidateCount) -or + $candidateCount -lt 0 -or $candidateCount -gt $expectedCandidateCount) { + throw "Search count drift for $Approach query '$($query.id)': $expectedCandidateCount eligible catalog rows, got $candidateCount candidates. Semantic must match the eligible count; lexical counts must be between zero and that count." } $judgments = @{} foreach ($judgment in $query.judgments) { $judgments[[string]$judgment.sku] = [int]$judgment.grade } $ranked = @($response.Data) - $expectedPageCount = [Math]::Min(5, $expectedCandidateCount) + $expectedPageCount = [Math]::Min(5, $candidateCount) if ($ranked.Count -ne $expectedPageCount) { throw "Incomplete result page for query '$($query.id)': expected $expectedPageCount rows, got $($ranked.Count)." } @@ -349,13 +382,14 @@ function Invoke-ProductSearchRelevanceEvaluation { ndcgAt5 = $metrics.NdcgAt5 precisionAt5 = $metrics.PrecisionAt5 eligibleCandidateCount = $expectedCandidateCount + matchedCandidateCount = $candidateCount constraintViolationCount = $constraintViolations.Count topFive = $topFive }) } $splitReports = @{} - foreach ($split in @('tuning', 'held-out')) { + foreach ($split in @($reports | ForEach-Object split | Select-Object -Unique)) { $splitQueries = @($reports | Where-Object split -EQ $split) $relevantQueries = @($splitQueries | Where-Object { !$_.noRelevant }) $exactQueries = @($splitQueries | Where-Object { $null -ne $_.exactSkuTop1 }) @@ -381,18 +415,41 @@ function Invoke-ProductSearchRelevanceEvaluation { } } + $comparison = $null + if ($BaselinePath) { + $baseline = Get-Content -LiteralPath $BaselinePath -Raw | ConvertFrom-Json + if ($baseline.status -ne 'completed' -or $baseline.datasetSha256 -ne $DatasetSha256 -or + $baseline.profile -ne $Dataset.profile -or $baseline.documentRecipe -ne $Dataset.documentRecipe -or + $baseline.catalogIdentity.liveApiCatalogSha256 -ne $liveCatalogSignature -or + $baseline.catalogIdentity.seedFileSha256 -ne $actualSeedHash) { + throw 'Baseline dataset/catalog identity does not match this evaluation.' + } + $baselineIds = @($baseline.queries | Where-Object split -EQ 'tuning' | ForEach-Object id | Sort-Object) + $actualIds = @($reports | ForEach-Object id | Sort-Object) + if (($baselineIds -join '|') -ne ($actualIds -join '|')) { throw 'Baseline tuning query IDs differ.' } + $comparison = [pscustomobject]@{ + baselineSha256 = (Get-FileHash -LiteralPath $BaselinePath -Algorithm SHA256).Hash + split = 'tuning' + baseline = $baseline.splits.tuning + meanNdcgAt5Delta = $splitReports.tuning.meanNdcgAt5 - $baseline.splits.tuning.meanNdcgAt5 + meanPrecisionAt5Delta = $splitReports.tuning.meanPrecisionAt5 - $baseline.splits.tuning.meanPrecisionAt5 + exactSkuTop1AccuracyDelta = $splitReports.tuning.exactSkuTop1Accuracy - $baseline.splits.tuning.exactSkuTop1Accuracy + } + } return [pscustomobject]@{ - evaluationVersion = 'product-search-relevance-runner-v1' + evaluationVersion = 'product-search-relevance-runner-v2' + approach = $Approach + selectedSplit = $requestedSplit completedAtUtc = [DateTime]::UtcNow.ToString('o') status = 'completed' datasetVersion = [string]$Dataset.datasetVersion - model = [string]$Dataset.model - modelDigest = [string]$Dataset.modelDigest - profile = [string]$Dataset.profile - dimensions = [int]$Dataset.dimensions - documentRecipe = [string]$Dataset.documentRecipe - documentInstruction = [string]$Dataset.documentInstruction - queryInstruction = [string]$Dataset.queryInstruction + model = if ($Approach -eq 'semantic') { [string]$Dataset.model } else { $null } + modelDigest = if ($Approach -eq 'semantic') { [string]$Dataset.modelDigest } else { $null } + profile = if ($Approach -eq 'semantic') { [string]$Dataset.profile } else { 'postgres:english:ts_rank_cd:exact-sku-first' } + dimensions = if ($Approach -eq 'semantic') { [int]$Dataset.dimensions } else { $null } + documentRecipe = if ($Approach -eq 'semantic') { [string]$Dataset.documentRecipe } else { $lexicalRecipe } + documentInstruction = if ($Approach -eq 'semantic') { [string]$Dataset.documentInstruction } else { $null } + queryInstruction = if ($Approach -eq 'semantic') { [string]$Dataset.queryInstruction } else { $null } datasetSha256 = $DatasetSha256 implementationIdentity = $implementationIdentity catalogIdentity = [pscustomobject]@{ @@ -412,6 +469,7 @@ function Invoke-ProductSearchRelevanceEvaluation { constraints = 'Returned price-filtered results are validated against the declared hard price condition; no threshold is imposed on relevance scores.' } splits = $splitReports + tuningComparison = $comparison queries = @($reports) } } @@ -428,12 +486,12 @@ if ($MyInvocation.InvocationName -ne '.') { } $request = { param([uri] $Uri) Invoke-WebRequest -Uri $Uri -TimeoutSec 120 } - $report = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress $BaseAddress -OllamaAddress $OllamaAddress -Request $request -DatasetSha256 $datasetHash + $report = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress $BaseAddress -OllamaAddress $OllamaAddress -Request $request -DatasetSha256 $datasetHash -Approach $Approach -Split $Split -BaselinePath $BaselinePath $json = ConvertTo-Json -InputObject $report -Depth 30 if ($OutputPath) { $outputFullPath = [IO.Path]::GetFullPath($OutputPath) [IO.File]::WriteAllText($outputFullPath, $json + [Environment]::NewLine, [Text.UTF8Encoding]::new($false)) - Write-Host "Wrote real-model product-search report to $outputFullPath" + Write-Host "Wrote $Approach product-search ($Split) report to $outputFullPath" } else { Write-Output $json } } diff --git a/tools/test-product-search-relevance.ps1 b/tools/test-product-search-relevance.ps1 index 793692a23..00ed0c604 100644 --- a/tools/test-product-search-relevance.ps1 +++ b/tools/test-product-search-relevance.ps1 @@ -137,4 +137,29 @@ Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseA $unavailableMock = { param([uri] $Uri) [pscustomobject]@{ StatusCode = 503; Content = '{"title":"Unavailable"}'; Headers = @{} } }.GetNewClosure() Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $unavailableMock } 'Service-unavailable responses must stop evaluation.' +$tuningOnlyMock = { + param([uri] $Uri) + if ($Uri.AbsolutePath -eq '/api/tags') { throw 'Full-text must not contact Ollama.' } + if ($Uri.AbsolutePath -eq '/api/products/$query') { + return [pscustomobject]@{ StatusCode = 200; Content = '{"search":{"mode":"fulltext","ranked":true}}'; Headers = @{} } + } + if ($Uri.Query.Contains('$search=')) { + $searchPart = $Uri.Query.TrimStart([char]'?') -split '&' | Where-Object { $_.StartsWith('$search=') } | Select-Object -First 1 + $queryText = [uri]::UnescapeDataString($searchPart.Substring(8)) + $query = $dataset.queries | Where-Object query -EQ $queryText | Select-Object -First 1 + if ($query.split -ne 'tuning') { throw "Non-tuning query executed: $queryText" } + return [pscustomobject]@{ StatusCode = 200; Content = '[]'; Headers = @{ 'X-Paging-Total-Count' = '0' } } + } + & $validMock $Uri +}.GetNewClosure() +$lexical = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $tuningOnlyMock -Approach fulltext -Split tuning -BaselinePath "$PSScriptRoot\..\samples\docs\product-search-relevance-baseline.json" +Assert-Equal $lexical.queries.Count 18 'Tuning-only must execute exactly 18 queries.' +Assert-Equal $lexical.splits.ContainsKey('held-out') $false 'Held-out metrics must not be generated while tuning.' +Assert-Equal $lexical.model $null 'Full-text must not claim model inference.' +Assert-Equal $lexical.splits.tuning.meanNdcgAt5 0 'Zero matching results are legitimate lexical results and must score zero.' +Assert-Equal $lexical.tuningComparison.split 'tuning' 'Only tuning scores may be compared with the baseline.' +Assert-Equal $mockReport.selectedSplit 'all' 'The original semantic full evaluation remains supported.' +Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $tuningOnlyMock -Approach fulltext } 'Full-text must require an explicit tuning-only split.' +Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $tuningOnlyMock -Approach fulltext -Split held-out } 'Held-out lexical execution must be rejected during this tuning milestone.' + Write-Host 'Product-search relevance tests passed: metric math, dataset validation, catalog drift, backfill completeness, constraint violations, and mocked HTTP failures.' From 0b6fd9d35d363fe94f6e9dcf3a8a4c64b8eba26f Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Thu, 15 Oct 2026 16:29:04 -0700 Subject: [PATCH 11/25] Add hybrid search and shared mountain bike demo catalog Combine bounded lexical and semantic retrieval with reciprocal rank fusion. Expand the reviewed demo catalog and evaluation set, share catalog and reference data across demo setup, bootstrap Shopping through its sync adapter, and target the E2E Runner at net10. Normalize internal catalog SKUs without artificial DEMO prefixes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .../workflows/product-search-relevance.yml | 6 +- Directory.Packages.props | 8 +- samples/README.md | 2 +- .../Contoso.Test.Aspire.csproj | 1 + .../DemoCatalogBootstrapTests.cs | 215 + .../aspire/Contoso.Test.Aspire/GlobalUsing.cs | 3 + samples/data/demo-catalog.seed.yaml | 187 + samples/data/demo-catalog.sources.json | 2399 +++++++++ samples/docs/aspire.md | 76 +- samples/docs/search.md | 200 +- samples/docs/testing.md | 5 + .../src/Contoso.Mcp/Tools/CommerceTools.cs | 4 +- .../Contoso.Products.Database.csproj | 1 + .../Data/ref-data.seed.yaml | 14 +- .../src/Contoso.Products.Database/Program.cs | 5 +- .../Contoso.Products.Infrastructure.csproj | 3 + .../Repositories/ProductQueryArgsConfig.cs | 22 +- .../Repositories/ProductQueryRepository.cs | 112 +- .../Repositories/ProductSearchFusion.cs | 100 + .../Contoso.E2E.Runner.csproj | 8 + .../Data/demo-baskets.seed.yaml | 24 + .../tests/Contoso.E2E.Runner/GlobalUsing.cs | 16 +- .../Infrastructure/DemoCatalogBootstrap.cs | 254 + samples/tests/Contoso.E2E.Runner/Program.cs | 14 +- .../Scenarios/DatabaseMigrationSetup.cs | 29 +- .../Contoso.Products.Test.Api/GlobalUsing.cs | 1 + .../ProductHybridSearchTests.cs | 254 + .../ProductLexicalSearchTests.cs | 4 +- .../ProductSearchRelevanceSetupTests.cs | 59 +- .../TestEmbeddingAdapter.cs | 12 +- tools/product-search-relevance.dataset.json | 4437 ++++++++++++++++- tools/product-search-relevance.ps1 | 163 +- tools/product-search-relevance.seed.yaml | 45 - tools/test-product-search-relevance.ps1 | 75 +- 34 files changed, 8322 insertions(+), 436 deletions(-) create mode 100644 samples/aspire/Contoso.Test.Aspire/DemoCatalogBootstrapTests.cs create mode 100644 samples/data/demo-catalog.seed.yaml create mode 100644 samples/data/demo-catalog.sources.json create mode 100644 samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchFusion.cs create mode 100644 samples/tests/Contoso.E2E.Runner/Data/demo-baskets.seed.yaml create mode 100644 samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogBootstrap.cs create mode 100644 samples/tests/Contoso.Products.Test.Api/ProductHybridSearchTests.cs delete mode 100644 tools/product-search-relevance.seed.yaml diff --git a/.github/workflows/product-search-relevance.yml b/.github/workflows/product-search-relevance.yml index cae6b04a7..bf5a2cbc6 100644 --- a/.github/workflows/product-search-relevance.yml +++ b/.github/workflows/product-search-relevance.yml @@ -66,7 +66,7 @@ jobs: dotnet build samples/tests/Contoso.Products.Test.Api/Contoso.Products.Test.Api.csproj --no-restore --configuration Release --framework net10.0 - name: Apply schema and reference data to the isolated database - run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- all --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" + run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- database --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" - name: Reset only the isolated Products schema run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- reset --accept-prompts --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" @@ -74,8 +74,8 @@ jobs: - name: Reload reference data run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- data --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" - - name: Load the dedicated evaluation catalog - run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- execute "$GITHUB_WORKSPACE/tools/product-search-relevance.seed.yaml" --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" + - name: Load the shared demo catalog + run: dotnet run --no-launch-profile --project samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj --no-build --configuration Release --framework net10.0 -- execute "$GITHUB_WORKSPACE/samples/data/demo-catalog.seed.yaml" --connection-string "$PRODUCTS_DATABASE_CONNECTION_STRING" - name: Prepare and verify the real-model index run: dotnet test samples/tests/Contoso.Products.Test.Api/Contoso.Products.Test.Api.csproj --no-build --configuration Release --framework net10.0 --filter "FullyQualifiedName~ProductSearchRelevanceSetupTests" --logger "console;verbosity=normal" diff --git a/Directory.Packages.props b/Directory.Packages.props index cd1c41553..2fb127c2d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -71,8 +71,6 @@ - - @@ -90,7 +88,7 @@ - + @@ -98,7 +96,7 @@ - + @@ -106,7 +104,7 @@ - + diff --git a/samples/README.md b/samples/README.md index 50346e06d..4ffb95ae1 100644 --- a/samples/README.md +++ b/samples/README.md @@ -158,7 +158,7 @@ dotnet run --project samples/src/Contoso.Orders.Database -- All > Customers (Cosmos DB) is schemaless, so `Contoso.Customers.Database` is not a schema migration tool: it declares the containers and seed data and runs them through the reusable `CoreEx.Cosmos.Provisioning` console (`dotnet run --project samples/src/Contoso.Customers.Database -- All`; use `DropAndAll` or `Create,ResetAndData` to start clean, noting `Drop`/`Reset` prompt for confirmation unless `--accept-prompts` is specified; `ResetAndData` alone fails where the database does not yet exist). Tests reuse the same definition via `Test.MigrateCosmosDataAsync(...)`. -> The E2E runner's **Database Migration and Base Data Refresh** option can also apply pending migrations across all domains without restarting hosts. See [Aspire & E2E](docs/aspire.md) for details. +> The E2E runner's **Database Migration and Base Data Refresh** option loads the [shared demo catalog](docs/aspire.md#shared-demo-catalog-bootstrap), bootstraps Shopping and prepares real-model search. This is a destructive reset: stop writers and restart serving hosts/caches afterwards. See [Aspire & E2E](docs/aspire.md) for details. See [Tooling](docs/tooling.md) for the full list of database commands and what each does. diff --git a/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj b/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj index 1c24ba83a..ec8af1af1 100644 --- a/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj +++ b/samples/aspire/Contoso.Test.Aspire/Contoso.Test.Aspire.csproj @@ -38,6 +38,7 @@ + diff --git a/samples/aspire/Contoso.Test.Aspire/DemoCatalogBootstrapTests.cs b/samples/aspire/Contoso.Test.Aspire/DemoCatalogBootstrapTests.cs new file mode 100644 index 000000000..03fbbdf8b --- /dev/null +++ b/samples/aspire/Contoso.Test.Aspire/DemoCatalogBootstrapTests.cs @@ -0,0 +1,215 @@ +namespace Contoso.Test.Aspire; + +/// Exercises the composition-only snapshot bridge over isolated real PostgreSQL and SQL Server databases. +[TestFixture, NonParallelizable, Explicit("Resets only contoso_search_eval and Contoso_demo_eval and requires the configured real embedding model.")] +public class DemoCatalogBootstrapTests +{ + private IConfiguration _configuration = new ConfigurationBuilder().Build(); + private string _productsConnection = string.Empty; + private string _shoppingConnection = string.Empty; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _configuration = new ConfigurationBuilder() + .SetBasePath(AppContext.BaseDirectory) + .AddJsonFile("products-settings.json") + .AddJsonFile("e2e-settings.json") + .AddEnvironmentVariables() + .Build(); + + _productsConnection = _configuration["E2E:Products:ConnectionString"].ThrowIfNullOrEmpty(); + _shoppingConnection = _configuration["E2E:Shopping:ConnectionString"].ThrowIfNullOrEmpty(); + + if (new Npgsql.NpgsqlConnectionStringBuilder(_productsConnection).Database != "contoso_search_eval" + || new Microsoft.Data.SqlClient.SqlConnectionStringBuilder(_shoppingConnection).InitialCatalog != "Contoso_demo_eval") + throw new InvalidOperationException("Demo bootstrap tests require E2E__Products__ConnectionString targeting contoso_search_eval and E2E__Shopping__ConnectionString targeting Contoso_demo_eval. No shared database may be reset."); + + if (StackExchange.Redis.ConfigurationOptions.Parse(_configuration["Aspire:StackExchange:Redis:ConnectionString"].ThrowIfNullOrEmpty()).DefaultDatabase != 14) + throw new InvalidOperationException("Demo bootstrap tests require an isolated Redis connection with defaultDatabase=14; shared serving reference caches must not be modified."); + } + + [Test] + public async Task DemoCatalog_ResetReconcileAndRepeat_ActualSourceAndSearchAreReady() + { + using (var cacheHost = DemoCatalogBootstrap.CreateServingCacheHost(_configuration, "Products")) + { + await using var scope = cacheHost.Services.CreateAsyncScope(); + var cache = scope.ServiceProvider.GetRequiredService(); + await cache.RemoveAsync(typeof(BrandCollection)).ConfigureAwait(false); + await cache.GetOrCreateAsync(typeof(BrandCollection), + (_, _) => Task.FromResult(new BrandCollection { new Brand { Id = "OLD-SNAPSHOT", Code = "OLD-SNAPSHOT", Text = "Stale brand" } })).ConfigureAwait(false); + } + + var bootstrap = await DemoCatalogBootstrap.RunAsync(_configuration).ConfigureAwait(false); + bootstrap.Should().Be(173); + + using (var cacheHost = DemoCatalogBootstrap.CreateServingCacheHost(_configuration, "Products")) + { + await using var scope = cacheHost.Services.CreateAsyncScope(); + var cache = scope.ServiceProvider.GetRequiredService(); + var brands = await cache.GetOrCreateAsync(typeof(BrandCollection), + (_, _) => Task.FromResult(new BrandCollection { new Brand { Id = "NEW-SNAPSHOT", Code = "NEW-SNAPSHOT", Text = "Fresh brand" } })).ConfigureAwait(false); + ((BrandCollection)brands).Single().Code.Should().Be("NEW-SNAPSHOT"); + await cache.RemoveAsync(typeof(BrandCollection)).ConfigureAwait(false); + } + + using var products = DemoCatalogBootstrap.CreateProductsHost(_configuration, _productsConnection); + using var shopping = DemoCatalogBootstrap.CreateShoppingHost(_configuration, _shoppingConnection); + await DemoCatalogBootstrap.VerifyAsync(products, shopping).ConfigureAwait(false); + + await using (var scope = products.Services.CreateAsyncScope()) + { + var ef = scope.ServiceProvider.GetRequiredService(); + var rows = await ef.Products.Model.Query().AsNoTracking().ToListAsync().ConfigureAwait(false); + rows.Should().HaveCount(173); + rows.Should().NotContain(p => p.Sku.StartsWith("DEMO-", StringComparison.Ordinal)); + rows.Select(p => p.Sku).Should().OnlyHaveUniqueItems(); + rows.Single(p => p.Id == 41.ToGuid().ToString()).Sku.Should().Be("SPECIALIZED-DEMO"); + rows.Count(p => !p.IsInactive && !p.IsNonStocked).Should().Be(170); + rows.Count(p => p.IsInactive).Should().Be(2); + var labor = rows.Single(p => p.Sku == "LABOR"); + labor.IsNonStocked.Should().BeTrue(); + labor.Description.Should().Contain("charged by the hour").And.Contain("mountain-bike maintenance and repair"); + labor.Tags.Should().Contain(["bike-service", "bicycle-repair", "workshop-labour", "workshop-labor", + "bike-mechanic", "maintenance", "wrench-monkey", "grease-monkey", "bike-whisperer", "spanner-man"]); + rows.Should().NotContain(p => p.Sku == "GIANT-TALON-2"); + + (await ef.Brands.Query().CountAsync().ConfigureAwait(false)).Should().Be(12); + (await ef.SubCategories.Query().CountAsync().ConfigureAwait(false)).Should().Be(31); + + var db = scope.ServiceProvider.GetRequiredService(); + (await db.Database.SqlQuery($"SELECT COUNT(*)::integer AS \"Value\" FROM products.product").SingleAsync().ConfigureAwait(false)).Should().Be(174); + (await db.Database.SqlQuery($"SELECT COUNT(*)::integer AS \"Value\" FROM products.product WHERE is_deleted = true").SingleAsync().ConfigureAwait(false)).Should().Be(1); + } + + await using (var scope = shopping.Services.CreateAsyncScope()) + { + var ef = scope.ServiceProvider.GetRequiredService(); + (await ef.Baskets.Query().CountAsync().ConfigureAwait(false)).Should().Be(8); + (await ef.BasketItems.Query().CountAsync().ConfigureAwait(false)).Should().Be(13); + var labor = await ef.Products.Query().SingleAsync(p => p.Sku == "LABOR").ConfigureAwait(false); + labor.UnitOfMeasureCode.Should().Be("HR"); + labor.Price.Should().Be(80); + labor.Id.Should().Be(32.ToGuid().ToString()); + } + + // Change actual source values, including flags; do not assert against a second hard-coded replica catalog. + string deletedId; + await using (var scope = products.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var ef = scope.ServiceProvider.GetRequiredService(); + var row = await ef.Products.Model.QueryTracked().SingleAsync(p => p.Sku == "FOXRACING-UNION-FLAT-SHOE").ConfigureAwait(false); + row.Sku = "SNAPSHOT-SHOE"; + row.Text = "Snapshot changed shoe"; + row.Price = 123.45m; + row.UnitOfMeasureCode = "SET"; + row.IsInactive = true; + row.IsNonStocked = true; + + var deleted = await ef.Products.Model.QueryTracked().SingleAsync(p => p.Sku == "FOXRACING-UNION-BOA-FLAT").ConfigureAwait(false); + deletedId = deleted.Id; + deleted.IsDeleted = true; + + await db.SaveChangesAsync().ConfigureAwait(false); + } + + var mismatch = () => DemoCatalogBootstrap.VerifyAsync(products, shopping); + await mismatch.Should().ThrowAsync().WithMessage("*replica mismatch*").ConfigureAwait(false); + (await DemoCatalogBootstrap.SynchronizeAsync(products, shopping).ConfigureAwait(false)).Should().Be(172); + await DemoCatalogBootstrap.VerifyAsync(products, shopping).ConfigureAwait(false); + (await DemoCatalogBootstrap.SynchronizeAsync(products, shopping).ConfigureAwait(false)).Should().Be(172); + + await using (var scope = shopping.Services.CreateAsyncScope()) + { + var ef = scope.ServiceProvider.GetRequiredService(); + + (await ef.Products.Query().AnyAsync(p => p.Id == deletedId).ConfigureAwait(false)).Should().BeFalse(); + var changed = await ef.Products.Query().SingleAsync(p => p.Sku == "SNAPSHOT-SHOE").ConfigureAwait(false); + changed.Id.Should().Be(138.ToGuid().ToString()); + changed.Text.Should().Be("Snapshot changed shoe"); + changed.Price.Should().Be(123.45m); + changed.UnitOfMeasureCode.Should().Be("SET"); + changed.IsInactive.Should().BeTrue(); + changed.IsNonStocked.Should().BeTrue(); + + await ef.Products.CreateAsync(new Contoso.Shopping.Infrastructure.Persistence.Product + { + Id = "snapshot-stale-product", Sku = "SNAPSHOT-STALE", Text = "Stale snapshot", UnitOfMeasureCode = "EA", Price = 1 + }).ConfigureAwait(false); + } + + (await DemoCatalogBootstrap.SynchronizeAsync(products, shopping).ConfigureAwait(false)).Should().Be(172); + await DemoCatalogBootstrap.VerifyAsync(products, shopping).ConfigureAwait(false); + + await using (var scope = shopping.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var ef = scope.ServiceProvider.GetRequiredService(); + var item = await ef.BasketItems.QueryTracked().FirstAsync().ConfigureAwait(false); + var originalId = item.ProductId; + try + { + item.ProductId = "missing-snapshot-product"; + await db.SaveChangesAsync().ConfigureAwait(false); + var integrity = () => DemoCatalogBootstrap.VerifyAsync(products, shopping); + await integrity.Should().ThrowAsync().WithMessage("*missing product or basket*").ConfigureAwait(false); + } + finally + { + item.ProductId = originalId; + await db.SaveChangesAsync().ConfigureAwait(false); + } + } + + await using (var scope = products.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var ef = scope.ServiceProvider.GetRequiredService(); + var brand = await ef.Brands.Query().SingleAsync(b => b.Code == "FOX").ConfigureAwait(false); + try + { + db.Remove(brand); + await db.SaveChangesAsync().ConfigureAwait(false); + using var fresh = DemoCatalogBootstrap.CreateProductsHost(_configuration, _productsConnection); + var reference = () => DemoCatalogBootstrap.VerifyAsync(fresh, shopping); + await reference.Should().ThrowAsync().WithMessage("*reference data is missing*").ConfigureAwait(false); + } + finally + { + db.Add(brand); + await db.SaveChangesAsync().ConfigureAwait(false); + } + } + + using var cancelled = new CancellationTokenSource(); + cancelled.Cancel(); + var cancel = () => DemoCatalogBootstrap.RunAsync(_configuration, cancelled.Token); + await cancel.Should().ThrowAsync().ConfigureAwait(false); + await DemoCatalogBootstrap.VerifyAsync(products, shopping).ConfigureAwait(false); + + var brokenConfiguration = new ConfigurationBuilder().AddConfiguration(_configuration) + .AddInMemoryCollection(new Dictionary { ["Ollama:BaseAddress"] = "http://127.0.0.1:1/" }).Build(); + using (var broken = DemoCatalogBootstrap.CreateProductsHost(brokenConfiguration, _productsConnection)) + { + var rebuild = () => DemoCatalogBootstrap.PrepareSearchAsync(broken); + var failure = await rebuild.Should().ThrowAsync().WithMessage("*search preparation failed*").ConfigureAwait(false); + failure.Which.InnerException.Should().NotBeNull(); + failure.Which.ToString().Should().Contain("127.0.0.1:1"); + } + + // A full rerun must clear stale projections and restore deterministic product/basket identities. + (await DemoCatalogBootstrap.RunAsync(_configuration).ConfigureAwait(false)).Should().Be(173); + await DemoCatalogBootstrap.VerifyAsync(products, shopping).ConfigureAwait(false); + await using (var scope = products.Services.CreateAsyncScope()) + { + await scope.ServiceProvider.GetRequiredService().EnsureReadyAsync().ConfigureAwait(false); + await scope.ServiceProvider.GetRequiredService().EnsureReadyAsync().ConfigureAwait(false); + var db = scope.ServiceProvider.GetRequiredService(); + var profile = scope.ServiceProvider.GetRequiredService().Profile; + (await db.Database.SqlQuery($"SELECT COUNT(*)::integer AS \"Value\" FROM products.product_search WHERE profile = {profile}").SingleAsync().ConfigureAwait(false)).Should().Be(173); + (await db.Database.SqlQuery($"SELECT COUNT(*)::integer AS \"Value\" FROM products.product_lexical_search").SingleAsync().ConfigureAwait(false)).Should().Be(173); + } + } +} diff --git a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs index fbea82256..8d5d31dde 100644 --- a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs +++ b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs @@ -4,11 +4,14 @@ global using AwesomeAssertions; global using Azure.Messaging.ServiceBus; global using Azure.Messaging.ServiceBus.Administration; +global using Contoso.E2E.Runner.Infrastructure; +global using Contoso.Products.Application.Adapters.Embeddings; global using Contoso.Products.Contracts; global using Contoso.Products.Infrastructure.Clients.Ollama; global using Contoso.Products.Infrastructure.Repositories; global using Contoso.Shopping.Contracts; global using CoreEx; +global using Microsoft.EntityFrameworkCore; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Hosting; diff --git a/samples/data/demo-catalog.seed.yaml b/samples/data/demo-catalog.seed.yaml new file mode 100644 index 000000000..c9deb41ad --- /dev/null +++ b/samples/data/demo-catalog.seed.yaml @@ -0,0 +1,187 @@ +products: + - product: + - { product_id: ^1, sku: YETI-ASR-C2-2025, text: Yeti ASR C2, sub_category_code: XC, unit_of_measure_code: EA, price: 5800, brand_code: YETI, tags_json: '["cross-country","full-suspension","mountain-bike","dentist-bike"]', description: 'A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack.' } + - { product_id: ^2, sku: YETI-SB120-C2, text: Yeti SB120 C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: YETI, tags_json: '["mountain-bike","dentist-bike"]', description: 'A carbon full-suspension trail mountain bike designed for efficient climbs, responsive handling and flowing singletrack descents.' } + - { product_id: ^3, sku: YETI-SB140-LR-C2, text: Yeti SB140 LR C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6600, brand_code: YETI, tags_json: '["mountain-bike","dentist-bike"]', description: 'A carbon full-suspension trail mountain bike in the Lunch Ride configuration, built for technical climbs and rough, fast descents.' } + - { product_id: ^4, sku: YETI-SB165-C3, text: Yeti SB165 C3, sub_category_code: EN, unit_of_measure_code: EA, price: 7200, brand_code: YETI, tags_json: '["mountain-bike","dentist-bike"]', description: 'A carbon full-suspension mountain bike for gravity-focused riding, steep descents and demanding bike-park terrain.' } + - { product_id: ^5, sku: YETI-MTE-C2, text: Yeti MTe C2 eMTB, sub_category_code: EM, unit_of_measure_code: EA, price: 9850, brand_code: YETI, tags_json: '["mountain-bike","dentist-bike","electric","e-bike","emtb"]', description: 'An electric-assist mountain bike with a carbon full-suspension frame, designed for technical trail riding and assisted climbs.' } + - { product_id: ^6, sku: CANYON-LUXTRAIL-CF6, text: Canyon Lux Trail CF 6, sub_category_code: XC, unit_of_measure_code: EA, price: 3899, brand_code: CANYON, tags_json: '["mountain-bike"]', description: 'A carbon full-suspension mountain bike combining cross-country pedalling efficiency with trail capability for long rides and technical singletrack.' } + - { product_id: ^7, sku: CANYON-SPECTRAL-CF8, text: Canyon Spectral CF 8, sub_category_code: TR, unit_of_measure_code: EA, price: 4999, brand_code: CANYON, tags_json: '["mountain-bike"]', description: 'A carbon full-suspension trail mountain bike for varied singletrack, balancing climbing capability with control on rough, technical descents.' } + - { product_id: ^8, sku: CANYON-NEURON-CF7, text: Canyon Neuron CF 7, sub_category_code: TR, unit_of_measure_code: EA, price: 3499, brand_code: CANYON, tags_json: '["mountain-bike"]', description: 'A carbon full-suspension trail mountain bike for long rides over varied terrain, combining efficient climbing with stable descending.' } + - { product_id: ^9, sku: CANYON-TORQUE-MULLET-CF8, text: Canyon Torque Mullet CF 8, sub_category_code: EN, unit_of_measure_code: EA, price: 3599, brand_code: CANYON, tags_json: '["mountain-bike"]', description: 'A carbon mixed-wheel mountain bike designed for freeride and bike-park use, including jumps, drops and steep descents.' } + - { product_id: ^10, sku: CANYON-SENDER-CFR, text: Canyon Sender CFR, sub_category_code: DH, unit_of_measure_code: EA, price: 7799, brand_code: CANYON, tags_json: '["mountain-bike"]', description: 'A carbon downhill mountain bike designed for racing and fast descending on steep, demanding tracks and bike-park terrain.' } + - { product_id: ^11, sku: SPEC-STUMPJUMPER-15-COMP, text: Specialized Stumpjumper 15 Comp, sub_category_code: TR, unit_of_measure_code: EA, price: 5500, brand_code: SPEC, tags_json: '["mountain-bike"]', description: 'A full-suspension trail mountain bike with adjustable geometry, designed to balance climbing efficiency and control on technical descents.' } + - { product_id: ^12, sku: SPEC-ENDURO-PRO, text: Specialized Enduro Pro, sub_category_code: EN, unit_of_measure_code: EA, price: 9500, brand_code: SPEC, tags_json: '["mountain-bike"]', description: 'A carbon full-suspension enduro mountain bike built for steep, rough descents and enduro racing while retaining the ability to pedal uphill.' } + - { product_id: ^13, sku: SPEC-EPIC-8-PRO, text: Specialized Epic 8 Pro, sub_category_code: XC, unit_of_measure_code: EA, price: 9000, is_inactive: true, brand_code: SPEC, tags_json: '["mountain-bike"]', description: 'A carbon full-suspension cross-country mountain bike designed for efficient pedalling, competitive racing and technical singletrack.' } + - { product_id: ^14, sku: SPEC-CHISEL-COMP, text: Specialized Chisel Comp, sub_category_code: HT, unit_of_measure_code: EA, price: 3400, brand_code: SPEC, tags_json: '["mountain-bike"]', description: 'An aluminium cross-country mountain bike designed for efficient pedalling, racing and fast singletrack riding.' } + - { product_id: ^15, sku: SPEC-STATUS-140, text: Specialized Status 140, sub_category_code: FR, unit_of_measure_code: EA, price: 2499, brand_code: SPEC, tags_json: '["mountain-bike"]', description: 'An aluminium mixed-wheel full-suspension mountain bike designed for playful trail riding, jumps and technical descents.' } + - { product_id: ^16, sku: GIANT-TRANCE-X-ADVPRO29, text: Giant Trance X Advanced Pro 29, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: GIANT, tags_json: '["mountain-bike"]', description: 'A carbon full-suspension trail mountain bike with 29-inch wheels and adjustable geometry, designed for climbing and technical descending.' } + - { product_id: ^17, sku: GIANT-ANTHEM-ADV29-1, text: Giant Anthem Advanced 29 1, sub_category_code: XC, unit_of_measure_code: EA, price: 7800, brand_code: GIANT, tags_json: '["mountain-bike"]', description: 'A carbon full-suspension cross-country mountain bike with 29-inch wheels, designed for efficient pedalling and fast riding on demanding race courses.' } + - { product_id: ^18, sku: GIANT-TALON-2, text: Giant Talon 2, sub_category_code: HT, unit_of_measure_code: EA, price: 800, is_deleted: true, brand_code: GIANT, tags_json: '["mountain-bike"]', description: 'An aluminium hardtail mountain bike with a suspension fork, designed as an accessible introduction to off-road trails.' } + - { product_id: ^19, sku: SHIMANO-XT-M8100-RD, text: Shimano Deore XT M8100 Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 129, brand_code: SHIMANO, tags_json: '["mountain-bike"]', description: 'A 12-speed mountain bike rear derailleur that moves the chain between cassette sprockets to change gears. Its clutch helps control chain movement on rough trails.' } + - { product_id: ^20, sku: SHIMANO-SLX-M7100-CASSETTE, text: Shimano SLX M7100 12-speed Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 99, brand_code: SHIMANO, tags_json: '["mountain-bike"]', description: 'A 12-speed mountain bike cassette that provides a wide range of gears for climbing and descending. It uses Shimano Micro Spline mounting and Hyperglide Plus shifting.' } + - { product_id: ^21, sku: SHIMANO-XTR-M9100-CRANK, text: Shimano XTR M9100 Crankset, sub_category_code: CR, unit_of_measure_code: EA, price: 499, brand_code: SHIMANO, tags_json: '["mountain-bike"]', description: 'A mountain bike crankset that transfers pedalling power to the chain through a direct-mount chainring, designed for lightweight cross-country and trail use.' } + - { product_id: ^22, sku: SRAM-GX-EAGLE-RD, text: SRAM GX Eagle Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 135, brand_code: SRAM, tags_json: '["mountain-bike"]', description: 'A cable-operated 12-speed mountain bike rear derailleur for SRAM Eagle drivetrains. It shifts the chain between rear sprockets and uses a clutch to control chain movement.' } + - { product_id: ^23, sku: SRAM-X0-TRANSMISSION-RD, text: SRAM X0 Transmission Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 400, brand_code: SRAM, tags_json: '["mountain-bike"]', description: 'A wireless electronic mountain bike rear derailleur for SRAM Eagle Transmission drivetrains. Its hangerless Full Mount design attaches around the rear axle on compatible frames.' } + - { product_id: ^24, sku: SRAM-XX1-EAGLE-CASSETTE, text: SRAM XX1 Eagle Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 449, brand_code: SRAM, tags_json: '["mountain-bike"]', description: 'A lightweight 12-speed mountain bike cassette with a wide gear range for climbing and descending, designed for SRAM Eagle drivetrains and XD freehub drivers.' } + - { product_id: ^25, sku: ONEUP-V3-DROPPER-180, text: OneUp Dropper Post V3 180mm, sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP, tags_json: '["mountain-bike"]', description: 'A mountain bike dropper seatpost with 180 mm of travel. Riders can lower the saddle for descending and raise it for pedalling using a compatible remote.' } + - { product_id: ^26, sku: ONEUP-CARBON-HANDLEBAR, text: OneUp Carbon Handlebar, sub_category_code: HB, unit_of_measure_code: EA, price: 159, brand_code: ONEUP, tags_json: '["mountain-bike"]', description: 'A carbon mountain bike handlebar with a shaped profile designed to absorb vibration and reduce hand fatigue while maintaining steering control.' } + - { product_id: ^27, sku: ONEUP-COMPOSITE-PEDALS, text: OneUp Composite Pedals, sub_category_code: PE, unit_of_measure_code: PR, price: 49, is_inactive: true, brand_code: ONEUP, tags_json: '["mountain-bike"]', description: 'Flat mountain bike pedals with composite bodies, broad platforms and replaceable traction pins, designed to grip ordinary flat-soled riding shoes.' } + - { product_id: ^28, sku: ONEUP-EDC-TOOL-V2, text: OneUp EDC Tool V2, sub_category_code: TO, unit_of_measure_code: EA, price: 59, brand_code: ONEUP, tags_json: '["mountain-bike"]', description: 'A compact trailside repair multi-tool with hex keys, a Torx key, a chain breaker and a tyre lever, designed for storage in compatible OneUp steerer or pump systems.' } + - { product_id: ^29, sku: MUCOFF-NANO-CLEANER, text: Muc-Off Nano Tech Bike Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 16.99, brand_code: MUCOFF, tags_json: '["mountain-bike"]', description: 'A spray-on bicycle cleaner that breaks down dirt and grime on frames and components before rinsing. Suitable for routine washing after muddy rides.' } + - { product_id: ^30, sku: MUCOFF-TUBELESS-SEALANT, text: Muc-Off MTB Tubeless Sealant, sub_category_code: SE, unit_of_measure_code: EA, price: 8.00, brand_code: MUCOFF, tags_json: '["mountain-bike"]', description: 'A tubeless mountain bike tyre sealant that helps seal punctures and maintain an airtight tyre system. Its UV-reactive dye helps locate leaks.' } + - { product_id: ^31, sku: MUCOFF-DRIVETRAIN-CLEANER, text: Muc-Off Drivetrain Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 25.00, brand_code: MUCOFF, tags_json: '["mountain-bike"]', description: 'A bicycle drivetrain cleaner that removes grease, oil and grime from chains, cassettes, chainrings and derailleurs. It can be sprayed on or used with a chain-cleaning device.' } + - { product_id: ^33, sku: YETI-ARC, text: "Yeti ARC Mountain Bike", sub_category_code: HT, unit_of_measure_code: EA, price: 3200, brand_code: YETI, tags_json: '["mountain-bike","hardtail","cross-country","dentist-bike"]', description: 'A hardtail mountain bike in Yeti Cycles'' ARC model family for cross-country and trail riding.' } + - { product_id: ^34, sku: YETI-SB135, text: "Yeti SB135 Mountain Bike", sub_category_code: TR, unit_of_measure_code: EA, price: 6400, brand_code: YETI, tags_json: '["mountain-bike","trail","dentist-bike"]', description: 'A full-suspension trail mountain bike in Yeti Cycles'' SB135 model family.' } + - { product_id: ^35, sku: YETI-SB160, text: "Yeti SB160 Mountain Bike", sub_category_code: EN, unit_of_measure_code: EA, price: 7900, brand_code: YETI, tags_json: '["mountain-bike","enduro","dentist-bike"]', description: 'A full-suspension enduro mountain bike in Yeti Cycles'' SB160 model family.' } + - { product_id: ^36, sku: CANYON-STOIC, text: "Canyon Stoic Mountain Bike", sub_category_code: HT, unit_of_measure_code: EA, price: 1899, brand_code: CANYON, tags_json: '["mountain-bike","hardtail","trail"]', description: 'A hardtail mountain bike in Canyon''s Stoic model family for trail riding.' } + - { product_id: ^37, sku: CANYON-STRIVE, text: "Canyon Strive Mountain Bike", sub_category_code: EN, unit_of_measure_code: EA, price: 5799, brand_code: CANYON, tags_json: '["mountain-bike","enduro"]', description: 'A full-suspension enduro mountain bike in Canyon''s Strive model family.' } + - { product_id: ^38, sku: CANYON-GRAND-CANYON, text: "Canyon Grand Canyon Mountain Bike", sub_category_code: HT, unit_of_measure_code: EA, price: 1499, brand_code: CANYON, tags_json: '["mountain-bike","hardtail"]', description: 'A hardtail mountain bike in Canyon''s Grand Canyon model family for off-road riding.' } + - { product_id: ^39, sku: CANYON-LUX-WORLD-CUP, text: "Canyon Lux World Cup Mountain Bike", sub_category_code: XC, unit_of_measure_code: EA, price: 6299, brand_code: CANYON, tags_json: '["mountain-bike","cross-country","race"]', description: 'A full-suspension cross-country race mountain bike in Canyon''s Lux World Cup family.' } + - { product_id: ^40, sku: SPECIALIZED-FUSE, text: "Specialized Fuse Mountain Bike", sub_category_code: HT, unit_of_measure_code: EA, price: 2799, brand_code: SPEC, tags_json: '["mountain-bike","hardtail","trail"]', description: 'A hardtail mountain bike in Specialized''s Fuse model family for trail riding.' } + - { product_id: ^41, sku: SPECIALIZED-DEMO, text: "Specialized Demo Mountain Bike", sub_category_code: DH, unit_of_measure_code: EA, price: 6499, brand_code: SPEC, tags_json: '["mountain-bike","downhill"]', description: 'A downhill mountain bike in Specialized''s Demo model family.' } + - { product_id: ^42, sku: SPECIALIZED-TURBO-LEVO, text: "Specialized Turbo Levo E-Mountain Bike", sub_category_code: EM, unit_of_measure_code: EA, price: 8999, brand_code: SPEC, tags_json: '["mountain-bike","electric","e-bike","emtb"]', description: 'An electric-assist mountain bike in Specialized''s Turbo Levo model family.' } + - { product_id: ^43, sku: SPECIALIZED-TURBO-KENEVO, text: "Specialized Turbo Kenevo E-Mountain Bike", sub_category_code: EM, unit_of_measure_code: EA, price: 9999, brand_code: SPEC, tags_json: '["mountain-bike","electric","enduro","e-bike","emtb"]', description: 'An electric-assist mountain bike in Specialized''s Turbo Kenevo model family.' } + - { product_id: ^44, sku: SPECIALIZED-EPIC-WORLD-CUP, text: "Specialized Epic World Cup Mountain Bike", sub_category_code: XC, unit_of_measure_code: EA, price: 8999, brand_code: SPEC, tags_json: '["mountain-bike","cross-country","race"]', description: 'A cross-country race mountain bike in Specialized''s Epic World Cup family.' } + - { product_id: ^45, sku: GIANT-TRANCE, text: "Giant Trance Mountain Bike", sub_category_code: TR, unit_of_measure_code: EA, price: 4599, brand_code: GIANT, tags_json: '["mountain-bike","trail"]', description: 'A full-suspension trail mountain bike in Giant''s Trance model family.' } + - { product_id: ^46, sku: GIANT-REIGN, text: "Giant Reign Mountain Bike", sub_category_code: EN, unit_of_measure_code: EA, price: 5499, brand_code: GIANT, tags_json: '["mountain-bike","enduro"]', description: 'A full-suspension enduro mountain bike in Giant''s Reign model family.' } + - { product_id: ^47, sku: GIANT-FATHOM, text: "Giant Fathom Mountain Bike", sub_category_code: HT, unit_of_measure_code: EA, price: 1699, brand_code: GIANT, tags_json: '["mountain-bike","hardtail","trail"]', description: 'A hardtail mountain bike in Giant''s Fathom model family for trail riding.' } + - { product_id: ^48, sku: GIANT-GLORY, text: "Giant Glory Mountain Bike", sub_category_code: DH, unit_of_measure_code: EA, price: 7199, brand_code: GIANT, tags_json: '["mountain-bike","downhill"]', description: 'A downhill mountain bike in Giant''s Glory model family.' } + - { product_id: ^49, sku: GIANT-STANCE, text: "Giant Stance Mountain Bike", sub_category_code: TR, unit_of_measure_code: EA, price: 2499, brand_code: GIANT, tags_json: '["mountain-bike","trail"]', description: 'A full-suspension trail mountain bike in Giant''s Stance model family.' } + - { product_id: ^50, sku: SPECIALIZED-STATUS-2, text: "Specialized Status 2 Mountain Bike", sub_category_code: FR, unit_of_measure_code: EA, price: 3299, brand_code: SPEC, tags_json: '["mountain-bike","freeride"]', description: 'A mixed-wheel mountain bike in Specialized''s Status model family for trail and freeride use.' } + - { product_id: ^51, sku: GIANT-ANTHEM, text: "Giant Anthem Mountain Bike", sub_category_code: XC, unit_of_measure_code: EA, price: 6399, brand_code: GIANT, tags_json: '["mountain-bike","cross-country"]', description: 'A full-suspension cross-country mountain bike in Giant''s Anthem model family.' } + - { product_id: ^52, sku: SHIMANO-SLX-M7100-DERAILLEUR, text: "Shimano SLX M7100 Rear Derailleur", sub_category_code: DR, unit_of_measure_code: EA, price: 89, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","derailleur"]', description: 'A rear derailleur from Shimano''s SLX M7100 mountain bike component series.' } + - { product_id: ^53, sku: SHIMANO-SLX-M7100-SHIFTER, text: "Shimano SLX M7100 Shifter", sub_category_code: DR, unit_of_measure_code: EA, price: 49, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","shifter"]', description: 'A mountain bike gear shifter from Shimano''s SLX M7100 component series.' } + - { product_id: ^54, sku: SHIMANO-SLX-M7100-CHAIN, text: "Shimano SLX M7100 Chain", sub_category_code: CH, unit_of_measure_code: EA, price: 39, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","chain"]', description: 'A bicycle chain from Shimano''s SLX M7100 mountain bike component series.' } + - { product_id: ^55, sku: SHIMANO-SLX-M7100-CRANKSET, text: "Shimano SLX M7100 Crankset", sub_category_code: CR, unit_of_measure_code: EA, price: 149, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","crankset"]', description: 'A mountain bike crankset from Shimano''s SLX M7100 component series.' } + - { product_id: ^56, sku: SHIMANO-XT-M8100-SHIFTER, text: "Shimano Deore XT M8100 Shifter", sub_category_code: DR, unit_of_measure_code: EA, price: 69, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","shifter"]', description: 'A mountain bike gear shifter from Shimano''s Deore XT M8100 component series.' } + - { product_id: ^57, sku: SHIMANO-XT-M8100-CHAIN, text: "Shimano Deore XT M8100 Chain", sub_category_code: CH, unit_of_measure_code: EA, price: 54, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","chain"]', description: 'A bicycle chain from Shimano''s Deore XT M8100 mountain bike component series.' } + - { product_id: ^58, sku: SHIMANO-XT-M8100-CRANKSET, text: "Shimano Deore XT M8100 Crankset", sub_category_code: CR, unit_of_measure_code: EA, price: 199, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","crankset"]', description: 'A mountain bike crankset from Shimano''s Deore XT M8100 component series.' } + - { product_id: ^59, sku: SHIMANO-XT-M8100-CASSETTE, text: "Shimano Deore XT M8100 Cassette", sub_category_code: CA, unit_of_measure_code: EA, price: 149, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","cassette"]', description: 'A mountain bike cassette from Shimano''s Deore XT M8100 component series.' } + - { product_id: ^60, sku: SHIMANO-XT-M8120-BRAKE, text: "Shimano Deore XT M8120 Disc Brake", sub_category_code: BR, unit_of_measure_code: EA, price: 219, brand_code: SHIMANO, tags_json: '["mountain-bike","brakes"]', description: 'A hydraulic disc brake from Shimano''s Deore XT M8120 mountain bike component series.' } + - { product_id: ^61, sku: SHIMANO-XT-M8100-BRAKE-LEVER, text: "Shimano Deore XT M8100 Brake Lever", sub_category_code: BR, unit_of_measure_code: EA, price: 89, brand_code: SHIMANO, tags_json: '["mountain-bike","brakes","lever"]', description: 'A hydraulic brake lever from Shimano''s Deore XT M8100 mountain bike component series.' } + - { product_id: ^62, sku: SHIMANO-XTR-M9100-DERAILLEUR, text: "Shimano XTR M9100 Rear Derailleur", sub_category_code: DR, unit_of_measure_code: EA, price: 259, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","derailleur"]', description: 'A rear derailleur from Shimano''s XTR M9100 mountain bike component series.' } + - { product_id: ^63, sku: SHIMANO-XTR-M9100-SHIFTER, text: "Shimano XTR M9100 Shifter", sub_category_code: DR, unit_of_measure_code: EA, price: 129, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","shifter"]', description: 'A mountain bike gear shifter from Shimano''s XTR M9100 component series.' } + - { product_id: ^64, sku: SHIMANO-XTR-M9100-CHAIN, text: "Shimano XTR M9100 Chain", sub_category_code: CH, unit_of_measure_code: EA, price: 79, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","chain"]', description: 'A bicycle chain from Shimano''s XTR M9100 mountain bike component series.' } + - { product_id: ^65, sku: SHIMANO-XTR-M9100-CASSETTE, text: "Shimano XTR M9100 Cassette", sub_category_code: CA, unit_of_measure_code: EA, price: 399, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","cassette"]', description: 'A mountain bike cassette from Shimano''s XTR M9100 component series.' } + - { product_id: ^66, sku: SHIMANO-XTR-M9100-CRANKSET, text: "Shimano XTR M9100 Crankset", sub_category_code: CR, unit_of_measure_code: EA, price: 499, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","crankset"]', description: 'A mountain bike crankset from Shimano''s XTR M9100 component series.' } + - { product_id: ^67, sku: SHIMANO-XTR-M9100-BRAKE, text: "Shimano XTR M9100 Disc Brake", sub_category_code: BR, unit_of_measure_code: EA, price: 399, brand_code: SHIMANO, tags_json: '["mountain-bike","brakes"]', description: 'A hydraulic disc brake from Shimano''s XTR M9100 mountain bike component series.' } + - { product_id: ^68, sku: SHIMANO-DEORE-M6100-DERAILLEUR, text: "Shimano Deore M6100 Rear Derailleur", sub_category_code: DR, unit_of_measure_code: EA, price: 64, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","derailleur"]', description: 'A rear derailleur from Shimano''s Deore M6100 mountain bike component series.' } + - { product_id: ^69, sku: SHIMANO-DEORE-M6100-CASSETTE, text: "Shimano Deore M6100 Cassette", sub_category_code: CA, unit_of_measure_code: EA, price: 119, brand_code: SHIMANO, tags_json: '["mountain-bike","drivetrain","cassette"]', description: 'A mountain bike cassette from Shimano''s Deore M6100 component series.' } + - { product_id: ^70, sku: SRAM-GX-EAGLE-CASSETTE, text: "SRAM GX Eagle Cassette", sub_category_code: CA, unit_of_measure_code: EA, price: 235, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","cassette"]', description: 'A mountain bike cassette from SRAM''s GX Eagle drivetrain family.' } + - { product_id: ^71, sku: SRAM-X01-EAGLE-CASSETTE, text: "SRAM X01 Eagle Cassette", sub_category_code: CA, unit_of_measure_code: EA, price: 349, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","cassette"]', description: 'A mountain bike cassette from SRAM''s X01 Eagle drivetrain family.' } + - { product_id: ^72, sku: SRAM-XX1-EAGLE-CHAIN, text: "SRAM XX1 Eagle Chain", sub_category_code: CH, unit_of_measure_code: EA, price: 89, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","chain"]', description: 'A bicycle chain from SRAM''s XX1 Eagle mountain bike drivetrain family.' } + - { product_id: ^73, sku: SRAM-GX-EAGLE-CHAIN, text: "SRAM GX Eagle Chain", sub_category_code: CH, unit_of_measure_code: EA, price: 42, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","chain"]', description: 'A bicycle chain from SRAM''s GX Eagle mountain bike drivetrain family.' } + - { product_id: ^74, sku: SRAM-GX-EAGLE-CRANKSET, text: "SRAM GX Eagle Crankset", sub_category_code: CR, unit_of_measure_code: EA, price: 159, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","crankset"]', description: 'A mountain bike crankset from SRAM''s GX Eagle drivetrain family.' } + - { product_id: ^75, sku: SRAM-GX-EAGLE-SHIFTER, text: "SRAM GX Eagle Trigger Shifter", sub_category_code: DR, unit_of_measure_code: EA, price: 49, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","shifter"]', description: 'A mountain bike gear shifter from SRAM''s GX Eagle drivetrain family.' } + - { product_id: ^76, sku: SRAM-X01-EAGLE-AXS-DERAILLEUR, text: "SRAM X01 Eagle AXS Rear Derailleur", sub_category_code: DR, unit_of_measure_code: EA, price: 499, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","derailleur","electronic"]', description: 'A wireless electronic rear derailleur from SRAM''s X01 Eagle AXS mountain bike drivetrain family.' } + - { product_id: ^77, sku: SRAM-XX1-EAGLE-AXS-DERAILLEUR, text: "SRAM XX1 Eagle AXS Rear Derailleur", sub_category_code: DR, unit_of_measure_code: EA, price: 699, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","derailleur","electronic"]', description: 'A wireless electronic rear derailleur from SRAM''s XX1 Eagle AXS mountain bike drivetrain family.' } + - { product_id: ^78, sku: SRAM-GX-TRANSMISSION-DERAILLEUR, text: "SRAM GX Eagle Transmission Rear Derailleur", sub_category_code: DR, unit_of_measure_code: EA, price: 599, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","derailleur","electronic"]', description: 'A rear derailleur from SRAM''s GX Eagle Transmission mountain bike drivetrain family.' } + - { product_id: ^79, sku: SRAM-CODE-RSC-BRAKE, text: "SRAM Code RSC Disc Brake", sub_category_code: BR, unit_of_measure_code: EA, price: 349, brand_code: SRAM, tags_json: '["mountain-bike","brakes"]', description: 'A hydraulic disc brake from SRAM''s Code mountain bike brake family.' } + - { product_id: ^80, sku: SRAM-MAVEN-ULTIMATE-BRAKE, text: "SRAM Maven Ultimate Disc Brake", sub_category_code: BR, unit_of_measure_code: EA, price: 699, brand_code: SRAM, tags_json: '["mountain-bike","brakes"]', description: 'A hydraulic disc brake from SRAM''s Maven mountain bike brake family.' } + - { product_id: ^81, sku: SRAM-MAVEN-SILVER-BRAKE, text: "SRAM Maven Silver Disc Brake", sub_category_code: BR, unit_of_measure_code: EA, price: 499, brand_code: SRAM, tags_json: '["mountain-bike","brakes"]', description: 'A hydraulic disc brake from SRAM''s Maven mountain bike brake family.' } + - { product_id: ^82, sku: SRAM-LEVEL-ULTIMATE-BRAKE, text: "SRAM Level Ultimate Disc Brake", sub_category_code: BR, unit_of_measure_code: EA, price: 399, brand_code: SRAM, tags_json: '["mountain-bike","brakes"]', description: 'A hydraulic disc brake from SRAM''s Level mountain bike brake family.' } + - { product_id: ^83, sku: SRAM-XX-TRANSMISSION-CASSETTE, text: "SRAM XX Eagle Transmission Cassette", sub_category_code: CA, unit_of_measure_code: EA, price: 699, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","cassette"]', description: 'A cassette from SRAM''s XX Eagle Transmission mountain bike drivetrain family.' } + - { product_id: ^84, sku: SRAM-XX-TRANSMISSION-CHAIN, text: "SRAM XX Eagle Transmission Chain", sub_category_code: CH, unit_of_measure_code: EA, price: 89, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","chain"]', description: 'A chain from SRAM''s XX Eagle Transmission mountain bike drivetrain family.' } + - { product_id: ^85, sku: SRAM-X01-EAGLE-CRANKSET, text: "SRAM X01 Eagle Crankset", sub_category_code: CR, unit_of_measure_code: EA, price: 329, brand_code: SRAM, tags_json: '["mountain-bike","drivetrain","crankset"]', description: 'A mountain bike crankset from SRAM''s X01 Eagle drivetrain family.' } + - { product_id: ^86, sku: FOX-32-SC-FACTORY, text: "FOX 32 Step-Cast Factory Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 1099, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork","cross-country"]', description: 'A mountain bike suspension fork in FOX''s 32 Step-Cast Factory series for cross-country riding.' } + - { product_id: ^87, sku: FOX-34-FACTORY, text: "FOX 34 Factory Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 1199, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork","trail"]', description: 'A mountain bike suspension fork in FOX''s 34 Factory series for trail and downcountry riding.' } + - { product_id: ^88, sku: FOX-34-FACTORY-GRIP-X, text: "FOX 34 Factory GRIP X Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 1199, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork","trail"]', description: 'A mountain bike suspension fork in FOX''s 34 Factory GRIP X series for trail riding.' } + - { product_id: ^89, sku: FOX-34-PERFORMANCE-ELITE, text: "FOX 34 Performance Elite Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 899, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork"]', description: 'A mountain bike suspension fork in FOX''s 34 Performance Elite series.' } + - { product_id: ^90, sku: FOX-36-SL-FACTORY, text: "FOX 36 SL Factory GRIP X Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 1199, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork","trail"]', description: 'A mountain bike suspension fork in FOX''s 36 SL Factory series for trail and downcountry riding.' } + - { product_id: ^91, sku: FOX-36-FACTORY-GRIP-X, text: "FOX 36 Factory GRIP X Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 1299, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork","trail"]', description: 'A mountain bike suspension fork in FOX''s 36 Factory GRIP X series for trail riding.' } + - { product_id: ^92, sku: FOX-36-FACTORY-GRIP-X2, text: "FOX 36 Factory GRIP X2 Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 1399, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork","all-mountain"]', description: 'A mountain bike suspension fork in FOX''s 36 Factory GRIP X2 series for all-mountain riding.' } + - { product_id: ^93, sku: FOX-36-PERFORMANCE-ELITE, text: "FOX 36 Performance Elite GRIP X Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 999, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork"]', description: 'A mountain bike suspension fork in FOX''s 36 Performance Elite series.' } + - { product_id: ^94, sku: FOX-38-FACTORY, text: "FOX 38 Factory Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 1399, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork","enduro"]', description: 'A mountain bike suspension fork in FOX''s 38 Factory series for enduro riding.' } + - { product_id: ^95, sku: FOX-38-PERFORMANCE-ELITE, text: "FOX 38 Performance Elite Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 1199, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork","enduro"]', description: 'A mountain bike suspension fork in FOX''s 38 Performance Elite series.' } + - { product_id: ^96, sku: FOX-40-FACTORY, text: "FOX 40 Factory Downhill Fork", sub_category_code: FS, unit_of_measure_code: EA, price: 1899, brand_code: FOX, tags_json: '["mountain-bike","suspension","fork","downhill"]', description: 'A dual-crown suspension fork in FOX''s 40 Factory downhill series.' } + - { product_id: ^97, sku: FOX-FLOAT-DPS-FACTORY, text: "FOX Float DPS Factory Rear Shock", sub_category_code: RS, unit_of_measure_code: EA, price: 599, brand_code: FOX, tags_json: '["mountain-bike","suspension","rear-shock"]', description: 'An air-sprung rear shock in FOX''s Float DPS Factory mountain bike suspension series.' } + - { product_id: ^98, sku: FOX-FLOAT-X-FACTORY, text: "FOX Float X Factory Rear Shock", sub_category_code: RS, unit_of_measure_code: EA, price: 699, brand_code: FOX, tags_json: '["mountain-bike","suspension","rear-shock"]', description: 'An air-sprung rear shock in FOX''s Float X Factory mountain bike suspension series.' } + - { product_id: ^99, sku: FOX-FLOAT-X2-FACTORY, text: "FOX Float X2 Factory Rear Shock", sub_category_code: RS, unit_of_measure_code: EA, price: 799, brand_code: FOX, tags_json: '["mountain-bike","suspension","rear-shock"]', description: 'An air-sprung rear shock in FOX''s Float X2 Factory mountain bike suspension series.' } + - { product_id: ^100, sku: FOX-DHX-FACTORY, text: "FOX DHX Factory Coil Shock", sub_category_code: RS, unit_of_measure_code: EA, price: 699, brand_code: FOX, tags_json: '["mountain-bike","suspension","rear-shock","coil"]', description: 'A coil-sprung rear shock in FOX''s DHX Factory mountain bike suspension series.' } + - { product_id: ^101, sku: MAXXIS-ASSEGAI, text: "Maxxis Assegai Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 99, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","enduro","downhill"]', description: 'A mountain bike tire in Maxxis''s Assegai tread family for enduro and downhill riding.' } + - { product_id: ^102, sku: MAXXIS-MINION-DHF, text: "Maxxis Minion DHF Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 95, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","front","enduro"]', description: 'A mountain bike tire in Maxxis''s Minion DHF tread family, commonly used at the front for trail, enduro, and downhill riding.' } + - { product_id: ^103, sku: MAXXIS-MINION-DHR-II, text: "Maxxis Minion DHR II Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 95, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","rear","braking"]', description: 'A mountain bike tire in Maxxis''s Minion DHR II tread family with a braking-oriented tread pattern.' } + - { product_id: ^104, sku: MAXXIS-DISSECTOR, text: "Maxxis Dissector Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 89, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","trail","enduro"]', description: 'A mountain bike tire in Maxxis''s Dissector tread family for trail and enduro riding.' } + - { product_id: ^105, sku: MAXXIS-REKON, text: "Maxxis Rekon Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 79, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","cross-country","trail"]', description: 'A mountain bike tire in Maxxis''s Rekon tread family for cross-country and trail riding.' } + - { product_id: ^106, sku: MAXXIS-REKON-RACE, text: "Maxxis Rekon Race Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 79, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","cross-country","race"]', description: 'A low-profile mountain bike tire in Maxxis''s Rekon Race family for cross-country racing.' } + - { product_id: ^107, sku: MAXXIS-FOREKASTER, text: "Maxxis Forekaster Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 84, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","trail","mixed-conditions"]', description: 'A mountain bike tire in Maxxis''s Forekaster tread family for trail riding in mixed conditions.' } + - { product_id: ^108, sku: MAXXIS-HIGH-ROLLER-II, text: "Maxxis High Roller II Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 89, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","loose-terrain"]', description: 'An aggressive mountain bike tire in Maxxis''s High Roller II tread family for loose and varied terrain.' } + - { product_id: ^109, sku: MAXXIS-SHORTY, text: "Maxxis Shorty Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 99, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","mud","downhill"]', description: 'A mountain bike tire in Maxxis''s Shorty tread family for muddy and loose conditions.' } + - { product_id: ^110, sku: MAXXIS-AGGRESSOR, text: "Maxxis Aggressor Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 89, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","rear","dry-trail"]', description: 'A mountain bike tire in Maxxis''s Aggressor tread family, often selected for the rear on dry, rocky trails.' } + - { product_id: ^111, sku: MAXXIS-ARDENT, text: "Maxxis Ardent Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 75, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","cross-country","trail"]', description: 'A mountain bike tire in Maxxis''s Ardent tread family for cross-country and trail riding.' } + - { product_id: ^112, sku: MAXXIS-ARDENT-RACE, text: "Maxxis Ardent Race Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 79, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","cross-country","race"]', description: 'A mountain bike tire in Maxxis''s Ardent Race tread family for cross-country riding.' } + - { product_id: ^113, sku: MAXXIS-IKON, text: "Maxxis Ikon Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 79, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","cross-country"]', description: 'A mountain bike tire in Maxxis''s Ikon tread family for cross-country riding.' } + - { product_id: ^114, sku: MAXXIS-ASPEN, text: "Maxxis Aspen Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 89, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","cross-country","race"]', description: 'A mountain bike tire in Maxxis''s Aspen tread family for cross-country riding and racing.' } + - { product_id: ^115, sku: MAXXIS-ASPEN-ST, text: "Maxxis Aspen ST Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 99, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","cross-country","race"]', description: 'A low-profile mountain bike tire in Maxxis''s Aspen ST family for short-track cross-country racing.' } + - { product_id: ^116, sku: MAXXIS-ASPEN-AT, text: "Maxxis Aspen AT Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 99, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","cross-country","all-terrain"]', description: 'A mountain bike tire in Maxxis''s Aspen AT family for cross-country riding across varied terrain.' } + - { product_id: ^117, sku: MAXXIS-CROSSMARK-II, text: "Maxxis Crossmark II Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 69, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","cross-country","fast-rolling"]', description: 'A mountain bike tire in Maxxis''s Crossmark II tread family with a fast-rolling center ridge for dry conditions.' } + - { product_id: ^118, sku: MAXXIS-MINION-SS, text: "Maxxis Minion SS Mountain Bike Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 89, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","semi-slick","fast-rolling"]', description: 'A semi-slick mountain bike tire in Maxxis''s Minion SS family for fast-rolling conditions.' } + - { product_id: ^119, sku: MAXXIS-MINION-FBF, text: "Maxxis Minion FBF Fat Bike Front Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 109, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","fat-bike","front"]', description: 'A front tire in Maxxis''s Minion FBF fat-bike tread family.' } + - { product_id: ^120, sku: MAXXIS-MINION-FBR, text: "Maxxis Minion FBR Fat Bike Rear Tire", sub_category_code: TY, unit_of_measure_code: EA, price: 109, brand_code: MAXXIS, tags_json: '["mountain-bike","tire","fat-bike","rear"]', description: 'A rear tire in Maxxis''s Minion FBR fat-bike tread family.' } + - { product_id: ^121, sku: ONEUP-V3-DROPPER-120, text: "OneUp Dropper Post V3 120mm", sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP, tags_json: '["mountain-bike","dropper","seatpost"]', description: 'A mountain bike dropper seatpost with 120 mm of travel from OneUp Components'' V3 series.' } + - { product_id: ^122, sku: ONEUP-V3-DROPPER-210, text: "OneUp Dropper Post V3 210mm", sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP, tags_json: '["mountain-bike","dropper","seatpost"]', description: 'A mountain bike dropper seatpost with 210 mm of travel from OneUp Components'' V3 series.' } + - { product_id: ^123, sku: ONEUP-V3-DROPPER-240, text: "OneUp Dropper Post V3 240mm", sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP, tags_json: '["mountain-bike","dropper","seatpost"]', description: 'A mountain bike dropper seatpost with 240 mm of travel from OneUp Components'' V3 series.' } + - { product_id: ^124, sku: ONEUP-V2-CARBON-HANDLEBAR, text: "OneUp V2 Carbon Handlebar", sub_category_code: HB, unit_of_measure_code: EA, price: 159, brand_code: ONEUP, tags_json: '["mountain-bike","cockpit","handlebar"]', description: 'A carbon mountain bike handlebar from OneUp Components'' V2 series.' } + - { product_id: ^125, sku: ONEUP-ALUMINUM-HANDLEBAR, text: "OneUp Aluminum Handlebar", sub_category_code: HB, unit_of_measure_code: EA, price: 89, brand_code: ONEUP, tags_json: '["mountain-bike","cockpit","handlebar"]', description: 'An aluminum handlebar for mountain bikes from OneUp Components.' } + - { product_id: ^126, sku: ONEUP-ALUMINUM-PEDAL, text: "OneUp Aluminum Pedals", sub_category_code: PE, unit_of_measure_code: PR, price: 149, brand_code: ONEUP, tags_json: '["mountain-bike","pedals","flat-pedal"]', description: 'Flat platform pedals for mountain bikes from OneUp Components.' } + - { product_id: ^127, sku: ONEUP-EDC-V2-TOOL, text: "OneUp EDC Tool V2", sub_category_code: TO, unit_of_measure_code: EA, price: 59, brand_code: ONEUP, tags_json: '["mountain-bike","tool","repair"]', description: 'A compact trailside multi-tool from OneUp Components'' EDC V2 family.' } + - { product_id: ^128, sku: ONEUP-EDC-LITE-TOOL, text: "OneUp EDC Lite Tool", sub_category_code: TO, unit_of_measure_code: EA, price: 39, brand_code: ONEUP, tags_json: '["mountain-bike","tool","repair"]', description: 'A compact multi-tool in OneUp Components'' EDC Lite tool family for mountain bike repairs.' } + - { product_id: ^129, sku: ONEUP-EDC-PUMP-100, text: "OneUp EDC Pump 100cc", sub_category_code: TO, unit_of_measure_code: EA, price: 59, brand_code: ONEUP, tags_json: '["mountain-bike","pump","repair"]', description: 'A compact hand pump in OneUp Components'' EDC pump family for carrying on mountain bike rides.' } + - { product_id: ^130, sku: ONEUP-EDC-PUMP-70, text: "OneUp EDC Pump 70cc", sub_category_code: TO, unit_of_measure_code: EA, price: 49, brand_code: ONEUP, tags_json: '["mountain-bike","pump","repair"]', description: 'A compact hand pump in OneUp Components'' EDC pump family for carrying on mountain bike rides.' } + - { product_id: ^131, sku: ONEUP-EDC-TOP-CAP, text: "OneUp EDC Top Cap", sub_category_code: TO, unit_of_measure_code: EA, price: 29, brand_code: ONEUP, tags_json: '["mountain-bike","tool-storage"]', description: 'A top cap in OneUp Components'' EDC system for mountain bike tool storage.' } + - { product_id: ^132, sku: ONEUP-CHAIN-GUIDE, text: "OneUp Chain Guide", sub_category_code: CG, unit_of_measure_code: EA, price: 79, brand_code: ONEUP, tags_json: '["mountain-bike","drivetrain","chain-guide"]', description: 'A chain guide for mountain bikes from OneUp Components.' } + - { product_id: ^133, sku: ONEUP-BASH-GUIDE, text: "OneUp Bash Guide", sub_category_code: BG, unit_of_measure_code: EA, price: 89, brand_code: ONEUP, tags_json: '["mountain-bike","drivetrain","bash-guide"]', description: 'A bash guide for mountain bikes from OneUp Components.' } + - { product_id: ^134, sku: ONEUP-V3-REMOTE, text: "OneUp V3 Dropper Remote", sub_category_code: SP, unit_of_measure_code: EA, price: 59, brand_code: ONEUP, tags_json: '["mountain-bike","dropper","remote"]', description: 'A handlebar remote control in OneUp Components'' V3 dropper post system.' } + - { product_id: ^135, sku: ONEUP-THICK-GRIP, text: "OneUp Thick Lock-On Grips", sub_category_code: HB, unit_of_measure_code: PR, price: 29, brand_code: ONEUP, tags_json: '["mountain-bike","cockpit","grips"]', description: 'Lock-on grips for mountain bike handlebars from OneUp Components.' } + - { product_id: ^136, sku: ONEUP-DIRECT-MOUNT-STEM, text: "OneUp Direct Mount Stem", sub_category_code: HB, unit_of_measure_code: EA, price: 109, brand_code: ONEUP, tags_json: '["mountain-bike","cockpit","stem"]', description: 'A direct-mount stem for mountain bike handlebars from OneUp Components.' } + - { product_id: ^137, sku: ONEUP-CARBON-EBAR, text: "OneUp Carbon E-Bar", sub_category_code: HB, unit_of_measure_code: EA, price: 169, brand_code: ONEUP, tags_json: '["mountain-bike","cockpit","handlebar","electric"]', description: 'A carbon handlebar from OneUp Components'' E-Bar series for electric mountain bikes.' } + - { product_id: ^138, sku: FOXRACING-UNION-FLAT-SHOE, text: "Fox Racing Union Flat Mountain Bike Shoe", sub_category_code: SH, unit_of_measure_code: PR, price: 179, brand_code: FOXRACING, tags_json: '["mountain-bike","shoe","flat-pedal"]', description: 'A flat-pedal mountain bike shoe in Fox Racing''s Union footwear family.' } + - { product_id: ^139, sku: FOXRACING-UNION-BOA-FLAT, text: "Fox Racing Union BOA Flat Mountain Bike Shoe", sub_category_code: SH, unit_of_measure_code: PR, price: 249, brand_code: FOXRACING, tags_json: '["mountain-bike","shoe","flat-pedal"]', description: 'A flat-pedal mountain bike shoe with BOA closure in Fox Racing''s Union footwear family.' } + - { product_id: ^140, sku: FOXRACING-UNION-CLIPLESS, text: "Fox Racing Union Clipless Mountain Bike Shoe", sub_category_code: SH, unit_of_measure_code: PR, price: 229, brand_code: FOXRACING, tags_json: '["mountain-bike","shoe","clipless"]', description: 'A clipless mountain bike shoe in Fox Racing''s Union footwear family.' } + - { product_id: ^141, sku: FOXRACING-UNION-BOA-CLIPLESS, text: "Fox Racing Union BOA Clipless Mountain Bike Shoe", sub_category_code: SH, unit_of_measure_code: PR, price: 279, brand_code: FOXRACING, tags_json: '["mountain-bike","shoe","clipless"]', description: 'A clipless mountain bike shoe with BOA closure in Fox Racing''s Union footwear family.' } + - { product_id: ^142, sku: FOXRACING-RANGER-SS-JERSEY, text: "Fox Racing Ranger TruDri Short Sleeve Jersey", sub_category_code: AP, unit_of_measure_code: EA, price: 69, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","jersey"]', description: 'A short-sleeve mountain bike jersey from Fox Racing''s Ranger TruDri apparel family.' } + - { product_id: ^143, sku: FOXRACING-RANGER-LS-JERSEY, text: "Fox Racing Ranger Long Sleeve Jersey", sub_category_code: AP, unit_of_measure_code: EA, price: 79, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","jersey"]', description: 'A long-sleeve mountain bike jersey from Fox Racing''s Ranger apparel family.' } + - { product_id: ^144, sku: FOXRACING-FLEXAIR-PRO-JERSEY, text: "Fox Racing Flexair Pro Jersey", sub_category_code: AP, unit_of_measure_code: EA, price: 99, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","jersey"]', description: 'A mountain bike jersey from Fox Racing''s Flexair Pro apparel family.' } + - { product_id: ^145, sku: FOXRACING-FLEXAIR-SS-JERSEY, text: "Fox Racing Flexair Short Sleeve Jersey", sub_category_code: AP, unit_of_measure_code: EA, price: 89, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","jersey"]', description: 'A short-sleeve mountain bike jersey from Fox Racing''s Flexair apparel family.' } + - { product_id: ^146, sku: FOXRACING-DEFEND-LS-JERSEY, text: "Fox Racing Defend Long Sleeve Jersey", sub_category_code: AP, unit_of_measure_code: EA, price: 89, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","jersey"]', description: 'A long-sleeve mountain bike jersey from Fox Racing''s Defend apparel family.' } + - { product_id: ^147, sku: FOXRACING-RANGER-PANTS, text: "Fox Racing Ranger Mountain Bike Pants", sub_category_code: AP, unit_of_measure_code: EA, price: 119, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","pants"]', description: 'Mountain bike riding pants in Fox Racing''s Ranger apparel family.' } + - { product_id: ^148, sku: FOXRACING-FLEXAIR-PANTS, text: "Fox Racing Flexair Mountain Bike Pants", sub_category_code: AP, unit_of_measure_code: EA, price: 169, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","pants"]', description: 'Mountain bike riding pants in Fox Racing''s Flexair apparel family.' } + - { product_id: ^149, sku: FOXRACING-DEFEND-PANTS, text: "Fox Racing Defend Mountain Bike Pants", sub_category_code: AP, unit_of_measure_code: EA, price: 159, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","pants"]', description: 'Mountain bike riding pants in Fox Racing''s Defend apparel family.' } + - { product_id: ^150, sku: FOXRACING-RANGER-SHORTS, text: "Fox Racing Ranger Mountain Bike Shorts", sub_category_code: AP, unit_of_measure_code: EA, price: 89, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","shorts"]', description: 'Mountain bike riding shorts in Fox Racing''s Ranger apparel family.' } + - { product_id: ^151, sku: FOXRACING-FLEXAIR-SHORTS, text: "Fox Racing Flexair Mountain Bike Shorts", sub_category_code: AP, unit_of_measure_code: EA, price: 129, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","shorts"]', description: 'Mountain bike riding shorts in Fox Racing''s Flexair apparel family.' } + - { product_id: ^152, sku: FOXRACING-DEFEND-SHORTS, text: "Fox Racing Defend Mountain Bike Shorts", sub_category_code: AP, unit_of_measure_code: EA, price: 139, brand_code: FOXRACING, tags_json: '["mountain-bike","apparel","shorts"]', description: 'Mountain bike riding shorts in Fox Racing''s Defend apparel family.' } + - { product_id: ^153, sku: FOXRACING-RANGER-GLOVES, text: "Fox Racing Ranger Mountain Bike Gloves", sub_category_code: GL, unit_of_measure_code: PR, price: 39, brand_code: FOXRACING, tags_json: '["mountain-bike","gloves"]', description: 'Mountain bike riding gloves from Fox Racing''s Ranger gear family.' } + - { product_id: ^154, sku: FOXRACING-RANGER-GEL-GLOVES, text: "Fox Racing Ranger Gel Mountain Bike Gloves", sub_category_code: GL, unit_of_measure_code: PR, price: 49, brand_code: FOXRACING, tags_json: '["mountain-bike","gloves"]', description: 'Mountain bike riding gloves in Fox Racing''s Ranger Gel family.' } + - { product_id: ^155, sku: FOXRACING-FLEXAIR-GLOVES, text: "Fox Racing Flexair Mountain Bike Gloves", sub_category_code: GL, unit_of_measure_code: PR, price: 45, brand_code: FOXRACING, tags_json: '["mountain-bike","gloves"]', description: 'Mountain bike riding gloves from Fox Racing''s Flexair gear family.' } + - { product_id: ^156, sku: FOXRACING-DEFEND-GLOVES, text: "Fox Racing Defend Mountain Bike Gloves", sub_category_code: GL, unit_of_measure_code: PR, price: 49, brand_code: FOXRACING, tags_json: '["mountain-bike","gloves"]', description: 'Mountain bike riding gloves from Fox Racing''s Defend gear family.' } + - { product_id: ^157, sku: FOXRACING-SPEEDFRAME-HELMET, text: "Fox Racing Speedframe Mountain Bike Helmet", sub_category_code: HE, unit_of_measure_code: EA, price: 179, brand_code: FOXRACING, tags_json: '["mountain-bike","helmet"]', description: 'An open-face mountain bike helmet in Fox Racing''s Speedframe family.' } + - { product_id: ^158, sku: FOXRACING-SPEEDFRAME-PRO, text: "Fox Racing Speedframe Pro Mountain Bike Helmet", sub_category_code: HE, unit_of_measure_code: EA, price: 279, brand_code: FOXRACING, tags_json: '["mountain-bike","helmet"]', description: 'An open-face mountain bike helmet in Fox Racing''s Speedframe Pro family.' } + - { product_id: ^159, sku: FOXRACING-DROPFRAME-PRO, text: "Fox Racing Dropframe Pro Mountain Bike Helmet", sub_category_code: HE, unit_of_measure_code: EA, price: 299, brand_code: FOXRACING, tags_json: '["mountain-bike","helmet"]', description: 'A mountain bike helmet in Fox Racing''s Dropframe Pro family.' } + - { product_id: ^160, sku: FOXRACING-PROFRAME-RS, text: "Fox Racing Proframe RS Mountain Bike Helmet", sub_category_code: HE, unit_of_measure_code: EA, price: 449, brand_code: FOXRACING, tags_json: '["mountain-bike","helmet","full-face"]', description: 'A full-face mountain bike helmet in Fox Racing''s Proframe RS family.' } + - { product_id: ^161, sku: FOXRACING-LAUNCH-D3O-KNEE, text: "Fox Racing Launch D3O Knee Guard", sub_category_code: PR, unit_of_measure_code: PR, price: 99, brand_code: FOXRACING, tags_json: '["mountain-bike","protection","knee-guard"]', description: 'A knee guard in Fox Racing''s Launch D3O mountain bike protection family.' } + - { product_id: ^162, sku: FOXRACING-LAUNCH-PRO-D3O-ELBOW, text: "Fox Racing Launch Pro D3O Elbow Guard", sub_category_code: PR, unit_of_measure_code: PR, price: 89, brand_code: FOXRACING, tags_json: '["mountain-bike","protection","elbow-guard"]', description: 'An elbow guard in Fox Racing''s Launch Pro D3O mountain bike protection range.' } + - { product_id: ^163, sku: MUCOFF-DRY-CHAIN-LUBE, text: "Muc-Off Dry Chain Lube", sub_category_code: LU, unit_of_measure_code: EA, price: 14, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","chain-lube","dry"]', description: 'A bicycle chain lubricant from Muc-Off for dry riding conditions.' } + - { product_id: ^164, sku: MUCOFF-WET-CHAIN-LUBE, text: "Muc-Off Wet Chain Lube", sub_category_code: LU, unit_of_measure_code: EA, price: 14, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","chain-lube","wet"]', description: 'A bicycle chain lubricant from Muc-Off for wet riding conditions.' } + - { product_id: ^165, sku: MUCOFF-C3-CERAMIC-DRY-LUBE, text: "Muc-Off C3 Ceramic Dry Chain Lube", sub_category_code: LU, unit_of_measure_code: EA, price: 28, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","chain-lube","dry"]', description: 'A ceramic bicycle chain lubricant from Muc-Off for dry conditions.' } + - { product_id: ^166, sku: MUCOFF-C3-CERAMIC-WET-LUBE, text: "Muc-Off C3 Ceramic Wet Chain Lube", sub_category_code: LU, unit_of_measure_code: EA, price: 28, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","chain-lube","wet"]', description: 'A ceramic bicycle chain lubricant from Muc-Off for wet conditions.' } + - { product_id: ^167, sku: MUCOFF-BIO-DRIVETRAIN-CLEANER, text: "Muc-Off Bio Drivetrain Cleaner", sub_category_code: CL, unit_of_measure_code: EA, price: 22, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","cleaner","drivetrain"]', description: 'A bicycle drivetrain cleaner from Muc-Off for cleaning grease and grime from drivetrain parts.' } + - { product_id: ^168, sku: MUCOFF-DISC-BRAKE-CLEANER, text: "Muc-Off Disc Brake Cleaner", sub_category_code: CL, unit_of_measure_code: EA, price: 16, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","cleaner","brakes"]', description: 'A cleaner from Muc-Off formulated for bicycle disc brake components.' } + - { product_id: ^169, sku: MUCOFF-BIKE-PROTECT, text: "Muc-Off Bike Protect", sub_category_code: MS, unit_of_measure_code: EA, price: 22, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","protection-spray"]', description: 'A post-wash protection spray for bicycles from Muc-Off.' } + - { product_id: ^170, sku: MUCOFF-MO94, text: "Muc-Off MO-94 Maintenance Spray", sub_category_code: MS, unit_of_measure_code: EA, price: 12, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","spray"]', description: 'A multi-purpose maintenance spray from Muc-Off for bicycle care.' } + - { product_id: ^171, sku: MUCOFF-PREMIUM-BRUSH-SET, text: "Muc-Off Premium Brush Set", sub_category_code: MS, unit_of_measure_code: EA, price: 35, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","brush"]', description: 'A bicycle cleaning brush set from Muc-Off for washing frames and components.' } + - { product_id: ^172, sku: MUCOFF-CLAW-BRUSH, text: "Muc-Off Claw Brush", sub_category_code: MS, unit_of_measure_code: EA, price: 12, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","brush","drivetrain"]', description: 'A bicycle cleaning brush from Muc-Off for drivetrain maintenance.' } + - { product_id: ^173, sku: MUCOFF-NANO-TECH-CONCENTRATE, text: "Muc-Off Nano Tech Bike Cleaner Concentrate", sub_category_code: CL, unit_of_measure_code: EA, price: 29, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","cleaner"]', description: 'A concentrated bicycle cleaner from Muc-Off for preparing bike wash solution.' } + - { product_id: ^174, sku: MUCOFF-TUBELESS-VALVE-KIT, text: "Muc-Off Tubeless Valve Kit", sub_category_code: MS, unit_of_measure_code: EA, price: 34, brand_code: MUCOFF, tags_json: '["mountain-bike","maintenance","tubeless"]', description: 'A tubeless valve kit from Muc-Off for bicycle wheel setups.' } + - { product_id: ^32, sku: LABOR, text: Labor, sub_category_code: LA, unit_of_measure_code: HR, price: 80.00, is_non_stocked: true, brand_code: NONE, tags_json: '["bike-service","bicycle-repair","workshop-labour","workshop-labor","bike-mechanic","maintenance","wrench-monkey","grease-monkey","bike-whisperer","spanner-man"]', description: 'Professional workshop labour charged by the hour for mountain-bike maintenance and repair. This is a non-stocked service rather than a physical product; no specific operations or parts are promised by this listing.' } + - inventory: + - { inventory_id: ^1, qty_on_hand: 3 } + - { inventory_id: ^16, qty_on_hand: 1 } + - { inventory_id: ^24, qty_on_hand: 3 } + - { inventory_id: ^28, qty_on_hand: 5 } + - movement: + - { movement_id: ^1, product_id: ^1, quantity: -2, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } + - { movement_id: ^2, product_id: ^6, quantity: -1, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } + - { movement_id: ^3, product_id: ^31, quantity: -3, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } + - { movement_id: ^4, product_id: ^24, quantity: -2, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } + - { movement_id: ^5, product_id: ^28, quantity: -1, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } diff --git a/samples/data/demo-catalog.sources.json b/samples/data/demo-catalog.sources.json new file mode 100644 index 000000000..58d1c6d24 --- /dev/null +++ b/samples/data/demo-catalog.sources.json @@ -0,0 +1,2399 @@ +{ + "catalogVersion": "product-search-mtb-expansion-v2", + "currency": "USD", + "skuPolicy": "Catalog SKUs are internal demo identifiers, not verified manufacturer part numbers; no DEMO- prefix is required. Demo in a genuine model name, such as Specialized Demo, is retained.", + "pricePolicy": "All prices are illustrative demo values in USD, not current retail prices.", + "sourcePolicy": "Manufacturer pages support all 142 expansion entries. The retained v1 catalog subset preserves two previously documented non-manufacturer references, explicitly marked retailer; BikesOnline is approved as a supplementary retailer source for additional research.", + "products": [ + { + "sku": "YETI-ARC", + "brandCode": "YETI", + "subCategoryCode": "HT", + "unitCode": "EA", + "price": 3200, + "text": "Yeti ARC Mountain Bike", + "description": "A hardtail mountain bike in Yeti Cycles' ARC model family for cross-country and trail riding.", + "sourceUrl": "https://yeticycles.com/bikes/arc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail", + "cross-country", + "dentist-bike" + ], + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "YETI-SB135", + "brandCode": "YETI", + "subCategoryCode": "TR", + "unitCode": "EA", + "price": 6400, + "text": "Yeti SB135 Mountain Bike", + "description": "A full-suspension trail mountain bike in Yeti Cycles' SB135 model family.", + "sourceUrl": "https://yeticycles.com/bikes/sb135", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "trail", + "dentist-bike" + ], + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "YETI-SB160", + "brandCode": "YETI", + "subCategoryCode": "EN", + "unitCode": "EA", + "price": 7900, + "text": "Yeti SB160 Mountain Bike", + "description": "A full-suspension enduro mountain bike in Yeti Cycles' SB160 model family.", + "sourceUrl": "https://yeticycles.com/bikes/sb160", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "enduro", + "dentist-bike" + ], + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "CANYON-STOIC", + "brandCode": "CANYON", + "subCategoryCode": "HT", + "unitCode": "EA", + "price": 1899, + "text": "Canyon Stoic Mountain Bike", + "description": "A hardtail mountain bike in Canyon's Stoic model family for trail riding.", + "sourceUrl": "https://www.canyon.com/en-us/mountain-bikes/trail-bikes/stoic/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail", + "trail" + ] + }, + { + "sku": "CANYON-STRIVE", + "brandCode": "CANYON", + "subCategoryCode": "EN", + "unitCode": "EA", + "price": 5799, + "text": "Canyon Strive Mountain Bike", + "description": "A full-suspension enduro mountain bike in Canyon's Strive model family.", + "sourceUrl": "https://www.canyon.com/en-us/mountain-bikes/enduro-bikes/strive/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "enduro" + ] + }, + { + "sku": "CANYON-GRAND-CANYON", + "brandCode": "CANYON", + "subCategoryCode": "HT", + "unitCode": "EA", + "price": 1499, + "text": "Canyon Grand Canyon Mountain Bike", + "description": "A hardtail mountain bike in Canyon's Grand Canyon model family for off-road riding.", + "sourceUrl": "https://www.canyon.com/en-us/mountain-bikes/hardtail-mountain-bikes/grand-canyon/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail" + ] + }, + { + "sku": "CANYON-LUX-WORLD-CUP", + "brandCode": "CANYON", + "subCategoryCode": "XC", + "unitCode": "EA", + "price": 6299, + "text": "Canyon Lux World Cup Mountain Bike", + "description": "A full-suspension cross-country race mountain bike in Canyon's Lux World Cup family.", + "sourceUrl": "https://media-centre.canyon.com/en-INT/264438-no-limits-on-the-limit-canyon-launches-new-lux-world-cup-cfr/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cross-country", + "race" + ] + }, + { + "sku": "SPECIALIZED-FUSE", + "brandCode": "SPEC", + "subCategoryCode": "HT", + "unitCode": "EA", + "price": 2799, + "text": "Specialized Fuse Mountain Bike", + "description": "A hardtail mountain bike in Specialized's Fuse model family for trail riding.", + "sourceUrl": "https://www.specialized.com/us/en/shop/bikes/mountain-bikes/trail-mountain-bikes/fuse", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail", + "trail" + ] + }, + { + "sku": "SPECIALIZED-DEMO", + "brandCode": "SPEC", + "subCategoryCode": "DH", + "unitCode": "EA", + "price": 6499, + "text": "Specialized Demo Mountain Bike", + "description": "A downhill mountain bike in Specialized's Demo model family.", + "sourceUrl": "https://www.specialized.com/us/en/shop/bikes/mountain-bikes/downhill-mountain-bikes/demo", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "downhill" + ] + }, + { + "sku": "SPECIALIZED-TURBO-LEVO", + "brandCode": "SPEC", + "subCategoryCode": "EM", + "unitCode": "EA", + "price": 8999, + "text": "Specialized Turbo Levo E-Mountain Bike", + "description": "An electric-assist mountain bike in Specialized's Turbo Levo model family.", + "sourceUrl": "https://www.specialized.com/us/en/turbo-levo/c/turbo-levo", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "electric", + "e-bike", + "emtb" + ], + "tagProvenance": [ + { + "tag": "e-bike", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + }, + { + "tag": "emtb", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + } + ] + }, + { + "sku": "SPECIALIZED-TURBO-KENEVO", + "brandCode": "SPEC", + "subCategoryCode": "EM", + "unitCode": "EA", + "price": 9999, + "text": "Specialized Turbo Kenevo E-Mountain Bike", + "description": "An electric-assist mountain bike in Specialized's Turbo Kenevo model family.", + "sourceUrl": "https://www.specialized.com/us/en/shop/bikes/electric-bikes/electric-mountain-bikes/turbo-kenevo", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "electric", + "enduro", + "e-bike", + "emtb" + ], + "tagProvenance": [ + { + "tag": "e-bike", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + }, + { + "tag": "emtb", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + } + ] + }, + { + "sku": "SPECIALIZED-EPIC-WORLD-CUP", + "brandCode": "SPEC", + "subCategoryCode": "XC", + "unitCode": "EA", + "price": 8999, + "text": "Specialized Epic World Cup Mountain Bike", + "description": "A cross-country race mountain bike in Specialized's Epic World Cup family.", + "sourceUrl": "https://www.specialized.com/us/en/s-works-epic-world-cup-sram-xx-sl-axs-rockshox-ultimate/p/4221515", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cross-country", + "race" + ] + }, + { + "sku": "GIANT-TRANCE", + "brandCode": "GIANT", + "subCategoryCode": "TR", + "unitCode": "EA", + "price": 4599, + "text": "Giant Trance Mountain Bike", + "description": "A full-suspension trail mountain bike in Giant's Trance model family.", + "sourceUrl": "https://www.giant-bicycles.com/us/trance", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "trail" + ] + }, + { + "sku": "GIANT-REIGN", + "brandCode": "GIANT", + "subCategoryCode": "EN", + "unitCode": "EA", + "price": 5499, + "text": "Giant Reign Mountain Bike", + "description": "A full-suspension enduro mountain bike in Giant's Reign model family.", + "sourceUrl": "https://www.giant-bicycles.com/us/reign", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "enduro" + ] + }, + { + "sku": "GIANT-FATHOM", + "brandCode": "GIANT", + "subCategoryCode": "HT", + "unitCode": "EA", + "price": 1699, + "text": "Giant Fathom Mountain Bike", + "description": "A hardtail mountain bike in Giant's Fathom model family for trail riding.", + "sourceUrl": "https://www.giant-bicycles.com/us/fathom", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail", + "trail" + ] + }, + { + "sku": "GIANT-GLORY", + "brandCode": "GIANT", + "subCategoryCode": "DH", + "unitCode": "EA", + "price": 7199, + "text": "Giant Glory Mountain Bike", + "description": "A downhill mountain bike in Giant's Glory model family.", + "sourceUrl": "https://www.giant-bicycles.com/us/glory", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "downhill" + ] + }, + { + "sku": "GIANT-STANCE", + "brandCode": "GIANT", + "subCategoryCode": "TR", + "unitCode": "EA", + "price": 2499, + "text": "Giant Stance Mountain Bike", + "description": "A full-suspension trail mountain bike in Giant's Stance model family.", + "sourceUrl": "https://www.giant-bicycles.com/us/stance", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "trail" + ] + }, + { + "sku": "SPECIALIZED-STATUS-2", + "brandCode": "SPEC", + "subCategoryCode": "FR", + "unitCode": "EA", + "price": 3299, + "text": "Specialized Status 2 Mountain Bike", + "description": "A mixed-wheel mountain bike in Specialized's Status model family for trail and freeride use.", + "sourceUrl": "https://www.specialized.com/us/en/status-140/p/199765", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "freeride" + ] + }, + { + "sku": "GIANT-ANTHEM", + "brandCode": "GIANT", + "subCategoryCode": "XC", + "unitCode": "EA", + "price": 6399, + "text": "Giant Anthem Mountain Bike", + "description": "A full-suspension cross-country mountain bike in Giant's Anthem model family.", + "sourceUrl": "https://www.giant-bicycles.com/global/anthem", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cross-country" + ] + }, + { + "sku": "SHIMANO-SLX-M7100-DERAILLEUR", + "brandCode": "SHIMANO", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 89, + "text": "Shimano SLX M7100 Rear Derailleur", + "description": "A rear derailleur from Shimano's SLX M7100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur" + ] + }, + { + "sku": "SHIMANO-SLX-M7100-SHIFTER", + "brandCode": "SHIMANO", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 49, + "text": "Shimano SLX M7100 Shifter", + "description": "A mountain bike gear shifter from Shimano's SLX M7100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "shifter" + ] + }, + { + "sku": "SHIMANO-SLX-M7100-CHAIN", + "brandCode": "SHIMANO", + "subCategoryCode": "CH", + "unitCode": "EA", + "price": 39, + "text": "Shimano SLX M7100 Chain", + "description": "A bicycle chain from Shimano's SLX M7100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ] + }, + { + "sku": "SHIMANO-SLX-M7100-CRANKSET", + "brandCode": "SHIMANO", + "subCategoryCode": "CR", + "unitCode": "EA", + "price": 149, + "text": "Shimano SLX M7100 Crankset", + "description": "A mountain bike crankset from Shimano's SLX M7100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ] + }, + { + "sku": "SHIMANO-XT-M8100-SHIFTER", + "brandCode": "SHIMANO", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 69, + "text": "Shimano Deore XT M8100 Shifter", + "description": "A mountain bike gear shifter from Shimano's Deore XT M8100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "shifter" + ] + }, + { + "sku": "SHIMANO-XT-M8100-CHAIN", + "brandCode": "SHIMANO", + "subCategoryCode": "CH", + "unitCode": "EA", + "price": 54, + "text": "Shimano Deore XT M8100 Chain", + "description": "A bicycle chain from Shimano's Deore XT M8100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ] + }, + { + "sku": "SHIMANO-XT-M8100-CRANKSET", + "brandCode": "SHIMANO", + "subCategoryCode": "CR", + "unitCode": "EA", + "price": 199, + "text": "Shimano Deore XT M8100 Crankset", + "description": "A mountain bike crankset from Shimano's Deore XT M8100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ] + }, + { + "sku": "SHIMANO-XT-M8100-CASSETTE", + "brandCode": "SHIMANO", + "subCategoryCode": "CA", + "unitCode": "EA", + "price": 149, + "text": "Shimano Deore XT M8100 Cassette", + "description": "A mountain bike cassette from Shimano's Deore XT M8100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ] + }, + { + "sku": "SHIMANO-XT-M8120-BRAKE", + "brandCode": "SHIMANO", + "subCategoryCode": "BR", + "unitCode": "EA", + "price": 219, + "text": "Shimano Deore XT M8120 Disc Brake", + "description": "A hydraulic disc brake from Shimano's Deore XT M8120 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ] + }, + { + "sku": "SHIMANO-XT-M8100-BRAKE-LEVER", + "brandCode": "SHIMANO", + "subCategoryCode": "BR", + "unitCode": "EA", + "price": 89, + "text": "Shimano Deore XT M8100 Brake Lever", + "description": "A hydraulic brake lever from Shimano's Deore XT M8100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes", + "lever" + ] + }, + { + "sku": "SHIMANO-XTR-M9100-DERAILLEUR", + "brandCode": "SHIMANO", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 259, + "text": "Shimano XTR M9100 Rear Derailleur", + "description": "A rear derailleur from Shimano's XTR M9100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur" + ] + }, + { + "sku": "SHIMANO-XTR-M9100-SHIFTER", + "brandCode": "SHIMANO", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 129, + "text": "Shimano XTR M9100 Shifter", + "description": "A mountain bike gear shifter from Shimano's XTR M9100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "shifter" + ] + }, + { + "sku": "SHIMANO-XTR-M9100-CHAIN", + "brandCode": "SHIMANO", + "subCategoryCode": "CH", + "unitCode": "EA", + "price": 79, + "text": "Shimano XTR M9100 Chain", + "description": "A bicycle chain from Shimano's XTR M9100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ] + }, + { + "sku": "SHIMANO-XTR-M9100-CASSETTE", + "brandCode": "SHIMANO", + "subCategoryCode": "CA", + "unitCode": "EA", + "price": 399, + "text": "Shimano XTR M9100 Cassette", + "description": "A mountain bike cassette from Shimano's XTR M9100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ] + }, + { + "sku": "SHIMANO-XTR-M9100-CRANKSET", + "brandCode": "SHIMANO", + "subCategoryCode": "CR", + "unitCode": "EA", + "price": 499, + "text": "Shimano XTR M9100 Crankset", + "description": "A mountain bike crankset from Shimano's XTR M9100 component series.", + "sourceUrl": "https://bike.shimano.com/products/components/pdp.P-FC-M9100-1.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ] + }, + { + "sku": "SHIMANO-XTR-M9100-BRAKE", + "brandCode": "SHIMANO", + "subCategoryCode": "BR", + "unitCode": "EA", + "price": 399, + "text": "Shimano XTR M9100 Disc Brake", + "description": "A hydraulic disc brake from Shimano's XTR M9100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ] + }, + { + "sku": "SHIMANO-DEORE-M6100-DERAILLEUR", + "brandCode": "SHIMANO", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 64, + "text": "Shimano Deore M6100 Rear Derailleur", + "description": "A rear derailleur from Shimano's Deore M6100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur" + ] + }, + { + "sku": "SHIMANO-DEORE-M6100-CASSETTE", + "brandCode": "SHIMANO", + "subCategoryCode": "CA", + "unitCode": "EA", + "price": 119, + "text": "Shimano Deore M6100 Cassette", + "description": "A mountain bike cassette from Shimano's Deore M6100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ] + }, + { + "sku": "SRAM-GX-EAGLE-CASSETTE", + "brandCode": "SRAM", + "subCategoryCode": "CA", + "unitCode": "EA", + "price": 235, + "text": "SRAM GX Eagle Cassette", + "description": "A mountain bike cassette from SRAM's GX Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle/gx-eagle?page=1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ] + }, + { + "sku": "SRAM-X01-EAGLE-CASSETTE", + "brandCode": "SRAM", + "subCategoryCode": "CA", + "unitCode": "EA", + "price": 349, + "text": "SRAM X01 Eagle Cassette", + "description": "A mountain bike cassette from SRAM's X01 Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/x01-eagle", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ] + }, + { + "sku": "SRAM-XX1-EAGLE-CHAIN", + "brandCode": "SRAM", + "subCategoryCode": "CH", + "unitCode": "EA", + "price": 89, + "text": "SRAM XX1 Eagle Chain", + "description": "A bicycle chain from SRAM's XX1 Eagle mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/models/cn-eagle-xx1-a1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ] + }, + { + "sku": "SRAM-GX-EAGLE-CHAIN", + "brandCode": "SRAM", + "subCategoryCode": "CH", + "unitCode": "EA", + "price": 42, + "text": "SRAM GX Eagle Chain", + "description": "A bicycle chain from SRAM's GX Eagle mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle/gx-eagle?page=1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ] + }, + { + "sku": "SRAM-GX-EAGLE-CRANKSET", + "brandCode": "SRAM", + "subCategoryCode": "CR", + "unitCode": "EA", + "price": 159, + "text": "SRAM GX Eagle Crankset", + "description": "A mountain bike crankset from SRAM's GX Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle/gx-eagle?page=1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ] + }, + { + "sku": "SRAM-GX-EAGLE-SHIFTER", + "brandCode": "SRAM", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 49, + "text": "SRAM GX Eagle Trigger Shifter", + "description": "A mountain bike gear shifter from SRAM's GX Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle/gx-eagle?page=1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "shifter" + ] + }, + { + "sku": "SRAM-X01-EAGLE-AXS-DERAILLEUR", + "brandCode": "SRAM", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 499, + "text": "SRAM X01 Eagle AXS Rear Derailleur", + "description": "A wireless electronic rear derailleur from SRAM's X01 Eagle AXS mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/models/rd-x0-1-e-a1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur", + "electronic" + ] + }, + { + "sku": "SRAM-XX1-EAGLE-AXS-DERAILLEUR", + "brandCode": "SRAM", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 699, + "text": "SRAM XX1 Eagle AXS Rear Derailleur", + "description": "A wireless electronic rear derailleur from SRAM's XX1 Eagle AXS mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/models/rd-xx-1-e-a1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur", + "electronic" + ] + }, + { + "sku": "SRAM-GX-TRANSMISSION-DERAILLEUR", + "brandCode": "SRAM", + "subCategoryCode": "DR", + "unitCode": "EA", + "price": 599, + "text": "SRAM GX Eagle Transmission Rear Derailleur", + "description": "A rear derailleur from SRAM's GX Eagle Transmission mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle-transmission/gx-eagle-transmission", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur", + "electronic" + ] + }, + { + "sku": "SRAM-CODE-RSC-BRAKE", + "brandCode": "SRAM", + "subCategoryCode": "BR", + "unitCode": "EA", + "price": 349, + "text": "SRAM Code RSC Disc Brake", + "description": "A hydraulic disc brake from SRAM's Code mountain bike brake family.", + "sourceUrl": "https://www.sram.com/en/sram/models/db-codr-d1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ] + }, + { + "sku": "SRAM-MAVEN-ULTIMATE-BRAKE", + "brandCode": "SRAM", + "subCategoryCode": "BR", + "unitCode": "EA", + "price": 699, + "text": "SRAM Maven Ultimate Disc Brake", + "description": "A hydraulic disc brake from SRAM's Maven mountain bike brake family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/maven/maven", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ] + }, + { + "sku": "SRAM-MAVEN-SILVER-BRAKE", + "brandCode": "SRAM", + "subCategoryCode": "BR", + "unitCode": "EA", + "price": 499, + "text": "SRAM Maven Silver Disc Brake", + "description": "A hydraulic disc brake from SRAM's Maven mountain bike brake family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/maven/maven", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ] + }, + { + "sku": "SRAM-LEVEL-ULTIMATE-BRAKE", + "brandCode": "SRAM", + "subCategoryCode": "BR", + "unitCode": "EA", + "price": 399, + "text": "SRAM Level Ultimate Disc Brake", + "description": "A hydraulic disc brake from SRAM's Level mountain bike brake family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/level/level", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ] + }, + { + "sku": "SRAM-XX-TRANSMISSION-CASSETTE", + "brandCode": "SRAM", + "subCategoryCode": "CA", + "unitCode": "EA", + "price": 699, + "text": "SRAM XX Eagle Transmission Cassette", + "description": "A cassette from SRAM's XX Eagle Transmission mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle-transmission/gx-eagle-transmission", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ] + }, + { + "sku": "SRAM-XX-TRANSMISSION-CHAIN", + "brandCode": "SRAM", + "subCategoryCode": "CH", + "unitCode": "EA", + "price": 89, + "text": "SRAM XX Eagle Transmission Chain", + "description": "A chain from SRAM's XX Eagle Transmission mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle-transmission/gx-eagle-transmission", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ] + }, + { + "sku": "SRAM-X01-EAGLE-CRANKSET", + "brandCode": "SRAM", + "subCategoryCode": "CR", + "unitCode": "EA", + "price": 329, + "text": "SRAM X01 Eagle Crankset", + "description": "A mountain bike crankset from SRAM's X01 Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/x01-eagle", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ] + }, + { + "sku": "FOX-32-SC-FACTORY", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 1099, + "text": "FOX 32 Step-Cast Factory Fork", + "description": "A mountain bike suspension fork in FOX's 32 Step-Cast Factory series for cross-country riding.", + "sourceUrl": "https://www.ridefox.com/family.php?m=bike&family=32", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "cross-country" + ] + }, + { + "sku": "FOX-34-FACTORY", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 1199, + "text": "FOX 34 Factory Fork", + "description": "A mountain bike suspension fork in FOX's 34 Factory series for trail and downcountry riding.", + "sourceUrl": "https://ridefox.com/collections/fox-34", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "trail" + ] + }, + { + "sku": "FOX-34-FACTORY-GRIP-X", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 1199, + "text": "FOX 34 Factory GRIP X Fork", + "description": "A mountain bike suspension fork in FOX's 34 Factory GRIP X series for trail riding.", + "sourceUrl": "https://ridefox.com/collections/fox-34", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "trail" + ] + }, + { + "sku": "FOX-34-PERFORMANCE-ELITE", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 899, + "text": "FOX 34 Performance Elite Fork", + "description": "A mountain bike suspension fork in FOX's 34 Performance Elite series.", + "sourceUrl": "https://ridefox.com/collections/fox-34", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork" + ] + }, + { + "sku": "FOX-36-SL-FACTORY", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 1199, + "text": "FOX 36 SL Factory GRIP X Fork", + "description": "A mountain bike suspension fork in FOX's 36 SL Factory series for trail and downcountry riding.", + "sourceUrl": "https://ridefox.com/products/my27-fox-36-sl-factory-gripx", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "trail" + ] + }, + { + "sku": "FOX-36-FACTORY-GRIP-X", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 1299, + "text": "FOX 36 Factory GRIP X Fork", + "description": "A mountain bike suspension fork in FOX's 36 Factory GRIP X series for trail riding.", + "sourceUrl": "https://ridefox.com/products/my27-fox-36-factory-gripx", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "trail" + ] + }, + { + "sku": "FOX-36-FACTORY-GRIP-X2", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 1399, + "text": "FOX 36 Factory GRIP X2 Fork", + "description": "A mountain bike suspension fork in FOX's 36 Factory GRIP X2 series for all-mountain riding.", + "sourceUrl": "https://ridefox.com/products/my27-fox-36-factory-gripx2", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "all-mountain" + ] + }, + { + "sku": "FOX-36-PERFORMANCE-ELITE", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 999, + "text": "FOX 36 Performance Elite GRIP X Fork", + "description": "A mountain bike suspension fork in FOX's 36 Performance Elite series.", + "sourceUrl": "https://ridefox.com/products/my27-fox-36-pse-gripx-fcb6ecad", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork" + ] + }, + { + "sku": "FOX-38-FACTORY", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 1399, + "text": "FOX 38 Factory Fork", + "description": "A mountain bike suspension fork in FOX's 38 Factory series for enduro riding.", + "sourceUrl": "https://ridefox.com/products/my27-fox-38-factory", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "enduro" + ] + }, + { + "sku": "FOX-38-PERFORMANCE-ELITE", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 1199, + "text": "FOX 38 Performance Elite Fork", + "description": "A mountain bike suspension fork in FOX's 38 Performance Elite series.", + "sourceUrl": "https://ridefox.com/products/my27-fox-38-pse-d795211b", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "enduro" + ] + }, + { + "sku": "FOX-40-FACTORY", + "brandCode": "FOX", + "subCategoryCode": "FS", + "unitCode": "EA", + "price": 1899, + "text": "FOX 40 Factory Downhill Fork", + "description": "A dual-crown suspension fork in FOX's 40 Factory downhill series.", + "sourceUrl": "https://ridefox.com/products/fox-40-factory-grip-x2-dh-fork-for-downhill-riding", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "downhill" + ] + }, + { + "sku": "FOX-FLOAT-DPS-FACTORY", + "brandCode": "FOX", + "subCategoryCode": "RS", + "unitCode": "EA", + "price": 599, + "text": "FOX Float DPS Factory Rear Shock", + "description": "An air-sprung rear shock in FOX's Float DPS Factory mountain bike suspension series.", + "sourceUrl": "https://www.ridefox.com/family.php?m=bike&family=floatdps", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "rear-shock" + ] + }, + { + "sku": "FOX-FLOAT-X-FACTORY", + "brandCode": "FOX", + "subCategoryCode": "RS", + "unitCode": "EA", + "price": 699, + "text": "FOX Float X Factory Rear Shock", + "description": "An air-sprung rear shock in FOX's Float X Factory mountain bike suspension series.", + "sourceUrl": "https://ridefox.com/pages/fox-float-x", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "rear-shock" + ] + }, + { + "sku": "FOX-FLOAT-X2-FACTORY", + "brandCode": "FOX", + "subCategoryCode": "RS", + "unitCode": "EA", + "price": 799, + "text": "FOX Float X2 Factory Rear Shock", + "description": "An air-sprung rear shock in FOX's Float X2 Factory mountain bike suspension series.", + "sourceUrl": "https://www.ridefox.com/family.php?m=bike&family=floatx2", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "rear-shock" + ] + }, + { + "sku": "FOX-DHX-FACTORY", + "brandCode": "FOX", + "subCategoryCode": "RS", + "unitCode": "EA", + "price": 699, + "text": "FOX DHX Factory Coil Shock", + "description": "A coil-sprung rear shock in FOX's DHX Factory mountain bike suspension series.", + "sourceUrl": "https://ca.ridefox.com/products/fox-dhx-factory", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "rear-shock", + "coil" + ] + }, + { + "sku": "MAXXIS-ASSEGAI", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 99, + "text": "Maxxis Assegai Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Assegai tread family for enduro and downhill riding.", + "sourceUrl": "https://shop.maxxis.com/products/assegai", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "enduro", + "downhill" + ] + }, + { + "sku": "MAXXIS-MINION-DHF", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 95, + "text": "Maxxis Minion DHF Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Minion DHF tread family, commonly used at the front for trail, enduro, and downhill riding.", + "sourceUrl": "https://shop.maxxis.com/products/minion-dhf-1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "front", + "enduro" + ] + }, + { + "sku": "MAXXIS-MINION-DHR-II", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 95, + "text": "Maxxis Minion DHR II Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Minion DHR II tread family with a braking-oriented tread pattern.", + "sourceUrl": "https://shop.maxxis.com/products/minion-dhr-ii", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "rear", + "braking" + ] + }, + { + "sku": "MAXXIS-DISSECTOR", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 89, + "text": "Maxxis Dissector Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Dissector tread family for trail and enduro riding.", + "sourceUrl": "https://shop.maxxis.com/products/dissector", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "trail", + "enduro" + ] + }, + { + "sku": "MAXXIS-REKON", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 79, + "text": "Maxxis Rekon Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Rekon tread family for cross-country and trail riding.", + "sourceUrl": "https://shop.maxxis.com/products/rekon", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "trail" + ] + }, + { + "sku": "MAXXIS-REKON-RACE", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 79, + "text": "Maxxis Rekon Race Mountain Bike Tire", + "description": "A low-profile mountain bike tire in Maxxis's Rekon Race family for cross-country racing.", + "sourceUrl": "https://www.maxxis.com/us/tire/rekon-race/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "race" + ] + }, + { + "sku": "MAXXIS-FOREKASTER", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 84, + "text": "Maxxis Forekaster Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Forekaster tread family for trail riding in mixed conditions.", + "sourceUrl": "https://www.maxxis.com/us/tire/forekaster/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "trail", + "mixed-conditions" + ] + }, + { + "sku": "MAXXIS-HIGH-ROLLER-II", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 89, + "text": "Maxxis High Roller II Mountain Bike Tire", + "description": "An aggressive mountain bike tire in Maxxis's High Roller II tread family for loose and varied terrain.", + "sourceUrl": "https://www.maxxis.com/us/tire/high-roller/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "loose-terrain" + ] + }, + { + "sku": "MAXXIS-SHORTY", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 99, + "text": "Maxxis Shorty Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Shorty tread family for muddy and loose conditions.", + "sourceUrl": "https://www.maxxis.com/us/tire/shorty/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "mud", + "downhill" + ] + }, + { + "sku": "MAXXIS-AGGRESSOR", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 89, + "text": "Maxxis Aggressor Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Aggressor tread family, often selected for the rear on dry, rocky trails.", + "sourceUrl": "https://www.maxxis.com/us/tire/aggressor/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "rear", + "dry-trail" + ] + }, + { + "sku": "MAXXIS-ARDENT", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 75, + "text": "Maxxis Ardent Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Ardent tread family for cross-country and trail riding.", + "sourceUrl": "https://www.maxxis.com/us/tire/ardent/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "trail" + ] + }, + { + "sku": "MAXXIS-ARDENT-RACE", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 79, + "text": "Maxxis Ardent Race Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Ardent Race tread family for cross-country riding.", + "sourceUrl": "https://www.maxxis.com/us/tire/ardent-race/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "race" + ] + }, + { + "sku": "MAXXIS-IKON", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 79, + "text": "Maxxis Ikon Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Ikon tread family for cross-country riding.", + "sourceUrl": "https://www.maxxis.com/us/tire/ikon/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country" + ] + }, + { + "sku": "MAXXIS-ASPEN", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 89, + "text": "Maxxis Aspen Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Aspen tread family for cross-country riding and racing.", + "sourceUrl": "https://shop.maxxis.com/products/aspen", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "race" + ] + }, + { + "sku": "MAXXIS-ASPEN-ST", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 99, + "text": "Maxxis Aspen ST Mountain Bike Tire", + "description": "A low-profile mountain bike tire in Maxxis's Aspen ST family for short-track cross-country racing.", + "sourceUrl": "https://www.maxxis.com/us/tire/aspen-st/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "race" + ] + }, + { + "sku": "MAXXIS-ASPEN-AT", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 99, + "text": "Maxxis Aspen AT Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Aspen AT family for cross-country riding across varied terrain.", + "sourceUrl": "https://www.maxxis.com/us/tire/aspen-at/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "all-terrain" + ] + }, + { + "sku": "MAXXIS-CROSSMARK-II", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 69, + "text": "Maxxis Crossmark II Mountain Bike Tire", + "description": "A mountain bike tire in Maxxis's Crossmark II tread family with a fast-rolling center ridge for dry conditions.", + "sourceUrl": "https://www.maxxis.com/us/tire/crossmark-ii/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "fast-rolling" + ] + }, + { + "sku": "MAXXIS-MINION-SS", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 89, + "text": "Maxxis Minion SS Mountain Bike Tire", + "description": "A semi-slick mountain bike tire in Maxxis's Minion SS family for fast-rolling conditions.", + "sourceUrl": "https://www.maxxis.com/us/tire/minion-ss/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "semi-slick", + "fast-rolling" + ] + }, + { + "sku": "MAXXIS-MINION-FBF", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 109, + "text": "Maxxis Minion FBF Fat Bike Front Tire", + "description": "A front tire in Maxxis's Minion FBF fat-bike tread family.", + "sourceUrl": "https://www.maxxis.com/us/tire/minion-fbf/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "fat-bike", + "front" + ] + }, + { + "sku": "MAXXIS-MINION-FBR", + "brandCode": "MAXXIS", + "subCategoryCode": "TY", + "unitCode": "EA", + "price": 109, + "text": "Maxxis Minion FBR Fat Bike Rear Tire", + "description": "A rear tire in Maxxis's Minion FBR fat-bike tread family.", + "sourceUrl": "https://www.maxxis.com/us/tire/minion-fbr/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "fat-bike", + "rear" + ] + }, + { + "sku": "ONEUP-V3-DROPPER-120", + "brandCode": "ONEUP", + "subCategoryCode": "SP", + "unitCode": "EA", + "price": 269, + "text": "OneUp Dropper Post V3 120mm", + "description": "A mountain bike dropper seatpost with 120 mm of travel from OneUp Components' V3 series.", + "sourceUrl": "https://www.oneupcomponents.com/products/dropper-post-v3", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "dropper", + "seatpost" + ] + }, + { + "sku": "ONEUP-V3-DROPPER-210", + "brandCode": "ONEUP", + "subCategoryCode": "SP", + "unitCode": "EA", + "price": 269, + "text": "OneUp Dropper Post V3 210mm", + "description": "A mountain bike dropper seatpost with 210 mm of travel from OneUp Components' V3 series.", + "sourceUrl": "https://www.oneupcomponents.com/products/dropper-post-v3", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "dropper", + "seatpost" + ] + }, + { + "sku": "ONEUP-V3-DROPPER-240", + "brandCode": "ONEUP", + "subCategoryCode": "SP", + "unitCode": "EA", + "price": 269, + "text": "OneUp Dropper Post V3 240mm", + "description": "A mountain bike dropper seatpost with 240 mm of travel from OneUp Components' V3 series.", + "sourceUrl": "https://www.oneupcomponents.com/products/dropper-post-v3", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "dropper", + "seatpost" + ] + }, + { + "sku": "ONEUP-V2-CARBON-HANDLEBAR", + "brandCode": "ONEUP", + "subCategoryCode": "HB", + "unitCode": "EA", + "price": 159, + "text": "OneUp V2 Carbon Handlebar", + "description": "A carbon mountain bike handlebar from OneUp Components' V2 series.", + "sourceUrl": "https://www.oneupcomponents.com/products/v2-carbon-handlebar", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "handlebar" + ] + }, + { + "sku": "ONEUP-ALUMINUM-HANDLEBAR", + "brandCode": "ONEUP", + "subCategoryCode": "HB", + "unitCode": "EA", + "price": 89, + "text": "OneUp Aluminum Handlebar", + "description": "An aluminum handlebar for mountain bikes from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/handlebars", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "handlebar" + ] + }, + { + "sku": "ONEUP-ALUMINUM-PEDAL", + "brandCode": "ONEUP", + "subCategoryCode": "PE", + "unitCode": "PR", + "price": 149, + "text": "OneUp Aluminum Pedals", + "description": "Flat platform pedals for mountain bikes from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/pedals", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "pedals", + "flat-pedal" + ] + }, + { + "sku": "ONEUP-EDC-V2-TOOL", + "brandCode": "ONEUP", + "subCategoryCode": "TO", + "unitCode": "EA", + "price": 59, + "text": "OneUp EDC Tool V2", + "description": "A compact trailside multi-tool from OneUp Components' EDC V2 family.", + "sourceUrl": "https://www.oneupcomponents.com/products/edc-v2-tool", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tool", + "repair" + ] + }, + { + "sku": "ONEUP-EDC-LITE-TOOL", + "brandCode": "ONEUP", + "subCategoryCode": "TO", + "unitCode": "EA", + "price": 39, + "text": "OneUp EDC Lite Tool", + "description": "A compact multi-tool in OneUp Components' EDC Lite tool family for mountain bike repairs.", + "sourceUrl": "https://www.oneupcomponents.com/collections/edc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tool", + "repair" + ] + }, + { + "sku": "ONEUP-EDC-PUMP-100", + "brandCode": "ONEUP", + "subCategoryCode": "TO", + "unitCode": "EA", + "price": 59, + "text": "OneUp EDC Pump 100cc", + "description": "A compact hand pump in OneUp Components' EDC pump family for carrying on mountain bike rides.", + "sourceUrl": "https://www.oneupcomponents.com/collections/edc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "pump", + "repair" + ] + }, + { + "sku": "ONEUP-EDC-PUMP-70", + "brandCode": "ONEUP", + "subCategoryCode": "TO", + "unitCode": "EA", + "price": 49, + "text": "OneUp EDC Pump 70cc", + "description": "A compact hand pump in OneUp Components' EDC pump family for carrying on mountain bike rides.", + "sourceUrl": "https://www.oneupcomponents.com/collections/edc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "pump", + "repair" + ] + }, + { + "sku": "ONEUP-EDC-TOP-CAP", + "brandCode": "ONEUP", + "subCategoryCode": "TO", + "unitCode": "EA", + "price": 29, + "text": "OneUp EDC Top Cap", + "description": "A top cap in OneUp Components' EDC system for mountain bike tool storage.", + "sourceUrl": "https://www.oneupcomponents.com/collections/edc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tool-storage" + ] + }, + { + "sku": "ONEUP-CHAIN-GUIDE", + "brandCode": "ONEUP", + "subCategoryCode": "CG", + "unitCode": "EA", + "price": 79, + "text": "OneUp Chain Guide", + "description": "A chain guide for mountain bikes from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/components", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain-guide" + ] + }, + { + "sku": "ONEUP-BASH-GUIDE", + "brandCode": "ONEUP", + "subCategoryCode": "BG", + "unitCode": "EA", + "price": 89, + "text": "OneUp Bash Guide", + "description": "A bash guide for mountain bikes from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/components", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "bash-guide" + ] + }, + { + "sku": "ONEUP-V3-REMOTE", + "brandCode": "ONEUP", + "subCategoryCode": "SP", + "unitCode": "EA", + "price": 59, + "text": "OneUp V3 Dropper Remote", + "description": "A handlebar remote control in OneUp Components' V3 dropper post system.", + "sourceUrl": "https://www.oneupcomponents.com/products/dropper-post-v3", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "dropper", + "remote" + ] + }, + { + "sku": "ONEUP-THICK-GRIP", + "brandCode": "ONEUP", + "subCategoryCode": "HB", + "unitCode": "PR", + "price": 29, + "text": "OneUp Thick Lock-On Grips", + "description": "Lock-on grips for mountain bike handlebars from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/grips", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "grips" + ] + }, + { + "sku": "ONEUP-DIRECT-MOUNT-STEM", + "brandCode": "ONEUP", + "subCategoryCode": "HB", + "unitCode": "EA", + "price": 109, + "text": "OneUp Direct Mount Stem", + "description": "A direct-mount stem for mountain bike handlebars from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/components", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "stem" + ] + }, + { + "sku": "ONEUP-CARBON-EBAR", + "brandCode": "ONEUP", + "subCategoryCode": "HB", + "unitCode": "EA", + "price": 169, + "text": "OneUp Carbon E-Bar", + "description": "A carbon handlebar from OneUp Components' E-Bar series for electric mountain bikes.", + "sourceUrl": "https://www.oneupcomponents.com/collections/handlebars", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "handlebar", + "electric" + ] + }, + { + "sku": "FOXRACING-UNION-FLAT-SHOE", + "brandCode": "FOXRACING", + "subCategoryCode": "SH", + "unitCode": "PR", + "price": 179, + "text": "Fox Racing Union Flat Mountain Bike Shoe", + "description": "A flat-pedal mountain bike shoe in Fox Racing's Union footwear family.", + "sourceUrl": "https://www.foxracing.com/product/fox-union-flat-shoes/29354.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "shoe", + "flat-pedal" + ] + }, + { + "sku": "FOXRACING-UNION-BOA-FLAT", + "brandCode": "FOXRACING", + "subCategoryCode": "SH", + "unitCode": "PR", + "price": 249, + "text": "Fox Racing Union BOA Flat Mountain Bike Shoe", + "description": "A flat-pedal mountain bike shoe with BOA closure in Fox Racing's Union footwear family.", + "sourceUrl": "https://www.foxracing.com/products/union-boa-flat-shoes/32347.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "shoe", + "flat-pedal" + ] + }, + { + "sku": "FOXRACING-UNION-CLIPLESS", + "brandCode": "FOXRACING", + "subCategoryCode": "SH", + "unitCode": "PR", + "price": 229, + "text": "Fox Racing Union Clipless Mountain Bike Shoe", + "description": "A clipless mountain bike shoe in Fox Racing's Union footwear family.", + "sourceUrl": "https://www.foxracing.com/products/union-clipless-shoes/31948.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "shoe", + "clipless" + ] + }, + { + "sku": "FOXRACING-UNION-BOA-CLIPLESS", + "brandCode": "FOXRACING", + "subCategoryCode": "SH", + "unitCode": "PR", + "price": 279, + "text": "Fox Racing Union BOA Clipless Mountain Bike Shoe", + "description": "A clipless mountain bike shoe with BOA closure in Fox Racing's Union footwear family.", + "sourceUrl": "https://www.foxracing.com/product/fox-union-boa-clipless-shoes/29353.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "shoe", + "clipless" + ] + }, + { + "sku": "FOXRACING-RANGER-SS-JERSEY", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 69, + "text": "Fox Racing Ranger TruDri Short Sleeve Jersey", + "description": "A short-sleeve mountain bike jersey from Fox Racing's Ranger TruDri apparel family.", + "sourceUrl": "https://www.foxracing.com/product/ranger-trudri-short-sleeve-jersey/28874.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ] + }, + { + "sku": "FOXRACING-RANGER-LS-JERSEY", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 79, + "text": "Fox Racing Ranger Long Sleeve Jersey", + "description": "A long-sleeve mountain bike jersey from Fox Racing's Ranger apparel family.", + "sourceUrl": "https://www.foxracing.com/products/ranger-long-sleeve-jersey/27362.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ] + }, + { + "sku": "FOXRACING-FLEXAIR-PRO-JERSEY", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 99, + "text": "Fox Racing Flexair Pro Jersey", + "description": "A mountain bike jersey from Fox Racing's Flexair Pro apparel family.", + "sourceUrl": "https://www.foxracing.com/product/flexair-pro-jersey/31296.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ] + }, + { + "sku": "FOXRACING-FLEXAIR-SS-JERSEY", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 89, + "text": "Fox Racing Flexair Short Sleeve Jersey", + "description": "A short-sleeve mountain bike jersey from Fox Racing's Flexair apparel family.", + "sourceUrl": "https://www.foxracing.com/products/flexair-short-sleeve-jersey/28863.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ] + }, + { + "sku": "FOXRACING-DEFEND-LS-JERSEY", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 89, + "text": "Fox Racing Defend Long Sleeve Jersey", + "description": "A long-sleeve mountain bike jersey from Fox Racing's Defend apparel family.", + "sourceUrl": "https://www.foxracing.com/product/defend-long-sleeve-jersey/28864.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ] + }, + { + "sku": "FOXRACING-RANGER-PANTS", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 119, + "text": "Fox Racing Ranger Mountain Bike Pants", + "description": "Mountain bike riding pants in Fox Racing's Ranger apparel family.", + "sourceUrl": "https://www.foxracing.com/product/ranger-pants/28891.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "pants" + ] + }, + { + "sku": "FOXRACING-FLEXAIR-PANTS", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 169, + "text": "Fox Racing Flexair Mountain Bike Pants", + "description": "Mountain bike riding pants in Fox Racing's Flexair apparel family.", + "sourceUrl": "https://www.foxracing.com/product/flexair-pant/28886.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "pants" + ] + }, + { + "sku": "FOXRACING-DEFEND-PANTS", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 159, + "text": "Fox Racing Defend Mountain Bike Pants", + "description": "Mountain bike riding pants in Fox Racing's Defend apparel family.", + "sourceUrl": "https://www.foxracing.com/product/defend-pant/25929.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "pants" + ] + }, + { + "sku": "FOXRACING-RANGER-SHORTS", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 89, + "text": "Fox Racing Ranger Mountain Bike Shorts", + "description": "Mountain bike riding shorts in Fox Racing's Ranger apparel family.", + "sourceUrl": "https://www.foxracing.com/products/ranger-shorts/28882.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "shorts" + ] + }, + { + "sku": "FOXRACING-FLEXAIR-SHORTS", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 129, + "text": "Fox Racing Flexair Mountain Bike Shorts", + "description": "Mountain bike riding shorts in Fox Racing's Flexair apparel family.", + "sourceUrl": "https://www.foxracing.com/products/flexair-shorts/28881.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "shorts" + ] + }, + { + "sku": "FOXRACING-DEFEND-SHORTS", + "brandCode": "FOXRACING", + "subCategoryCode": "AP", + "unitCode": "EA", + "price": 139, + "text": "Fox Racing Defend Mountain Bike Shorts", + "description": "Mountain bike riding shorts in Fox Racing's Defend apparel family.", + "sourceUrl": "https://www.foxracing.com/product/defend-shorts/28881.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "shorts" + ] + }, + { + "sku": "FOXRACING-RANGER-GLOVES", + "brandCode": "FOXRACING", + "subCategoryCode": "GL", + "unitCode": "PR", + "price": 39, + "text": "Fox Racing Ranger Mountain Bike Gloves", + "description": "Mountain bike riding gloves from Fox Racing's Ranger gear family.", + "sourceUrl": "https://www.foxracing.com/product/ranger-gloves/27162.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "gloves" + ] + }, + { + "sku": "FOXRACING-RANGER-GEL-GLOVES", + "brandCode": "FOXRACING", + "subCategoryCode": "GL", + "unitCode": "PR", + "price": 49, + "text": "Fox Racing Ranger Gel Mountain Bike Gloves", + "description": "Mountain bike riding gloves in Fox Racing's Ranger Gel family.", + "sourceUrl": "https://www.foxracing.com/products/ranger-gel-gloves/33607.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "gloves" + ] + }, + { + "sku": "FOXRACING-FLEXAIR-GLOVES", + "brandCode": "FOXRACING", + "subCategoryCode": "GL", + "unitCode": "PR", + "price": 45, + "text": "Fox Racing Flexair Mountain Bike Gloves", + "description": "Mountain bike riding gloves from Fox Racing's Flexair gear family.", + "sourceUrl": "https://www.foxracing.com/products/flexair-gloves/31308.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "gloves" + ] + }, + { + "sku": "FOXRACING-DEFEND-GLOVES", + "brandCode": "FOXRACING", + "subCategoryCode": "GL", + "unitCode": "PR", + "price": 49, + "text": "Fox Racing Defend Mountain Bike Gloves", + "description": "Mountain bike riding gloves from Fox Racing's Defend gear family.", + "sourceUrl": "https://www.foxracing.com/products/defend-gloves/33792.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "gloves" + ] + }, + { + "sku": "FOXRACING-SPEEDFRAME-HELMET", + "brandCode": "FOXRACING", + "subCategoryCode": "HE", + "unitCode": "EA", + "price": 179, + "text": "Fox Racing Speedframe Mountain Bike Helmet", + "description": "An open-face mountain bike helmet in Fox Racing's Speedframe family.", + "sourceUrl": "https://www.foxracing.com/product/speedframe-helmet/26712.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "helmet" + ] + }, + { + "sku": "FOXRACING-SPEEDFRAME-PRO", + "brandCode": "FOXRACING", + "subCategoryCode": "HE", + "unitCode": "EA", + "price": 279, + "text": "Fox Racing Speedframe Pro Mountain Bike Helmet", + "description": "An open-face mountain bike helmet in Fox Racing's Speedframe Pro family.", + "sourceUrl": "https://www.foxracing.com/product/speedframe-pro-helmet/25102.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "helmet" + ] + }, + { + "sku": "FOXRACING-DROPFRAME-PRO", + "brandCode": "FOXRACING", + "subCategoryCode": "HE", + "unitCode": "EA", + "price": 299, + "text": "Fox Racing Dropframe Pro Mountain Bike Helmet", + "description": "A mountain bike helmet in Fox Racing's Dropframe Pro family.", + "sourceUrl": "https://www.foxracing.com/products/dropframe-pro-helmet/31459.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "helmet" + ] + }, + { + "sku": "FOXRACING-PROFRAME-RS", + "brandCode": "FOXRACING", + "subCategoryCode": "HE", + "unitCode": "EA", + "price": 449, + "text": "Fox Racing Proframe RS Mountain Bike Helmet", + "description": "A full-face mountain bike helmet in Fox Racing's Proframe RS family.", + "sourceUrl": "https://www.foxracing.com/products/proframe-rs-helmet/32497.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "helmet", + "full-face" + ] + }, + { + "sku": "FOXRACING-LAUNCH-D3O-KNEE", + "brandCode": "FOXRACING", + "subCategoryCode": "PR", + "unitCode": "PR", + "price": 99, + "text": "Fox Racing Launch D3O Knee Guard", + "description": "A knee guard in Fox Racing's Launch D3O mountain bike protection family.", + "sourceUrl": "https://www.foxracing.com/product/launch-d3o-knee-guard/26430.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "protection", + "knee-guard" + ] + }, + { + "sku": "FOXRACING-LAUNCH-PRO-D3O-ELBOW", + "brandCode": "FOXRACING", + "subCategoryCode": "PR", + "unitCode": "PR", + "price": 89, + "text": "Fox Racing Launch Pro D3O Elbow Guard", + "description": "An elbow guard in Fox Racing's Launch Pro D3O mountain bike protection range.", + "sourceUrl": "https://www.foxracing.com/products/launch-pro-d3o-elbow-guards/18495.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "protection", + "elbow-guard" + ] + }, + { + "sku": "MUCOFF-DRY-CHAIN-LUBE", + "brandCode": "MUCOFF", + "subCategoryCode": "LU", + "unitCode": "EA", + "price": 14, + "text": "Muc-Off Dry Chain Lube", + "description": "A bicycle chain lubricant from Muc-Off for dry riding conditions.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "chain-lube", + "dry" + ] + }, + { + "sku": "MUCOFF-WET-CHAIN-LUBE", + "brandCode": "MUCOFF", + "subCategoryCode": "LU", + "unitCode": "EA", + "price": 14, + "text": "Muc-Off Wet Chain Lube", + "description": "A bicycle chain lubricant from Muc-Off for wet riding conditions.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "chain-lube", + "wet" + ] + }, + { + "sku": "MUCOFF-C3-CERAMIC-DRY-LUBE", + "brandCode": "MUCOFF", + "subCategoryCode": "LU", + "unitCode": "EA", + "price": 28, + "text": "Muc-Off C3 Ceramic Dry Chain Lube", + "description": "A ceramic bicycle chain lubricant from Muc-Off for dry conditions.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "chain-lube", + "dry" + ] + }, + { + "sku": "MUCOFF-C3-CERAMIC-WET-LUBE", + "brandCode": "MUCOFF", + "subCategoryCode": "LU", + "unitCode": "EA", + "price": 28, + "text": "Muc-Off C3 Ceramic Wet Chain Lube", + "description": "A ceramic bicycle chain lubricant from Muc-Off for wet conditions.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "chain-lube", + "wet" + ] + }, + { + "sku": "MUCOFF-BIO-DRIVETRAIN-CLEANER", + "brandCode": "MUCOFF", + "subCategoryCode": "CL", + "unitCode": "EA", + "price": 22, + "text": "Muc-Off Bio Drivetrain Cleaner", + "description": "A bicycle drivetrain cleaner from Muc-Off for cleaning grease and grime from drivetrain parts.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "cleaner", + "drivetrain" + ] + }, + { + "sku": "MUCOFF-DISC-BRAKE-CLEANER", + "brandCode": "MUCOFF", + "subCategoryCode": "CL", + "unitCode": "EA", + "price": 16, + "text": "Muc-Off Disc Brake Cleaner", + "description": "A cleaner from Muc-Off formulated for bicycle disc brake components.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "cleaner", + "brakes" + ] + }, + { + "sku": "MUCOFF-BIKE-PROTECT", + "brandCode": "MUCOFF", + "subCategoryCode": "MS", + "unitCode": "EA", + "price": 22, + "text": "Muc-Off Bike Protect", + "description": "A post-wash protection spray for bicycles from Muc-Off.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "protection-spray" + ] + }, + { + "sku": "MUCOFF-MO94", + "brandCode": "MUCOFF", + "subCategoryCode": "MS", + "unitCode": "EA", + "price": 12, + "text": "Muc-Off MO-94 Maintenance Spray", + "description": "A multi-purpose maintenance spray from Muc-Off for bicycle care.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "spray" + ] + }, + { + "sku": "MUCOFF-PREMIUM-BRUSH-SET", + "brandCode": "MUCOFF", + "subCategoryCode": "MS", + "unitCode": "EA", + "price": 35, + "text": "Muc-Off Premium Brush Set", + "description": "A bicycle cleaning brush set from Muc-Off for washing frames and components.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "brush" + ] + }, + { + "sku": "MUCOFF-CLAW-BRUSH", + "brandCode": "MUCOFF", + "subCategoryCode": "MS", + "unitCode": "EA", + "price": 12, + "text": "Muc-Off Claw Brush", + "description": "A bicycle cleaning brush from Muc-Off for drivetrain maintenance.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "brush", + "drivetrain" + ] + }, + { + "sku": "MUCOFF-NANO-TECH-CONCENTRATE", + "brandCode": "MUCOFF", + "subCategoryCode": "CL", + "unitCode": "EA", + "price": 29, + "text": "Muc-Off Nano Tech Bike Cleaner Concentrate", + "description": "A concentrated bicycle cleaner from Muc-Off for preparing bike wash solution.", + "sourceUrl": "https://muc-off.com/products/bike-cleaner", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "cleaner" + ] + }, + { + "sku": "MUCOFF-TUBELESS-VALVE-KIT", + "brandCode": "MUCOFF", + "subCategoryCode": "MS", + "unitCode": "EA", + "price": 34, + "text": "Muc-Off Tubeless Valve Kit", + "description": "A tubeless valve kit from Muc-Off for bicycle wheel setups.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "tubeless" + ] + } + ], + "services": [ + { + "sku": "LABOR", + "sourceType": "sample-service-definition", + "sourcePath": "samples/data/demo-catalog.seed.yaml", + "brandCode": "NONE", + "subCategoryCode": "LA", + "unitCode": "HR", + "price": 80, + "text": "Labor", + "description": "Professional workshop labour charged by the hour for mountain-bike maintenance and repair. This is a non-stocked service rather than a physical product; no specific operations or parts are promised by this listing.", + "tags": [ + "bike-service", + "bicycle-repair", + "workshop-labour", + "workshop-labor", + "bike-mechanic", + "maintenance", + "wrench-monkey", + "grease-monkey", + "bike-whisperer", + "spanner-man" + ], + "tagProvenance": [ + { + "tag": "wrench-monkey", + "origin": "user-provided-informal-search-alias", + "note": "Curated service-intent alias, not a manufacturer fact, staff designation or guarantee of specific work." + }, + { + "tag": "grease-monkey", + "origin": "user-provided-informal-search-alias", + "note": "Curated service-intent alias, not a manufacturer fact, staff designation or guarantee of specific work." + }, + { + "tag": "bike-whisperer", + "origin": "user-provided-informal-search-alias", + "note": "Curated service-intent alias, not a manufacturer fact, staff designation or guarantee of specific work." + }, + { + "tag": "spanner-man", + "origin": "user-provided-informal-search-alias", + "note": "Curated service-intent alias, not a manufacturer fact, staff designation or guarantee of specific work." + } + ] + } + ], + "tagPolicy": { + "manufacturerFacts": "For manufacturer/retailer product entries, names, category tags and factual descriptions are supported by the per-product sourceUrl. Services are locally authored sample definitions, not manufacturer claims. Prices are illustrative USD.", + "colloquialAliases": "e-bike and emtb are user-requested colloquial search aliases for verified electric-assist mountain bikes; they are not extra specifications.", + "informalYetiAlias": "dentist-bike is a user-provided informal demo search alias on Yeti mountain-bike listings only. It is not a manufacturer fact, intended-rider claim, or affordability statement.", + "serviceIntentAliases": "LABOR uses curated workshop/maintenance intent terms. wrench-monkey, grease-monkey, bike-whisperer and spanner-man are user-requested informal demo aliases, not manufacturer facts, staff designations, guaranteed operations or included parts." + } +} diff --git a/samples/docs/aspire.md b/samples/docs/aspire.md index b10524cb8..6f3aec118 100644 --- a/samples/docs/aspire.md +++ b/samples/docs/aspire.md @@ -83,7 +83,7 @@ Both commands start the AppHost, which in turn launches all registered projects `Contoso.Mcp` is a local-development sample: it has no authentication and must not be exposed to an untrusted network. It is a cross-domain HTTP facade over the real Products and Shopping APIs. Aspire wires both API references through service discovery; it does not add MCP endpoints to either domain API. The server uses stateless Streamable HTTP at `/mcp` (not stdio). Copy the `contoso-mcp` HTTP endpoint from the Aspire Dashboard and append `/mcp`; the port can vary between runs. -The tools support product search/detail and the complete basket lifecycle: list/create baskets, read a basket, add/update/remove items, apply `SAVE10`, update the shipping address, and checkout. Successful calls return typed structured data with a tool output schema; errors are reported as MCP tool errors, not success-shaped strings. The ordinary development database seed does not include customer or product test data. For an interactive basket demo, start `Contoso.E2E.Runner` and run **Database Migration and Base Data Refresh** followed by **Data Seeding for E2E Testing**; this resets the local sample databases to test data, seeds customer `16.ToGuid()` with an address, and stocks active products. Basket creation requires an existing Customers-domain ID. Checkout is consequential: it reserves stock and triggers asynchronous confirmation through the existing outbox, relay, Service Bus, and Products subscriber. +The tools support product search/detail and the complete basket lifecycle: list/create baskets, read a basket, add/update/remove items, apply `SAVE10`, update the shipping address, and checkout. Successful calls return typed structured data with a tool output schema; errors are reported as MCP tool errors, not success-shaped strings. The ordinary development database seed does not include customer or product test data. For an interactive basket demo, explicitly run the E2E Runner's **Database Migration and Base Data Refresh** followed by **Data Seeding for E2E Testing**. The [shared demo bootstrap](#shared-demo-catalog-bootstrap) loads the expanded catalog, synchronizes Shopping and prepares real-model search; the full setup also seeds Customers (including `16.ToGuid()` with an address). Review the destructive-reset cautions below first. Basket creation requires an existing Customers-domain ID. Checkout is consequential: it reserves stock and triggers asynchronous confirmation through the existing outbox, relay, Service Bus, and Products subscriber. To connect GitHub Copilot CLI while Aspire is running, register that endpoint from a terminal: @@ -155,7 +155,7 @@ The E2E Runner calls the live APIs over real HTTP. No mocks. Cross-domain flows ## E2E Runner -`Contoso.E2E.Runner` is an interactive console application (`samples/tests/Contoso.E2E.Runner`) that drives cross-domain scenarios against the running APIs. It serves two purposes: +`Contoso.E2E.Runner` is a single-target .NET 10 interactive console application (`samples/tests/Contoso.E2E.Runner`) that drives cross-domain scenarios against the running APIs. It stays with the sample tests, but explicitly clears inherited multi-targeting like the Aspire tests, avoiding older-framework dependency-version conflicts. It serves two purposes: 1. **Functional validation** — verify that the end-to-end flows work correctly against real infrastructure, catching issues that intra-domain tests cannot (network timeouts, Service Bus lag, cross-domain state consistency). 2. **Load simulation** — run all scenarios in parallel workers to exercise the system under realistic concurrency, surfacing performance issues and race conditions. @@ -172,7 +172,7 @@ On startup, the runner checks `/health/ready` on all APIs and displays their sta ### Configuration -Default endpoints and simulation parameters are defined in `appsettings.json`. Override with environment variables using `__` as the separator: +Default endpoints and simulation parameters are defined in the runner's `appsettings.json` (copied as `e2e-settings.json`); Products embedding defaults are linked from its API's development settings as `products-settings.json`. Both files are resolved relative to the executable, not the current directory. Override with environment variables using `__` as the separator: ```bash E2E__Products__BaseAddress=https://localhost:7200 @@ -197,7 +197,7 @@ Per-scenario simulation parallelism and delay bounds are also configurable: | Scenario | What it does | |---|---| -| Database Migration and Base Data Refresh | Runs DbEx migrations for all domain databases and resets base reference data. | +| Database Migration and Base Data Refresh | Resets/migrates Products and Shopping, loads the shared demo catalog, syncs Shopping through its adapter before baskets, and rebuilds/verifies both search projections. Also resets/provisions Customers and Orders using their existing domain setup. | | Data Seeding for E2E Testing | Calls `POST /api/inventory/adjust` to set on-hand quantity to 1 000 units for every active, stocked product. Required before basket scenarios to ensure stock is available. | #### Functional scenarios (repeatable) @@ -211,6 +211,72 @@ Per-scenario simulation parallelism and delay bounds are also configurable: The Shopping Basket Lifecycle is the richest scenario — it exercises every inter-domain integration path in a single run and is the primary validation tool for the full system. +### Shared demo catalog bootstrap + +[`samples/data/demo-catalog.seed.yaml`](../data/demo-catalog.seed.yaml) is the single catalog source for +interactive demo/MCP, `DatabaseMigrationSetup`, and search relevance evaluation. It is linked/embedded in +`Contoso.Products.Database`, so the runner needs neither a repository checkout nor cwd-relative seed paths. +Research provenance is in [`demo-catalog.sources.json`](../data/demo-catalog.sources.json). Ordinary +Products/Shopping API, subscriber and automated Aspire test catalogs are deliberately unchanged. + +The snapshot has 174 source rows: 170 active MTB products, two inactive MTB products, one logically deleted +product, and the original non-stocked `LABOR` service. Original deterministic `^1`–`^32` IDs are preserved. +The **173 non-deleted** rows are read through Products' CoreEx accessor/mapping (inactive rows are retained), +serialized across the same JSON contract boundary used by events, and passed to Shopping's normal +`IProductSyncAdapter.ModifyAsync`/`DeleteAsync`. Existing Shopping mapping/upsert owns persistence. +There is no second Shopping catalog and no cross-database SQL between PostgreSQL and SQL Server. +Only this setup composition directly resolves both domains; business services still use adapters/HTTP/events. +The domains have separate DI/execution scopes and fresh local reference-data caches. Added Products +brands/subcategories live in Products' normal reference seed; Shopping only loads its own reference data. + +Shopping's product-free [`demo-baskets.seed.yaml`](../tests/Contoso.E2E.Runner/Data/demo-baskets.seed.yaml) +is loaded **after** product synchronization, retaining the original basket/item IDs and scenario state. +Setup verifies all replica IDs/SKUs/text/units/prices/activity/non-stocked flags, absence of stale/deleted +replicas, domain reference-data availability and basket/item references. It builds both lexical and +semantic projections using `AddProductEmbeddings` and the configured real provider/profile, verifies readiness +and query inference, and fails visibly if migrations, mappings, inference or integrity checks fail. +It evicts only the affected domains' reference-data collection keys through the normal Redis/FusionCache +backplane, waiting for distributed operations and surfacing failures; it never flushes shared Redis. + +**Snapshot bootstrap is not replication evidence.** It does not publish historical events or exercise +outbox/relay/broker delivery. Subsequent E2E product-change and basket scenarios continue exercising those paths. +Run setup with writers/relays/subscribers stopped, then restart the hosts; never use it to reconcile a live +catalog while scenarios are writing. Restart/clear serving hosts' reference-data caches after resetting data. + +For Products/Shopping-only maintenance (no API health checks, Customers/Cosmos or Orders reset): + +```powershell +$env:CoreExLocalTargetFramework = 'net10.0' +# Review/override E2E__Products__ConnectionString and E2E__Shopping__ConnectionString first. +# Ollama__BaseAddress selects the same real provider used by Products; the model must already be installed. +dotnet run --project samples\tests\Contoso.E2E.Runner -- --demo-catalog-only --accept-demo-reset +``` + +**Destructive:** the command resets demo data and runs migrations in the configured Products/Shopping databases. +The explicit flag acknowledges that reset; never target another person's shared databases without agreement. +The full interactive migration option additionally resets Customers and Orders. A model failure may leave a +partially prepared database; fix the reported cause and rerun the complete bootstrap before starting hosts. +`Data Seeding for E2E Testing` still performs stock adjustments through the API after hosts are running; +the base snapshot's inventory is intentionally small. + +The opt-in `DemoCatalogBootstrapTests` fixture lives in the single-target `Contoso.Test.Aspire` project, +but directly exercises setup composition without starting AppHost or the ordinary smoke fixtures. +It refuses to run unless E2E connections target exactly +`contoso_search_eval` and `Contoso_demo_eval`, and its Redis connection must use `defaultDatabase=14`. +With existing PostgreSQL/SQL Server/Redis/Ollama infrastructure, set the two database overrides and run: + +```powershell +$env:CoreExLocalTargetFramework = 'net10.0' +$settings = Get-Content samples\tests\Contoso.E2E.Runner\appsettings.json -Raw | ConvertFrom-Json +$env:E2E__Products__ConnectionString = $settings.E2E.Products.ConnectionString -replace 'Database=contoso;', 'Database=contoso_search_eval;' +$env:E2E__Shopping__ConnectionString = $settings.E2E.Shopping.ConnectionString -replace 'Initial Catalog=Contoso;', 'Initial Catalog=Contoso_demo_eval;' +$env:Aspire__StackExchange__Redis__ConnectionString = 'localhost:6379,defaultDatabase=14' +dotnet test samples\aspire\Contoso.Test.Aspire --framework net10.0 --filter 'FullyQualifiedName~DemoCatalogBootstrapTests' +``` + +It exercises actual source/replica agreement, flags/soft deletion, stale removal, basket integrity, +repeatable reconciliation/reset, cancellation and explicit real-provider failure. It does not touch Cosmos. + #### Load simulation Selecting **Run all scenarios as simulation** starts all four scenarios simultaneously in parallel worker pools. Each scenario runs its own workers concurrently with configurable parallelism and randomised per-step delays (to simulate realistic user behavior rather than a hammer load). @@ -229,7 +295,7 @@ Press `ESC` to stop gracefully. Errors are written to `logs/load-simulation-erro 2. Migrate and seed: `dotnet run --project samples/src/Contoso.Products.Database` (and Shopping, Orders) 3. Start Aspire: `aspire run` 4. Start the runner: `dotnet run --project samples/tests/Contoso.E2E.Runner` -5. Run **Database Migration and Base Data Refresh** (picks up any pending migrations) +5. Stop writers/relays/subscribers before **Database Migration and Base Data Refresh**; review all connection strings and its destructive resets. Skip startup health checks with ESC when hosts are stopped, then restart hosts after setup (including their reference-data caches). 6. Run **Data Seeding for E2E Testing** (stocks inventory for basket scenarios) 7. Run individual scenarios, or select **Run all scenarios as simulation** for load testing 8. Observe distributed traces, logs, and metrics in the Aspire Dashboard while the runner executes diff --git a/samples/docs/search.md b/samples/docs/search.md index 32053a789..04184aaed 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -19,12 +19,12 @@ The goal is not merely to store some numbers and label the result "AI". It is to | Local embedding provider | Implemented | Ollama/Nomic document and query instructions, response validation, and an Application adapter boundary are available. | | Separate search projection and pgvector semantic retrieval | Implemented | PostgreSQL stores 768-dimensional embeddings and computes exact, filtered cosine-distance rankings. | | Independent PostgreSQL full-text retrieval | Implemented; opt-in server mode | Weighted English `tsvector`/GIN matching, safe websearch parsing, exact eligible SKU first, no embedding calls. | -| Hybrid ranking / RRF | Proposed | The two retrieval branches are not fused. | +| Hybrid ranking / RRF | Implemented; opt-in server mode | Bounded lexical and semantic lists share live eligibility and structured filters, then fuse by RRF with exact eligible-SKU priority. | | Explicit backfill / asynchronous search indexing | Implemented | Test setup publishes an atomic initial backfill; one globbed product-event subscriber reconciles subsequent changes, including logical deletion. | -| Real-model relevance evaluation | Baseline measured; monthly/manual workflow configured | A version-controlled, 24-query judged set, isolated-catalog setup, opt-in REST/Ollama runner, and real report are available. Scores are observations, not quality gates. | +| Real-model relevance evaluation | v1 baseline historical; expanded MTB v2 catalog and judgments measured locally | A version-controlled 170-product mountain-bike catalog and 60-query reviewed set use an isolated seed and opt-in REST/Ollama runner. The checked-in v1 report remains historical and is not comparable to v2. Scores are observations, not quality gates. | | HNSW benchmark | Proposed | Exact full-catalog retrieval is still the comparison baseline; index performance has not been measured. | -The default baseline is PostgreSQL as the search store, one embedding per product/profile/recipe, a rebuildable search projection, and indexing outside product-write transactions. It uses local Ollama with Nomic v1.5 and 768 dimensions. Exact semantic retrieval ranks eligible **indexed** products; count means the number of those candidates, not the number judged relevant or the complete catalog size. The independent full-text branch counts only lexical matches (including exact SKUs). Hybrid retrieval windows remain future work. +The default mode remains semantic. Hybrid is enabled only with `Products:Search:Mode=Hybrid`; it requires both independently ready projections and a successful query embedding. The fixed initial policy takes the top 100 eligible candidates from each branch, uses RRF with `k=60`, prioritizes an eligible exact canonical SKU, and deterministically breaks remaining ties. Hybrid count is the deduplicated bounded union (at most 200), not a full-catalog or relevance count; paging is stable within that pool. These are explicit initial policy choices, not proven optimal tuning. ## Reading map @@ -97,7 +97,7 @@ This should retrieve the most relevant eligible Yeti products, not search the en Keyword mode allows ordinary ordering. Ranked modes own relevance ordering: CoreEx rejects an explicit `$orderby` and does not apply the configured default ordering. -The server configuration selects the mode; Products defaults to semantic mode. Set `Products:Search:Mode` to `FullText` (environment variable `Products__Search__Mode=FullText`) to exercise lexical retrieval instead. The only supported values are `Semantic` and `FullText`; an unsupported value fails explicitly. There is no public `$searchMode` parameter or test-only endpoint. Both ranked modes limit text to 200 normalized characters; the keyword-only eight-term limit does not apply. REST query schema and GraphQL-lite use the same selected configuration and unchanged filter fields. +The server configuration selects the mode; Products defaults to semantic mode. Set `Products:Search:Mode` to `FullText` for lexical retrieval or `Hybrid` for RRF (environment variable `Products__Search__Mode=Hybrid`). The supported values are `Semantic`, `FullText` and `Hybrid`; an unsupported value fails explicitly. There is no public `$searchMode` parameter or test-only endpoint. All ranked modes limit text to 200 normalized characters; the keyword-only eight-term limit does not apply. REST query schema and GraphQL-lite use the same selected configuration and unchanged filter fields. Price, availability, and other hard constraints belong in structured filtering, not in assumptions about what an embedding understands. Products exposes a decimal `price` filter with `eq`, `ne`, `in`, `gt`, `ge`, `lt` and `le`; inventory filters are not currently exposed. @@ -181,7 +181,7 @@ CREATE TABLE products.product_search ( ); ``` -The `768` is Nomic's selected native dimensionality. The composite primary key permits separate profiles/recipes without mixing them in retrieval. There is no HNSW or IVFFlat index: PostgreSQL evaluates exact distances. Full-text storage and coherent profile rollout remain later milestones. +The `768` is Nomic's selected native dimensionality. The composite primary key permits separate profiles/recipes without mixing them in retrieval. There is no HNSW or IVFFlat index: PostgreSQL evaluates exact distances. Coherent semantic profile rollout remains future work. Why a separate table instead of another product column? @@ -297,6 +297,17 @@ Several sources describe a model family rather than the exact seeded build. The Keep these sources and descriptions aligned when changing the fixtures. A product-family page is not a licence to invent build-specific claims, and manufacturer pages can change over time. +The expanded shared demo catalog has a [product/source manifest](../data/demo-catalog.sources.json): each added entry carries a source URL, source type, original short description, internal demo SKU, and illustrative USD price. FOX in the catalog means FOX Factory suspension; FOXRACING is the separate Fox Racing apparel/protection brand. Product facts and model-family descriptions are supported by their linked manufacturer pages (with retailer sources explicitly identified where necessary). Tags `e-bike` and `emtb` are user-requested colloquial search aliases on the three verified electric mountain bikes, alongside the `electric` category term. `dentist-bike` is a user-provided informal search alias on Yeti bikes for an offhand demo query. It is neither manufacturer provenance nor a factual claim about intended riders or affordability. + +The shared `LABOR` listing retains its hourly, non-stocked workshop-service identity and explicitly describes +professional MTB maintenance/repair labour without promising particular operations or included parts. +Its curated service-intent tags include `bike-service`, `bicycle-repair`, `workshop-labour`/`workshop-labor`, +`bike-mechanic` and `maintenance`. User-requested `wrench-monkey`, `grease-monkey`, `bike-whisperer` and +`spanner-man` are informal demo search aliases, not manufacturer facts or staff credentials; their provenance +is recorded separately under the manifest's sample service definitions. `hourly bike mechanic` and +`bike-whisperer` are interactive service-intent probes, not new MTB benchmark judgments: the benchmark's +`sku ne 'LABOR'` filter intentionally excludes this workshop service. + ### Keep rapidly changing facts relational Generally exclude price, current stock, and activity status from embeddings. Retrieve or filter these from live product data. @@ -340,7 +351,7 @@ flowchart LR API --> Client ``` -This is the implemented semantic flow. Requests without `$search` do not call the embedding provider or require a populated projection. +This is the semantic flow. In opt-in Hybrid mode the API also asks PostgreSQL's lexical projection for a bounded filtered list before fusing both ranked lists. Requests without `$search` do not call the embedding provider or require a populated projection. **CoreEx responsibilities:** carry intent through `QueryArgs`, validate configured constraints, apply keyword predicates, and distinguish ranked search from ordinary ordering. @@ -358,7 +369,7 @@ MCP (Model Context Protocol) lets an AI client discover and invoke tools such as The tool sends search text to Products and returns product data. It does not generate its own embeddings, query PostgreSQL directly, or maintain its own index. Non-MCP clients use the same capability. -The tool description defers to the Products server's configured ranked mode: semantic by default, or lexical matches when explicitly selected. It does not promise exact-SKU matching in semantic mode. Price/brand filters remain available in REST/GraphQL-lite but are not yet arguments on the MCP tool. +The tool description defers to the Products server's configured ranked mode: semantic by default, or lexical/hybrid when explicitly selected. It does not promise exact-SKU matching in semantic mode; FullText and Hybrid prioritize an exact eligible SKU. Hybrid requires both indexes and embedding inference, and its results come from a bounded union. Price/brand filters remain available in REST/GraphQL-lite but are not yet arguments on the MCP tool. Search does not resolve authentication or authorization. The sample's existing authentication gaps remain separate concerns. @@ -366,9 +377,9 @@ Search does not resolve authentication or authorization. The sample's existing a ### Initial readiness: an explicit, atomic backfill -Rebuilding is reusable Infrastructure logic, **not an API startup responsibility**. The earlier API -`--rebuild-search` maintenance mode has been removed. There is no HTTP rebuild endpoint or standalone indexing -console at this milestone. +Rebuilding is reusable Infrastructure logic, **not an API startup responsibility**. There is no HTTP rebuild +endpoint. The E2E Runner's [shared demo bootstrap](aspire.md#shared-demo-catalog-bootstrap) invokes the same +Infrastructure rebuild explicitly, including its `--demo-catalog-only --accept-demo-reset` maintenance path. Products read-test `OneTimeSetUp` seeds the catalog and invokes `ProductSearchIndex.RebuildAsync` with controlled embeddings. Aspire E2E setup seeds the catalog, clears Redis, then creates a small test-owned service scope using @@ -406,7 +417,7 @@ reject a parent transaction: the lexical publication must commit independently b `products.product_search_readiness` records successful initial preparation for each profile/recipe. The readiness marker commits atomically with backfill publication, even for an empty catalog. Incremental events never mark a partially indexed catalog ready. A failed initial backfill leaves it unready; a failed later backfill preserves the previous published projection and readiness. -For a semantic request, the repository checks this small readiness record before inference and again inside its `RepeatableRead` snapshot. It joins compatible vectors to **live** product data, orders by exact cosine distance and product ID (a deterministic tie-breaker), and applies paging/count. There are no full-catalog freshness scans. +For a semantic request, the repository checks this small readiness record before inference and again inside its `RepeatableRead` snapshot. It joins compatible vectors to **live** product data, orders by exact cosine distance and product ID (a deterministic tie-breaker), and applies paging/count. Hybrid additionally verifies lexical readiness both before inference and inside the same snapshot; both branch queries share the live product/subcategory/inventory joins and parsed structured filters. There are no full-catalog freshness scans. | Condition | Search behavior | |---|---| @@ -419,13 +430,13 @@ For a semantic request, the repository checks this small readiness record before Caller cancellation propagates; a broken internal vector contract remains a programming error, not a successful empty result. There is no silent keyword fallback. -The framework manages commit, rollback and cleanup, and enlists the EF context in the shared database transaction. A nested unit of work uses a savepoint, not a new independent transaction: it inherits the parent's isolation. Semantic search therefore explicitly rejects an existing parent unless it is `RepeatableRead` or `Serializable`; silently accepting `ReadCommitted` would lose the consistency guarantee. For standalone requests, inference occurs before opening the snapshot. When a caller already owns a compatible parent transaction, that caller owns its lifetime, including time spent on inference. The search callback does not enqueue events, but any events already queued by a consumer follow the normal root-unit-of-work outbox rules. +The framework manages commit, rollback and cleanup, and enlists the EF context in the shared database transaction. A nested unit of work uses a savepoint, not a new independent transaction: it inherits the parent's isolation. Semantic and Hybrid search therefore explicitly reject an existing parent unless it is `RepeatableRead` or `Serializable`; silently accepting `ReadCommitted` would lose the consistency guarantee. For standalone requests, inference occurs before opening the snapshot. When a caller already owns a compatible parent transaction, that caller owns its lifetime, including time spent on inference. The search callback does not enqueue events, but any events already queued by a consumer follow the normal root-unit-of-work outbox rules. Price, inventory and activity changes do not require new embeddings. Name, description, tags and reference-data labels do. Labels use the existing orchestrator/cache; reference-data-driven invalidation and reindexing are **not implemented**. A label-only change needs an explicit backfill with appropriate cache invalidation. API integration tests rebuild with controlled vectors, isolating distance ordering and failure behavior from model quality. Subscriber integration tests cover duplicate/out-of-order notifications, logical deletion, content-hash skips, concurrent edits and provider retries. Test.Aspire's MCP lifecycle asserts real semantic cleaning intent; its product-search lifecycle drives create/update/activate/delete through HTTP and polls the stored document and search results through the real relay, broker, subscriber and model, with no intervening rebuild. The MCP and checkout scenarios no longer require ordering to avoid freshness failures. -There is no relevance cutoff: every eligible compatible vector is a candidate. `$count=true` reports eligible indexed candidates, including poor semantic matches and excluding not-yet-indexed products. An unrelated query still returns neighbors; clients must not interpret a returned product as guaranteed suitable. +There is no relevance cutoff: semantic mode returns every eligible compatible vector, including poor matches, and `$count=true` reports eligible indexed candidates. FullText count reports matching lexical candidates. Hybrid `$count=true` reports the deduplicated union of its two bounded candidate lists (0–200), not all eligible products. Hybrid's branch filters are applied in PostgreSQL before each 100-row limit; paging and count use the same repeatable-read snapshot and the fused pool is independent of page size/offset. Consequently, `$skip` beyond that pool returns no items, and `take` cannot expand the maximum reachable set. An unrelated query still returns semantic neighbors in semantic mode; returned products are candidates, not guaranteed suitable matches. ### Independent lexical projection @@ -604,9 +615,9 @@ PostgreSQL full-text search uses: A `tsvector` is not a neural embedding. PostgreSQL's built-in rank functions are not BM25 by default. -`plainto_tsquery` is a straightforward candidate for our opaque text contract: it processes plain text and ANDs surviving words. Long conversational queries may produce few lexical matches. +Products uses `websearch_to_tsquery` in its independent full-text branch: it processes words and supports PostgreSQL web-search syntax such as quoted phrases, OR, and negation. Long conversational queries may produce few lexical matches. -`websearch_to_tsquery` recognizes syntax such as quotes, OR, and negation. Choosing it would deliberately introduce provider-specific query interpretation that needs documenting. CoreEx does not currently promise that grammar. +That interpretation is provider-specific; CoreEx does not promise that grammar. SKUs need explicit treatment because punctuation and language tokenization can behave differently from ordinary prose. Exact SKU matching is a useful separate signal, not something to leave to semantic similarity. @@ -631,7 +642,14 @@ Semantic list: B, D, A A and B benefit from appearing in both lists. The constant `k` smooths the effect of position; 60 is a common starting value, not a universal optimum. -Retrieve a bounded candidate pool from each eligible branch, combine by product ID, fuse ranks, apply a deterministic tie-breaker, and return the requested window. +The opt-in Products policy retrieves up to **100 eligible candidates per branch** after live product eligibility and the complete structured query filters have been applied in PostgreSQL. Branch ranks are one-based. It combines by product ID and uses equal-weight RRF with fixed **`k=60`**: + +```text +score = (lexical candidate ? 1 / (60 + lexical rank) : 0) + + (semantic candidate ? 1 / (60 + semantic rank) : 0) +``` + +An eligible exact canonical SKU is ordered ahead of the fused scores, so a vector candidate budget cannot hide it. Remaining order is descending fused score, then lexical rank, semantic rank, and ordinal product ID. The unique union contains at most **200** candidates. This deterministic initial budget and `k` are policy, not a claim of optimal tuning. The candidate pool is usually larger than the final page. Its size affects both quality and work; ten returned products does not imply ten candidates per branch are sufficient. @@ -651,7 +669,7 @@ Ranked search must define its result window and count semantics: - Deep offset paging can require increasingly large candidate retrieval. - Stable tie-breaking does not freeze rankings when content or indexes change. -Before implementation is complete, record the chosen behavior for `$skip`, `$take`, `$count`, maximum ranked window, and empty/weak matches. Do not report candidate count as total match count without making that meaning explicit. +For hybrid mode, the branch candidate lists are independent of requested paging, `$skip` and `$take` are applied only after fusion, and `$count` is the deduplicated candidate-union size (maximum 200). It is not a total match count or the complete eligible catalog. Semantic and FullText keep their existing count behavior. No branch is silently dropped: an unready lexical/semantic projection or failed query embedding makes Hybrid unavailable (HTTP 503), rather than returning a partial branch. Empty lists are valid when both indexes are ready; weak semantic matches are not assigned a confidence score. ## Model hosting and operational costs @@ -717,11 +735,11 @@ Both expected first products survive the larger candidate set. A price-under-20 The live MCP HTTP transport was also exercised against this API: `search_products` returned the Nano cleaner first for the original cleaning intent, including its description and live price. Temporarily invalidating its indexed fingerprint produced a tool error rather than a successful partial result; the fingerprint was restored afterwards. This bounded manual check is distinct from the full Test.Aspire topology. -The lower ranks expose the remaining quality problem: sealant is not a cleaner, and a repair multi-tool ranks above some drivetrain components. Exact search does not fix that; it faithfully returns the model's nearest candidates. These examples motivate hybrid retrieval and a curated evaluation set, not an invented confidence threshold or a claim that search is finished. +The lower ranks expose the remaining quality problem: sealant is not a cleaner, and a repair multi-tool ranks above some drivetrain components. Exact search does not fix that; it faithfully returns the model's nearest candidates. These examples motivated hybrid retrieval and a curated evaluation set, not an invented confidence threshold or a claim that search is finished. ### Curated real-model relevance evaluation -The opt-in evaluation lives in [`tools/product-search-relevance.dataset.json`](../../tools/product-search-relevance.dataset.json). It contains 24 realistic requests split into 18 tuning and 6 held-out queries: exact SKUs, product terminology, paraphrases, broad/ambiguous intent, hard price filters, and out-of-catalog requests. Every query attests that all 29 default-eligible seeded products were reviewed; listed judgments carry a grade and rationale, while each omitted SKU explicitly means grade 0. The eligible snapshot and the exact seed-file SHA-256 are pinned so changed seed data or live API catalog fields stop the run rather than silently changing the benchmark. +The opt-in evaluation lives in [`tools/product-search-relevance.dataset.json`](../../tools/product-search-relevance.dataset.json). Its v2 snapshot contains 170 eligible mountain-bike products and 60 reviewed requests split into 48 tuning and 12 held-out queries: exact identifiers, product terminology, paraphrases, focused category intent, hard price filters, and out-of-catalog requests. Every query attests that the full 170-product catalog was reviewed; listed judgments carry a grade and rationale, while each omitted SKU explicitly means grade 0. The snapshot, shared seed SHA-256, source URLs, product classifications, price policy, and curated-tag provenance are pinned so catalog drift cannot silently change the benchmark. The product expansion and per-product research references are in [`samples/data/demo-catalog.sources.json`](../data/demo-catalog.sources.json). Catalog SKUs are internal demo identifiers, not verified manufacturer part numbers; no `DEMO-` prefix is required. Genuine model names retain Demo (`SPECIALIZED-DEMO`). Every catalog price is illustrative USD, not a retail quote. Run the evaluator only when the Products REST API, its PostgreSQL database, and Ollama are already available and the configured profile's backfill is ready: @@ -730,30 +748,49 @@ Run the evaluator only when the Products REST API, its PostgreSQL database, and .\tools\product-search-relevance.ps1 -OutputPath artifacts\product-search-relevance-report.json ``` -The first command exercises metric math, invalid judgments, catalog drift, incomplete backfill, and mocked HTTP failure without external services. The second makes real HTTP requests; it never rebuilds the index or substitutes synthetic embeddings. It verifies the pinned Nomic model digest, semantic/ranked API mode, exact 29-product live catalog signature, and candidate count for every query. A missing API, model/profile mismatch, seed/catalog drift, or incomplete/unready index fails explicitly. Reports identify the model, profile, document recipe, seed and catalog hashes, and include per-query results plus separate tuning/held-out aggregates. Do not edit held-out judgments or tune against held-out scores. +The first command exercises metric math, dataset and alias-provenance validation, multi-page catalog retrieval, drift, incomplete backfill, bounded-count semantics, split isolation, and mocked HTTP failure without external services. The second makes real HTTP requests; it never rebuilds the index or substitutes synthetic embeddings. It verifies the pinned Nomic model digest when the approach uses semantics, the selected ranked API mode, the full 170-product live catalog signature, and approach-appropriate candidate counts. A missing API, model/profile mismatch, seed/catalog drift, or incomplete/unready index fails explicitly. Hybrid count is checked against the bounded eligible union, while branch completeness is checked separately by the guarded setup fixture. Reports identify model and lexical recipe identities, policy, seed/catalog hashes, and per-query results. Tune only on tuning queries; held-out judgments remain untouched until one final evaluation. + +The shared [`samples/data/demo-catalog.seed.yaml`](../data/demo-catalog.seed.yaml) serves interactive demo/MCP, the E2E Runner bootstrap, and this evaluation; ordinary API test fixtures remain small and specialized. It contains 174 source rows: 170 active MTB products, two inactive MTB products, one deleted MTB product, and the original non-stocked `LABOR` service needed by existing baskets. Bootstrap replicates and indexes all 173 non-deleted rows. The evaluator adds the existing structured filter `sku ne 'LABOR'` to every catalog/search request, excluding the workshop service while preserving the same 170 reviewed MTB identities and 60 queries. Excluding the entire workshop category would incorrectly discard five reviewed Muc-Off products assigned to the existing Miscellaneous subcategory; those classifications remain unchanged. This changes the seed file hash/path, not MTB judgments or ranking. The historical v1 baseline remains immutable with its original path/hash. -The dedicated [`tools/product-search-relevance.seed.yaml`](../../tools/product-search-relevance.seed.yaml) is an exact copy of the reviewed test catalog, but is loaded explicitly through DbEx. The safe sequence is: create/migrate the isolated database (`all`), reset the `products` schema (`reset --accept-prompts`), restore its reference rows (`data`), then load the evaluation catalog (`execute `). The `Reset` filter is scoped to the `products` schema, and in practice clears its reference and transactional tables, search projection, and readiness marker. `Data` restores embedded reference data; `Execute` then inserts the product, inventory, and movement rows. Execute is **not** a reset or upsert: repeating it against populated tables fails with a duplicate-primary-key error and leaves existing rows unchanged. Do not rerun Execute without first doing the explicit Reset and reference-data reload. +Additional brands/subcategories are normal Products-owned reference data in `Contoso.Products.Database/Data/ref-data.seed.yaml`; existing codes/text are unchanged. Shopping receives only its own reference data (including units), never Products brands/categories. For an evaluation-only database, the safe DbEx sequence is: create/migrate the isolated database (`database`, which avoids code generation), reset the `products` schema (`reset --accept-prompts`), restore its reference rows (`data`), then load the shared catalog (`execute `). Reset clears reference/transactional tables, projections and readiness. Execute is **not** a reset/upsert: repeating it against populated tables fails with duplicate keys. Never run this sequence against a shared/default database. For a complete demo including Shopping, use the [bootstrap](aspire.md#shared-demo-catalog-bootstrap) instead. From the repository root, first set `$connectionString` to a connection whose database is exactly `contoso_search_eval`, then run: ```powershell $databaseProject = 'samples\src\Contoso.Products.Database\Contoso.Products.Database.csproj' -dotnet run --no-launch-profile --project $databaseProject --no-build --framework net10.0 -- all --connection-string $connectionString +dotnet run --no-launch-profile --project $databaseProject --no-build --framework net10.0 -- database --connection-string $connectionString dotnet run --no-launch-profile --project $databaseProject --no-build --framework net10.0 -- reset --accept-prompts --connection-string $connectionString dotnet run --no-launch-profile --project $databaseProject --no-build --framework net10.0 -- data --connection-string $connectionString -$seedPath = (Resolve-Path 'tools\product-search-relevance.seed.yaml').Path +$seedPath = (Resolve-Path 'samples\data\demo-catalog.seed.yaml').Path dotnet run --no-launch-profile --project $databaseProject --no-build --framework net10.0 -- execute $seedPath --connection-string $connectionString ``` The NUnit fixture below is the next step: it builds the vector projection, not the catalog data. The REST evaluator itself never resets, loads, or reindexes anything. -The isolated preparation fixture, [`ProductSearchRelevanceSetupTests.cs`](../tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs), refuses to run unless the configured database is exactly `contoso_search_eval`; it does not seed or reset data. It uses the real indexing path to rebuild embeddings from the already-loaded catalog and verifies profile readiness. The separate [monthly/manual GitHub Actions workflow](../../.github/workflows/product-search-relevance.yml) provisions PostgreSQL, Redis, and Ollama, installs the pinned model, creates that dedicated database, runs the explicit DbEx sequence, the fixture, and the live API evaluator, then uploads the JSON report and API log as a 90-day artifact. Setup failures fail the workflow, and cleanup runs regardless. It is not part of default CI. Run it manually with `workflow_dispatch`, or download the artifact from the monthly run. +The isolated preparation fixture, [`ProductSearchRelevanceSetupTests.cs`](../tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs), refuses to run unless the configured database is exactly `contoso_search_eval`; it does not seed or reset data. It uses the real indexing path to rebuild projections from the already-loaded catalog and verifies row completeness separately from search-result counts (Hybrid candidate counts are intentionally bounded). The separate [monthly/manual GitHub Actions workflow](../../.github/workflows/product-search-relevance.yml) provisions PostgreSQL, Redis, and Ollama, installs the pinned model, creates that dedicated database, runs the explicit DbEx sequence, the fixture, and the live API evaluator, then uploads the JSON report and API log as a 90-day artifact. Setup failures fail the workflow, and cleanup runs regardless. It is not part of default CI. Run it manually with `workflow_dispatch`, or download the artifact from the monthly run. + +The checked-in [`baseline report`](product-search-relevance-baseline.json) is the historical v1 measurement over 29 eligible products and 24 queries, not a target or a v2 comparison. Its v1 results remain in the report for historical context, but the changed catalog, labels, and split make its metrics incomparable with v2. Do not pass it as `-BaselinePath` for v2 runs. + +#### Expanded MTB v2 local measurements (before SKU prefix removal) -The captured [`baseline report`](product-search-relevance-baseline.json) is one measured run, not a target: tuning mean nDCG@5 is **0.8377** over 17 relevant queries (one no-relevant query is reported separately), and tuning mean Precision@5 is **0.3765**. Exact-SKU top-1 is **2/3**; the Shimano XT derailleur query returned Muc-Off cleaner. Held-out mean nDCG@5 is **0.8770** over only **3** relevant queries, mean Precision@5 is **0.2667**, and exact-SKU top-1 is **0/1** (the SRAM X0 query returned Muc-Off cleaner). The held-out split also has three no-relevant/out-of-catalog queries; its tiny relevant denominator makes these descriptive results highly uncertain. The report records zero price-filter violations in 15 tuning and 5 held-out constrained results. No score threshold is imposed, and these results do not justify a claim that search is generally good or bad. +The expanded v2 catalog was evaluated locally against the isolated `contoso_search_eval` database using the pinned `nomic-embed-text:v1.5` model, before removing the leading `DEMO-` from 142 internal SKUs. The live REST catalog matched all 170 then-reviewed eligible products. The 48 tuning queries include 46 queries with judged relevant products; the two no-relevant queries are reported separately and excluded from mean relevance metrics. The metrics below remain historical observations, not measurements of the renamed catalog, quality gates, or comparisons to v1: + +| Approach and split | Mean nDCG@5 | Mean Precision@5 | Exact-SKU top-1 | Constrained results | Constraint violations | +|---|---:|---:|---:|---:|---:| +| Semantic, tuning (48 queries) | 0.8514 | 0.5522 | 1/6 | 15 | 0 | +| FullText, tuning (48 queries) | 0.4956 | 0.2783 | 6/6 | 0 | N/A (no results) | +| Hybrid, tuning (48 queries) | 0.9562 | 0.5783 | 6/6 | 15 | 0 | +| Hybrid, held-out (12 queries) | 0.8774 | 0.3600 | 4/4 | 10 | 0 | + +The Hybrid policy and judgments were reviewed on tuning before the held-out split was run once. The held-out set has 10 relevant queries and two no-relevant/out-of-catalog queries; its result was not used to change the policy, labels, or catalog. The local JSON reports are retained under ignored `artifacts/` paths (`product-search-mtb-semantic-tuning.json`, `product-search-mtb-fulltext-tuning.json`, `product-search-mtb-hybrid-tuning.json`, and `product-search-mtb-hybrid-held-out.json`) and are not promoted to the checked-in v1 baseline. + +Prefix removal updates the shared seed, research-source identifiers, reviewed catalog, judgment references and exact-SKU query text without changing grades, rationales or query splits. It changes the seed/dataset/catalog hashes and canonical search documents, so existing lexical/semantic projections and Shopping replicas must be rebuilt from the renamed source. The guarded [isolated bootstrap fixture](aspire.md#shared-demo-catalog-bootstrap) resets only `contoso_search_eval` / `Contoso_demo_eval` with Redis database 14 and rebuilds both projections using the real model. Shared/default databases are not refreshed by that test: stop their writers and use the explicitly acknowledged demo bootstrap before serving the new identifiers. Prior reports and their measured metrics remain unchanged; they cannot be used as identical-identity baselines for the renamed catalog. No live relevance evaluation has been rerun for this identifier revision. + +The post-rename isolated bootstrap passed locally on 2026-10-15: 174 unique source SKUs, 173 non-deleted Products/Shopping rows and 173 rows in each real-model semantic/lexical projection. The live API's 170 eligible products matched the reviewed catalog signature `088C254161DDB99DD27D9AB06208616532708F200F1B00A5AD4302D8DCD77B03`. A single renamed exact-SKU Hybrid smoke request also passed; this is setup/identity verification, not a rerun of the relevance benchmark. Routine reports belong in the git-ignored `artifacts/` directory, not in the checked-in baseline. Replacing `samples/docs/product-search-relevance-baseline.json` is an explicit review step: compare the new report's per-query results, aggregate metrics, and model/dataset/catalog/implementation identities with the existing baseline; explain why the replacement is appropriate in the change description; then deliberately copy the reviewed report over the baseline and update the documented findings. Git history preserves the previous baseline. Do not automatically promote scheduled reports or replace the baseline merely because a score decreased. -Rerun and review the benchmark when the embedding model/profile, document recipe or text fields, ranking/retrieval implementation, judged catalog/seed data, or judgment set changes. Preserve the prior report for comparison and record the changed identity; do not silently replace labels. Full-text tuning compares only the tuning split with this semantic baseline; held-out execution is deferred until an explicitly approved final comparison, not used as a tuning target. Any new catalog revision requires reviewing all eligible products for every query before updating the pinned catalog identity. +Rerun and review the benchmark when the embedding model/profile, document recipe or text fields, ranking/retrieval implementation, judged catalog/seed data, or judgment set changes. Preserve prior reports and record the changed identity; do not silently replace labels. The old v1 report cannot be used as a baseline comparison for v2. FullText and Hybrid can be evaluated on the v2 tuning split without a baseline comparison. Once the Hybrid policy and v2 judgments are settled, run the Hybrid held-out split exactly once as the final v2 observation; do not tune from that result. Any new catalog revision requires reviewing all eligible products for every query before updating the pinned catalog identity. ### Lexical tuning-only comparison @@ -766,23 +803,24 @@ $env:Products__Search__Mode = 'FullText' dotnet test samples\tests\Contoso.Products.Test.Api --no-build --framework net10.0 --filter 'FullyQualifiedName~ProductSearchRelevanceSetupTests' # Start the API separately with the same environment and your chosen BaseAddress. .\tools\product-search-relevance.ps1 -BaseAddress http://localhost:5217/ -Approach fulltext -Split tuning ` - -BaselinePath samples\docs\product-search-relevance-baseline.json ` -OutputPath artifacts\product-search-lexical-tuning.json ``` -This evaluator makes **no Ollama requests** in full-text mode, identifies the lexical recipe and embedded SQL +This evaluator makes **no Ollama requests** in FullText mode, identifies the lexical recipe and embedded SQL hashes, and selects tuning queries **before issuing any search requests**. Full-text without `-Split tuning` -is rejected at this milestone. Semantic evaluation retains its original full-split default; `-Split tuning` -also supports a semantic tuning-only run. Baseline comparison validates dataset/catalog/seed identities and -tuning query IDs and emits aggregate deltas without replacing the original baseline. No judgments change. +is rejected. Semantic evaluation retains its original full-split default; `-Split tuning` also supports a +semantic tuning-only run. A baseline comparison is valid only for a report with the identical dataset, +catalog, seed, and query IDs; the historical v1 baseline therefore cannot be used for these v2 commands. +No judgments change. Lexical counts are matching-set counts, not semantic full-catalog candidate counts. The runner checks that they are bounded by the live eligible catalog and that page size agrees with count. It cannot infer lexical index completeness from a naturally sparse/no-match result; the guarded host fixture performs the actual full -backfill and verifies its 31 non-deleted source rows (29 active). No fake “every product must match” completeness +backfill and verifies the database-derived non-deleted source-row count (172 in the current v2 seed; 170 active). No fake “every product must match” completeness probe is sent to the lexical branch. -Measured locally on **2026-10-15**, through the real FullText REST API over the isolated, dedicated seed: +Historical v1 measurement, recorded locally on **2026-10-15**, through the real FullText REST API over the +then-current isolated seed. These results are not measurements of v2: | Tuning metric | Original semantic baseline | Independent lexical v1 | |---|---:|---:| @@ -798,13 +836,78 @@ requests (often containing price words/numbers) are too restrictive under websea stemming does not supply absent synonyms or paraphrases. Only the three exact-SKU requests and the cross-country race request returned results. No parsing policy, catalog content, or judgments were adjusted to improve these scores. This is a useful independent lexical baseline, **not** an improvement to replace semantic defaults. -RRF/fusion, ANN, and reranking are not implemented. -The ignored report `artifacts/product-search-lexical-tuning.json` contains all 18 per-query results, -matched counts, identities and baseline deltas; test/setup/API diagnostics are also under `artifacts/`. -The semantic baseline file remains unchanged. **No held-out requests or lexical held-out scores were run.** +The ignored historical report `artifacts/product-search-lexical-tuning.json` contains the 18 v1 per-query +results, matched counts, identities and baseline deltas; test/setup/API diagnostics are also under `artifacts/`. +The checked-in semantic v1 baseline remains unchanged. No held-out lexical requests or scores were run. + +### Hybrid tuning and final held-out comparison + +Use the same isolated setup sequence above. Hybrid setup invokes the real semantic rebuild, which also rebuilds +the lexical projection, and checks both projections against database-derived row counts (172 non-deleted, 170 active). Set Hybrid mode both for the +explicit setup fixture and for the separately started Products API: + +```powershell +$env:Aspire__Npgsql__ConnectionString = $connectionString +$env:Products__Search__Mode = 'Hybrid' +dotnet test samples\tests\Contoso.Products.Test.Api --no-build --framework net10.0 --filter 'FullyQualifiedName~ProductSearchRelevanceSetupTests' +# Start the API separately with the same environment and your chosen BaseAddress. +.\tools\product-search-relevance.ps1 -BaseAddress http://localhost:5217/ -Approach hybrid -Split tuning ` + -OutputPath artifacts\product-search-hybrid-tuning.json +``` + +Hybrid evaluation verifies the same exact eligible catalog identity, asks Ollama for the pinned model identity, +and validates every returned count against the constrained eligible catalog and 200-candidate ceiling. The +report retains the separate full eligible catalog count and the bounded result count, plus the lexical recipe, +`k=60`, and per-branch budget of 100. Candidate counts are not interpreted as proof +that either index is complete; the guarded setup fixture checks each stored index row count directly. + +For a new catalog or ranking revision, review all tuning rankings and settle the policy before running the +held-out split once. Do not adjust `k`, candidate budget, labels, document content, or any other ranking choice +after that final observation: + +```powershell +.\tools\product-search-relevance.ps1 -BaseAddress http://localhost:5217/ -Approach hybrid -Split held-out ` + -OutputPath artifacts\product-search-hybrid-held-out.json +``` + +This was the final pre-rename v2 held-out observation, not a comparison to the historical v1 report. It was +run before SKU prefix removal; do not relabel it as a current-catalog measurement or change the policy based on its output. The evaluator's +held-out command is available for Semantic or Hybrid; FullText remains tuning-only. The checked-in v1 semantic +baseline is not replaced, and reports remain in ignored `artifacts/`. The separate monthly/manual workflow remains semantic-only and has not been trialed remotely from this branch. +Historical v1 comparison, measured locally on **2026-10-15** against the then-reviewed catalog, seed, +judgments and pinned Nomic model. It is not a v2 result: + +| Tuning metric | Semantic baseline | Independent FullText | Hybrid (100 per branch, `k=60`) | +|---|---:|---:|---:| +| Mean nDCG@5 (17 relevant queries) | 0.8377 | 0.2072 | **0.8966** | +| Mean Precision@5 | 0.3765 | 0.0588 | **0.3882** | +| Exact-SKU top-1 | 2/3 | 3/3 | **3/3** | +| Price-constrained result rows / violations | 15 / 0 | 0 / 0 | **15 / 0** | + +In that v1 tuning run, Hybrid exceeded its semantic baseline by 0.0588 nDCG@5 and 0.0118 Precision@5, while preserving +all 15 returned price-constrained rows without a violation and fixing the missed exact SKU. FullText's zero +constrained rows are not useful filter-retrieval evidence; its violation rate is undefined. These observed +tuning results support retaining the fixed Hybrid policy as an opt-in, not making it the default. + +In the historical v1 run, the fixed policy was compared with its semantic baseline on held-out exactly once, +without changing the policy or judgments: + +| Held-out metric | Semantic baseline | Hybrid | +|---|---:|---:| +| Mean nDCG@5 (3 relevant queries) | 0.8770 | **1.0000** | +| Mean Precision@5 | 0.2667 | 0.2667 | +| Exact-SKU top-1 | 0/1 | **1/1** | +| Price-constrained result rows / violations | 5 / 0 | **5 / 0** | + +That v1 held-out set contained only three relevant queries; its quality figures are descriptive and highly uncertain. +It also contains three no-relevant/out-of-catalog queries, for which both approaches still return candidates. +Those measured v1 results do not justify changing the semantic default or a general quality claim. The complete +historical per-query reports are `artifacts/product-search-hybrid-tuning.json` and +`artifacts/product-search-hybrid-held-out.json`; the checked-in semantic baseline remains unchanged. + Judgments are graded 0 (irrelevant), 1 (marginal), 2 (useful secondary result), or 3 (direct fit). The report uses gain `2^grade - 1` for nDCG@5; Precision@5 counts grade 2 or 3 and always divides by five. Exact-SKU top-1 accuracy is shown only for exact-SKU queries. Queries with no grade-above-zero product are listed separately and excluded from nDCG/precision means rather than counted as successful empty results. Price-filter violations are counted and reported; no similarity/confidence cutoff or quality pass threshold is invented. The checked-in query set is not itself a measured baseline: capture and review a real report before making claims about full-catalog relevance. Measure document and query embedding separately. Caching repeated query embeddings can help, but keys must include the profile and relevant preprocessing; user text may be sensitive, so retention/logging needs care. @@ -817,9 +920,9 @@ Trace embedding time, database time, indexing lag, retries, and failures. Avoid |---|---|---| | 1. Documents and embeddings | See how verified catalog content becomes vectors. | Selected profile, reproducible builder, provider configuration, provenance. | | 2. Exact pgvector retrieval (implemented) | Understand distance ordering with real filters. | Real database tests for ordering, eligibility, profile isolation, paging, freshness and atomic rebuild failure. | -| 3. Full-text (implemented); RRF (future) | Compare exact terminology with semantic intent. | Independent lexical matching, exact-SKU priority, model-free lifecycle and measured tuning-only comparison. Hybrid comparisons remain future work. | +| 3. Full-text and Hybrid/RRF (implemented; Hybrid opt-in) | Compare exact terminology with semantic intent. | Independent lexical matching, fixed bounded RRF (`k=60`, 100 candidates per branch), exact-SKU priority, deterministic paging/counts, and historical v1 comparison evidence. | | 4. Indexing and backfill (implemented; operational recovery remains) | Maintain rebuildable derived state. | Duplicate/out-of-order handling, stale-write protection, updates/logical deletes and explicit readiness. Resumable reconciliation remains future work. | -| 5. Curated relevance evaluation (implemented; baseline captured) | Assess real-model rankings against reviewed judgments. | Pinned full-catalog labels, isolated held-out queries, metric definitions and a reproducible live report. | +| 5. Curated relevance evaluation (v1 baseline captured; v2 catalog prepared, not measured) | Assess real-model rankings against reviewed judgments. | Pinned 170-product MTB labels, isolated held-out queries, metric definitions and a reproducible live report. | | 6. HNSW benchmark | Measure speed versus recall at meaningful scale. | Exact baseline, larger corpus, real query plans, latency and recall results. | | 7. MCP demonstration | Reuse the domain capability from an AI client. | Same API behavior, accurate tool descriptions, cross-host demonstration. | @@ -857,18 +960,21 @@ These files own the implemented behavior and the foundations for later milestone | [Ollama client](../src/Contoso.Products.Infrastructure/Clients/Ollama/OllamaHttpClient.cs) | Task prefixes, model profile, explicit truncation policy, and response validation. | | [Embedding client tests](../tests/Contoso.Products.Test.Unit/Clients/Ollama/OllamaHttpClientTests.cs) | Mocked success/error responses and exact request construction. | | [Live embedding smoke check](../../tools/test-product-embeddings.ps1) | Explicit local-model probe and retained relevance expectations. | -| [Curated relevance judgments](../../tools/product-search-relevance.dataset.json) | Versioned graded labels, reviewed catalog snapshot, tuning/held-out split, and pinned model/seed identity. | -| [Relevance evaluator](../../tools/product-search-relevance.ps1) | Opt-in real REST/Ollama evaluation, drift/backfill checks, per-query metrics, and split aggregates. | -| [Relevance evaluator tests](../../tools/test-product-search-relevance.ps1) | Local metric, dataset validation, drift, backfill, and mocked-HTTP failure tests. | -| [Dedicated evaluation seed](../../tools/product-search-relevance.seed.yaml) | Exact reviewed catalog copy loaded using DbEx Execute after isolated reset/reference-data reload. | +| [Curated relevance judgments](../../tools/product-search-relevance.dataset.json) | Versioned graded labels, reviewed 170-product MTB snapshot, 48/12 tuning/held-out split, sources and pinned model/seed identity. | +| [Demo expansion source manifest](../data/demo-catalog.sources.json) | Product-level source URLs, short descriptions, illustrative prices and provenance for the 142 added products. | +| [Relevance evaluator](../../tools/product-search-relevance.ps1) | Opt-in real REST/Ollama evaluation, complete paged catalog verification, drift/backfill checks and per-query metrics. | +| [Relevance evaluator tests](../../tools/test-product-search-relevance.ps1) | Local metric, dataset/provenance validation, paged-catalog drift, backfill, split isolation and mocked HTTP tests. | +| [Shared demo catalog](../data/demo-catalog.seed.yaml) | One embedded Products seed for interactive demo/MCP, runner bootstrap, and evaluation; domain reference data stays in the owning Database project. | | [Isolated setup fixture](../tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs) | Explicitly guarded real backfill and readiness verification for `contoso_search_eval` only. | | [Lexical projection](../src/Contoso.Products.Infrastructure/Repositories/ProductLexicalSearchIndex.cs) | Model-free rebuild/synchronization, separate recipe/readiness, composable matches and weighted embedded SQL. | | [Lexical API tests](../tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs) | English parsing, exact SKUs, live eligibility/filters, paging, weights/ties, lifecycle, readiness and cancellation. | | [Lexical subscriber tests](../tests/Contoso.Products.Test.Subscribe/ProductLexicalSearchSubscriberTests.cs) | Real event dispatch and idempotent/deleted state with an unavailable embedding provider. | | [Monthly/manual workflow](../../.github/workflows/product-search-relevance.yml) | Separate scheduled run with isolated services and retained JSON/API-log artifact. | -| [Measured baseline report](product-search-relevance-baseline.json) | Captured model/profile/catalog identities, per-query results, and split metrics; no quality threshold. | -| [Product query configuration](../src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs) | Server-selected semantic mode and existing filter/order fields. | -| [Product repository](../src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs) | Existing filters, joins, ordering, mapping, and paging to preserve. | +| [Measured v1 baseline report](product-search-relevance-baseline.json) | Historical model/profile/catalog identities, per-query results and split metrics; not comparable to the v2 dataset. | +| [Product query configuration](../src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs) | Server-selected Semantic/FullText/Hybrid mode and unchanged filter/order fields. | +| [Product query repository](../src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs) | Existing eligibility/filter joins, single-branch retrieval, bounded candidate queries, RRF fusion, mapping, paging and count semantics. | +| [RRF fusion helper](../src/Contoso.Products.Infrastructure/Repositories/ProductSearchFusion.cs) | Fixed `k=60`, per-branch budget, deduplication, exact-SKU override, and deterministic tie-breaking. | +| [Hybrid API tests](../tests/Contoso.Products.Test.Api/ProductHybridSearchTests.cs) | Controlled RRF scores, candidate bounds, filters, exact SKU, paging/count, readiness failures and transaction requirements. | | [Product read service](../src/Contoso.Products.Application/ProductReadService.cs) | Current query delegation through the Application boundary. | | [Product service](../src/Contoso.Products.Application/ProductService.cs) | Atomic product mutations and outbox events. | | [Product contract](../src/Contoso.Products.Contracts/ProductBase.cs) | Current authoritative catalog attributes. | @@ -899,7 +1005,7 @@ When a stage lands: 4. Add runnable setup/backfill/evaluation commands when they exist. Put environment setup in [local-dev.md](local-dev.md) and cross-host wiring in [aspire.md](aspire.md), linking from here instead of duplicating them. 5. Update API examples, schema expectations, MCP descriptions, and observable behavior together. -Keep unresolved choices explicitly unresolved. If the implementation remains keyword-only, this page must continue to say so. Never turn a proposed design into present-tense documentation just because the SQL looks convincing. +Keep unresolved choices explicitly unresolved. Never turn a proposed design into present-tense documentation just because the SQL looks convincing; the bounded Hybrid behavior described here is implemented and covered by controlled tests. ## Primary references diff --git a/samples/docs/testing.md b/samples/docs/testing.md index 34a94b8a6..1c085bf26 100644 --- a/samples/docs/testing.md +++ b/samples/docs/testing.md @@ -300,3 +300,8 @@ Each `*.Test.Api` and `*.Test.Relay` project has a `Resources/` folder containin `Contoso.Test.Aspire` is the automated, CI-friendly counterpart — an NUnit test that self-hosts the whole `Contoso.Aspire` AppHost via `WithAspireTester<...>` (no need for Aspire to already be running) and asserts the same cross-domain flow as a single deterministic pass/fail test. See [aspire.md](aspire.md) for the full Aspire setup, `Contoso.Test.Aspire`, E2E Runner usage, scenario descriptions, load-simulation configuration, and the recommended first-run order. + +The runner's [demo snapshot bootstrap](aspire.md#shared-demo-catalog-bootstrap) is distinct from those event-flow +scenarios: it directly composes the Products accessor and Shopping sync adapter before loading basket data, +then builds real-model search. Its explicit `DemoCatalogBootstrapTests` fixture uses only guarded isolated +PostgreSQL/SQL Server databases; it neither resets Cosmos nor replaces ordinary host-test catalogs. diff --git a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs index a101301e7..0c954e36f 100644 --- a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs +++ b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs @@ -4,10 +4,10 @@ namespace Contoso.Mcp.Tools; public static class CommerceTools { /// Searches products using the Products server's configured ranked retrieval mode. - [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products using the Products server's ranked search mode (semantic by default; optionally English full-text). Returns up to the requested number of summaries, most relevant first. Results are candidates, not guaranteed matches; full-text can return none. Requires a prepared index.")] + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products using the Products server's ranked search mode (semantic by default; optionally English full-text or hybrid). Hybrid fuses up to 100 eligible candidates per branch with Reciprocal Rank Fusion and prioritizes an exact eligible SKU. Returns up to the requested number of summaries; candidates are not guaranteed matches and may come from a bounded pool. Requires every configured index and embedding provider to be ready.")] public static Task SearchProductsAsync( [FromServices] IHttpClientFactory clientFactory, - [Description("Product text, intended use, or a SKU. Semantic mode does not guarantee exact SKU matching; FullText mode prioritizes an exact eligible SKU and otherwise requires lexical matches.")] string searchTerm, + [Description("Product text, intended use, or a SKU. Semantic mode does not guarantee exact SKU matching; FullText and Hybrid prioritize an exact eligible SKU, while Hybrid otherwise combines lexical and semantic ranks.")] string searchTerm, [Description("Maximum number of products to return (1-25).")] int take = 10, CancellationToken cancellationToken = default) { diff --git a/samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj b/samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj index 98eb3555b..b92e84c61 100644 --- a/samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj +++ b/samples/src/Contoso.Products.Database/Contoso.Products.Database.csproj @@ -12,6 +12,7 @@ + diff --git a/samples/src/Contoso.Products.Database/Data/ref-data.seed.yaml b/samples/src/Contoso.Products.Database/Data/ref-data.seed.yaml index 000626e5e..d85ab5779 100644 --- a/samples/src/Contoso.Products.Database/Data/ref-data.seed.yaml +++ b/samples/src/Contoso.Products.Database/Data/ref-data.seed.yaml @@ -28,6 +28,15 @@ products: - { code: LU, text: Lubricant, category_code: A } - { code: LA, text: Labor, category_code: M } - { code: MS, text: Miscellaneous, category_code: M } + - { code: AP, text: Apparel, category_code: G } + - { code: BG, text: Bash guide, category_code: P } + - { code: BR, text: Brakes, category_code: P } + - { code: CG, text: Chain guide, category_code: P } + - { code: FS, text: Front suspension, category_code: P } + - { code: PE, text: Pedals, category_code: P } + - { code: PR, text: Protection, category_code: G } + - { code: RS, text: Rear shock, category_code: P } + - { code: SH, text: Shoes, category_code: G } - $^unit_of_measure: - EA: Each - SET: Set @@ -47,6 +56,9 @@ products: - ONEUP: OneUp Components - MUCOFF: Muc-Off - NONE: None/Not Applicable + - FOX: Fox Factory + - FOXRACING: Fox Racing + - MAXXIS: Maxxis - $^movement_kind: - A: Adjust - I: Issue @@ -54,4 +66,4 @@ products: - $^movement_status: - C: Confirmed - P: Pending - - X: Cancelled \ No newline at end of file + - X: Cancelled diff --git a/samples/src/Contoso.Products.Database/Program.cs b/samples/src/Contoso.Products.Database/Program.cs index 557091666..a228ef323 100644 --- a/samples/src/Contoso.Products.Database/Program.cs +++ b/samples/src/Contoso.Products.Database/Program.cs @@ -35,6 +35,9 @@ public static MigrationArgs ConfigureMigrationArgs(MigrationArgs args) // Only reset data for the 'products' schema. args.DataResetFilterPredicate = ts => ts.Schema == "products"; + // The demo catalog is explicitly opt-in; ordinary database/test setup loads reference data only. + args.DataParserArgs.AddNamed("ref-data.seed.yaml"); + return args; } -} \ No newline at end of file +} diff --git a/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj b/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj index 1b3edfdc3..1213564c2 100644 --- a/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj +++ b/samples/src/Contoso.Products.Infrastructure/Contoso.Products.Infrastructure.csproj @@ -10,6 +10,9 @@ + + + diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs index 0ff6d2ac6..48544d7e7 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryArgsConfig.cs @@ -12,17 +12,26 @@ public ProductQueryArgsConfig() : this(QuerySearchMode.Semantic) { } /// The server-selected full-text configuration; filter fields remain identical. public static ProductQueryArgsConfig FullText { get; } = new(QuerySearchMode.FullText); + /// The server-selected hybrid configuration; filter fields remain identical. + public static ProductQueryArgsConfig Hybrid { get; } = new(QuerySearchMode.Hybrid); + /// Resolves the server-selected search mode without constructing a query configuration. public static QuerySearchMode ResolveMode(IConfiguration configuration) => configuration.ThrowIfNull()["Products:Search:Mode"] switch { null or "" or "Semantic" => QuerySearchMode.Semantic, "FullText" => QuerySearchMode.FullText, - var mode => throw new InvalidOperationException($"Unsupported Products:Search:Mode '{mode}'. Expected Semantic or FullText.") + "Hybrid" => QuerySearchMode.Hybrid, + var mode => throw new InvalidOperationException($"Unsupported Products:Search:Mode '{mode}'. Expected Semantic, FullText or Hybrid.") }; /// Resolves the explicit server setting without changing the public query arguments. - public static ProductQueryArgsConfig FromConfiguration(IConfiguration configuration) => ResolveMode(configuration) == QuerySearchMode.FullText ? FullText : Default; + public static ProductQueryArgsConfig FromConfiguration(IConfiguration configuration) => ResolveMode(configuration) switch + { + QuerySearchMode.FullText => FullText, + QuerySearchMode.Hybrid => Hybrid, + _ => Default + }; private ProductQueryArgsConfig(QuerySearchMode mode) { @@ -46,8 +55,11 @@ private ProductQueryArgsConfig(QuerySearchMode mode) // Ranked search is executed by the repository; ordinary queries retain the configured SKU ordering. WithSearch(search => search .WithMode(mode) - .WithHelpText(mode == QuerySearchMode.FullText - ? "English full-text matching with websearch syntax, weighted SKU/name, category/brand, tags and description; exact canonical SKU first, then cover-density rank. No semantic expansion." - : "Semantic similarity using exact cosine distance over the configured product embedding profile.")); + .WithHelpText(mode switch + { + QuerySearchMode.FullText => "English full-text matching with websearch syntax, weighted SKU/name, category/brand, tags and description; exact canonical SKU first, then cover-density rank. No semantic expansion.", + QuerySearchMode.Hybrid => $"Reciprocal Rank Fusion (RRF) combines English full-text and exact semantic rankings. Each branch contributes at most {ProductSearchFusion.CandidateLimit} eligible candidates (RRF k={ProductSearchFusion.RrfK}); an exact eligible canonical SKU is prioritized. $count reports the deduplicated candidate union (at most {ProductSearchFusion.MaximumCandidateCount}), not all eligible products. Paging is over this stable bounded pool.", + _ => "Semantic similarity using exact cosine distance over the configured product embedding profile." + })); } } diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs index 5da8f76b8..8f7d4497e 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs @@ -22,7 +22,9 @@ public class ProductQueryRepository(ProductsEfDb ef, ProductSearchIndex searchIn public async Task> QueryAsync(QueryArgs? query, PagingArgs? paging, CancellationToken cancellationToken = default) { var parsed = _queryConfig.Parse(query).ThrowOnError(); - var isLexical = _queryConfig.SearchParser.Mode == QuerySearchMode.FullText; + var searchMode = _queryConfig.SearchParser.Mode; + var isLexical = searchMode == QuerySearchMode.FullText; + var isHybrid = searchMode == QuerySearchMode.Hybrid; var products = _ef.Products.Model.Query(); if (query?.IsIncludeInactive is false) products = products.Where(p => !p.IsInactive); @@ -49,9 +51,14 @@ from i in ig.DefaultIfEmpty() if (parsed.IsRankedSearch) { EnsureSnapshotIsolation(); - if (!isLexical) + if (isLexical) + await _lexicalIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + else { await _searchIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + if (isHybrid) + await _lexicalIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + vector = await _searchIndex.EmbedQueryAsync((parsed.SearchResult?.Text).ThrowIfNull(), cancellationToken).ConfigureAwait(false); } } @@ -62,23 +69,28 @@ from i in ig.DefaultIfEmpty() async Task> ReadAsync(CancellationToken cancellationToken) { - if (parsed.IsRankedSearch && isLexical) + if (parsed.IsRankedSearch) { EnsureSnapshotIsolation(); - await _lexicalIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); - q = from candidate in q - join match in _lexicalIndex.Matches((parsed.SearchResult?.Text).ThrowIfNull()) on candidate.Product.Id equals match.Id - orderby match.ExactSku descending, match.Rank descending, candidate.Product.Id - select candidate; - } - else if (vector is not null) - { - EnsureSnapshotIsolation(); - await _searchIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); - q = (from candidate in q - join match in _searchIndex.Matches(vector) on candidate.Product.Id equals match.Id - orderby match.Distance, candidate.Product.Id - select candidate); + if (isLexical || isHybrid) + await _lexicalIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + + if (!isLexical) + await _searchIndex.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + + if (isHybrid) + return await ReadHybridAsync(cancellationToken).ConfigureAwait(false); + + if (isLexical) + q = from candidate in q + join match in _lexicalIndex.Matches((parsed.SearchResult?.Text).ThrowIfNull()) on candidate.Product.Id equals match.Id + orderby match.ExactSku descending, match.Rank descending, candidate.Product.Id + select candidate; + else + q = (from candidate in q + join match in _searchIndex.Matches(vector.ThrowIfNull()) on candidate.Product.Id equals match.Id + orderby match.Distance, candidate.Product.Id + select candidate); } else q = q.OrderBy(parsed); @@ -99,12 +111,76 @@ join match in _searchIndex.Matches(vector) on candidate.Product.Id equals match. QtyOnHand = x.QtyOnHand }, paging, cancellationToken: cancellationToken).ConfigureAwait(false); } + + async Task> ReadHybridAsync(CancellationToken cancellationToken) + { + var searchText = (parsed.SearchResult?.Text).ThrowIfNull(); + var lexical = await ProductSearchFusion.TakeCandidatesAsync( + from candidate in q + join match in _lexicalIndex.Matches(searchText) on candidate.Product.Id equals match.Id + orderby match.ExactSku descending, match.Rank descending, candidate.Product.Id + select new { candidate.Product.Id, match.ExactSku }, + ProductSearchFusion.CandidateLimit, + cancellationToken).ConfigureAwait(false); + + var semantic = await ProductSearchFusion.TakeCandidatesAsync( + from candidate in q + join match in _searchIndex.Matches(vector.ThrowIfNull()) on candidate.Product.Id equals match.Id + orderby match.Distance, candidate.Product.Id + select candidate.Product.Id, + ProductSearchFusion.CandidateLimit, + cancellationToken).ConfigureAwait(false); + + var fused = ProductSearchFusion.Fuse( + lexical.Select(item => new ProductSearchFusion.LexicalCandidate(item.Id, item.ExactSku)).ToArray(), + semantic); + var ids = fused.Select(candidate => candidate.Id).ToArray(); + var mapped = ids.Length == 0 + ? [] + : await q.Where(candidate => ids.Contains(candidate.Product.Id)) + .ToMappedItemsAsync(x => new Contracts.ProductLite + { + Id = x.Product.Id, + Sku = x.Product.Sku, + Text = x.Product.Text, + Description = x.Product.Description, + CategoryCode = x.CategoryCode, + SubCategoryCode = x.SubCategoryCode, + BrandCode = x.Product.BrandCode, + UnitOfMeasureCode = x.Product.UnitOfMeasureCode, + Price = x.Product.Price, + IsInactive = x.Product.IsInactive, + IsNonStocked = x.Product.IsNonStocked, + QtyOnHand = x.QtyOnHand + }, cancellationToken).ConfigureAwait(false); + var byId = mapped.ToDictionary( + product => product.Id ?? throw new InvalidOperationException("A product search candidate must have an identifier."), + StringComparer.Ordinal); + var effectivePaging = paging ?? new PagingArgs(); + IEnumerable pageIds = effectivePaging.IsNone + ? fused.Select(candidate => candidate.Id) + : fused.Skip(effectivePaging.Skip).Take(effectivePaging.Take).Select(candidate => candidate.Id); + var result = new ItemsResult(paging) + { + Items = pageIds.Select(id => byId[id]).ToArray() + }; + + if (paging?.IsCountRequested is true) + ((ITotalCount)result).WithTotalCount(fused.Count); + + return result; + } } /// Rejects a parent transaction that cannot provide a consistent PostgreSQL read snapshot. private void EnsureSnapshotIsolation() { if (_unitOfWork.Database.CurrentTransaction is { } transaction && transaction.IsolationLevel is not (IsolationLevel.RepeatableRead or IsolationLevel.Serializable)) - throw new InvalidOperationException($"{(_queryConfig.SearchParser.Mode == QuerySearchMode.FullText ? "Full-text" : "Semantic")} product search requires a RepeatableRead or Serializable parent transaction."); + throw new InvalidOperationException($"{_queryConfig.SearchParser.Mode switch + { + QuerySearchMode.FullText => "Full-text", + QuerySearchMode.Hybrid => "Hybrid", + _ => "Semantic" + }} product search requires a RepeatableRead or Serializable parent transaction."); } } diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchFusion.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchFusion.cs new file mode 100644 index 000000000..5127ec182 --- /dev/null +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchFusion.cs @@ -0,0 +1,100 @@ +namespace Contoso.Products.Infrastructure.Repositories; + +/// Combines bounded lexical and semantic candidate lists using Reciprocal Rank Fusion. +internal static class ProductSearchFusion +{ + /// The maximum number of candidates retained from each retrieval branch. + internal const int CandidateLimit = 100; + + /// The fixed RRF smoothing constant. + internal const int RrfK = 60; + + /// The maximum size of the deduplicated candidate union. + internal const int MaximumCandidateCount = CandidateLimit * 2; + + /// Materializes the first bounded candidates from an already filtered and deterministically ranked query. + /// The branch query after its eligibility and structured filters. + /// The branch candidate limit. + /// The cancellation token. + /// The bounded candidate list. + internal static Task> TakeCandidatesAsync(IQueryable orderedCandidates, int candidateLimit, CancellationToken cancellationToken) + { + if (candidateLimit < 1) + throw new ArgumentOutOfRangeException(nameof(candidateLimit), "The candidate limit must be greater than zero."); + + return orderedCandidates.Take(candidateLimit).ToListAsync(cancellationToken); + } + + /// Fuses ordered branch candidates, prioritizing an eligible exact SKU before the combined score. + /// Lexical candidates ordered by exact-SKU status, rank and product ID. + /// Semantic product IDs ordered by distance and product ID. + /// The deduplicated candidates in deterministic fused order. + internal static IReadOnlyList Fuse(IReadOnlyList lexical, IReadOnlyList semantic) + { + var candidates = new Dictionary(StringComparer.Ordinal); + for (var i = 0; i < lexical.Count; i++) + { + var item = lexical[i]; + if (!candidates.TryGetValue(item.Id, out var ranks)) + candidates.Add(item.Id, ranks = new CandidateRanks(item.Id)); + + if (ranks.LexicalRank is null) + { + ranks.LexicalRank = i + 1; + ranks.ExactSku = item.ExactSku; + } + } + + for (var i = 0; i < semantic.Count; i++) + { + var id = semantic[i]; + if (!candidates.TryGetValue(id, out var ranks)) + candidates.Add(id, ranks = new CandidateRanks(id)); + + ranks.SemanticRank ??= i + 1; + } + + return [.. candidates.Values + .Select(ranks => new FusedCandidate( + ranks.Id, + ranks.LexicalRank, + ranks.SemanticRank, + ranks.ExactSku, + (ranks.LexicalRank is { } lexicalRank ? 1d / (RrfK + lexicalRank) : 0d) + + (ranks.SemanticRank is { } semanticRank ? 1d / (RrfK + semanticRank) : 0d))) + .OrderByDescending(candidate => candidate.ExactSku) + .ThenByDescending(candidate => candidate.Score) + .ThenBy(candidate => candidate.LexicalRank ?? int.MaxValue) + .ThenBy(candidate => candidate.SemanticRank ?? int.MaxValue) + .ThenBy(candidate => candidate.Id, StringComparer.Ordinal)]; + } + + /// Represents a lexical candidate and whether it is an exact canonical SKU match. + /// The product identifier. + /// Whether the complete query matches the product SKU. + internal sealed record LexicalCandidate(string Id, bool ExactSku); + + /// Represents a fused candidate and its component rank positions. + /// The product identifier. + /// The one-based lexical rank, when present. + /// The one-based semantic rank, when present. + /// Whether the candidate is an eligible exact canonical SKU match. + /// The sum of reciprocal rank contributions. + internal sealed record FusedCandidate(string Id, int? LexicalRank, int? SemanticRank, bool ExactSku, double Score); + + /// Collects the independently ranked positions for one product. + private sealed class CandidateRanks(string id) + { + /// The product identifier. + public string Id { get; } = id; + + /// The lexical rank, if the product appears in that branch. + public int? LexicalRank { get; set; } + + /// The semantic rank, if the product appears in that branch. + public int? SemanticRank { get; set; } + + /// Whether the lexical branch recognized an exact canonical SKU. + public bool ExactSku { get; set; } + } +} diff --git a/samples/tests/Contoso.E2E.Runner/Contoso.E2E.Runner.csproj b/samples/tests/Contoso.E2E.Runner/Contoso.E2E.Runner.csproj index ee3a474cc..20c6931a1 100644 --- a/samples/tests/Contoso.E2E.Runner/Contoso.E2E.Runner.csproj +++ b/samples/tests/Contoso.E2E.Runner/Contoso.E2E.Runner.csproj @@ -1,6 +1,8 @@ + + net10.0 Exe enable enable @@ -10,6 +12,7 @@ + @@ -27,10 +30,15 @@ + + + + + e2e-settings.json PreserveNewest diff --git a/samples/tests/Contoso.E2E.Runner/Data/demo-baskets.seed.yaml b/samples/tests/Contoso.E2E.Runner/Data/demo-baskets.seed.yaml new file mode 100644 index 000000000..f374de824 --- /dev/null +++ b/samples/tests/Contoso.E2E.Runner/Data/demo-baskets.seed.yaml @@ -0,0 +1,24 @@ +Shopping: + - Basket: + - { BasketId: ^3001, CustomerId: ^11, BasketStatusCode: E, SubTotal: 0, Total: 0 } + - { BasketId: ^3002, CustomerId: ^12, BasketStatusCode: C, SubTotal: 9679, DiscountCouponCode: XMAS2025, DiscountAmount: 145.19, Total: 9533.81 } + - { BasketId: ^3003, CustomerId: ^11, BasketStatusCode: A, SubTotal: 9999, Total: 9999 } + - { BasketId: ^3004, CustomerId: ^13, BasketStatusCode: B, SubTotal: 98, Total: 98 } + - { BasketId: ^3005, CustomerId: ^13, BasketStatusCode: A, SubTotal: 98, Total: 98 } + - { BasketId: ^3006, CustomerId: ^13, BasketStatusCode: A, SubTotal: 98, Total: 98 } + - { BasketId: ^3007, CustomerId: ^13, BasketStatusCode: A, SubTotal: 98, Total: 98, ShippingAddressJson: { street1: "123 Main St", city: "Anytown", state: "CA", postCode: "12345" } } + - { BasketId: ^3008, CustomerId: ^13, BasketStatusCode: A, SubTotal: 98, Total: 98, ShippingAddressJson: { street1: "123 Main St", city: "Anytown", state: "CA", postCode: "12345" } } + - BasketItem: + - { BasketItemId: ^4001, BasketId: ^3006, ProductId: ^27, Sku: ONEUP-COMPOSITE-PEDALS, Text: OneUp Composite Pedals, UnitOfMeasureCode: PR, Quantity: 2, UnitPrice: 49 } + - { BasketItemId: ^4002, BasketId: ^3001, ProductId: ^1, Sku: YETI-ASR-C2-2025, Text: Yeti ASR C2, UnitOfMeasureCode: EA, Quantity: 2, UnitPrice: 129 } + - { BasketItemId: ^4003, BasketId: ^3002, ProductId: ^1, Sku: YETI-ASR-C2-2025, Text: Yeti ASR C2, UnitOfMeasureCode: EA, Quantity: 1, UnitPrice: 9850 } + - { BasketItemId: ^4004, BasketId: ^3002, ProductId: ^20, Sku: SHIMANO-SLX-M7100-CASSETTE, Text: Shimano SLX M7100 12-speed Cassette, UnitOfMeasureCode: EA, Quantity: 1, UnitPrice: 99 } + - { BasketItemId: ^4005, BasketId: ^3003, ProductId: ^12, Sku: SPEC-ENDURO-PRO, Text: Specialized Enduro Pro, UnitOfMeasureCode: EA, Quantity: 1, UnitPrice: 9500 } + - { BasketItemId: ^4006, BasketId: ^3003, ProductId: ^21, Sku: SHIMANO-XTR-M9100-CRANK, Text: Shimano XTR M9100 Crankset, UnitOfMeasureCode: EA, Quantity: 1, UnitPrice: 499 } + - { BasketItemId: ^4007, BasketId: ^3004, ProductId: ^27, Sku: ONEUP-COMPOSITE-PEDALS, Text: OneUp Composite Pedals, UnitOfMeasureCode: PR, Quantity: 2, UnitPrice: 49 } + - { BasketItemId: ^4008, BasketId: ^3005, ProductId: ^27, Sku: ONEUP-COMPOSITE-PEDALS, Text: OneUp Composite Pedals, UnitOfMeasureCode: PR, Quantity: 2, UnitPrice: 49 } + - { BasketItemId: ^4009, BasketId: ^3007, ProductId: ^1, Sku: YETI-ASR-C2-2025, Text: Yeti ASR C2, UnitOfMeasureCode: EA, Quantity: 2, UnitPrice: 9850 } + - { BasketItemId: ^4010, BasketId: ^3007, ProductId: ^20, Sku: SHIMANO-SLX-M7100-CASSETTE, Text: Shimano SLX M7100 12-speed Cassette, UnitOfMeasureCode: EA, Quantity: 1, UnitPrice: 99 } + - { BasketItemId: ^4011, BasketId: ^3008, ProductId: ^1, Sku: YETI-ASR-C2-2025, Text: Yeti ASR C2, UnitOfMeasureCode: EA, Quantity: 2, UnitPrice: 9850 } + - { BasketItemId: ^4012, BasketId: ^3008, ProductId: ^20, Sku: SHIMANO-SLX-M7100-CASSETTE, Text: Shimano SLX M7100 12-speed Cassette, UnitOfMeasureCode: EA, Quantity: 1, UnitPrice: 99 } + - { BasketItemId: ^4013, BasketId: ^3007, ProductId: ^32, Sku: LABOR, Text: Labor, UnitOfMeasureCode: HR, Quantity: 1.5, UnitPrice: 80 } diff --git a/samples/tests/Contoso.E2E.Runner/GlobalUsing.cs b/samples/tests/Contoso.E2E.Runner/GlobalUsing.cs index 5a45b36da..af0082c3a 100644 --- a/samples/tests/Contoso.E2E.Runner/GlobalUsing.cs +++ b/samples/tests/Contoso.E2E.Runner/GlobalUsing.cs @@ -1,20 +1,34 @@ +global using Contoso.Customers.Contracts; global using Contoso.E2E.Runner.Infrastructure; global using Contoso.E2E.Runner.Scenarios; global using Contoso.Products.Contracts; +global using Contoso.Products.Infrastructure.Clients.Ollama; +global using Contoso.Products.Infrastructure.Repositories; global using Contoso.Shopping.Contracts; -global using Contoso.Customers.Contracts; +global using Contoso.Shopping.Infrastructure.Repositories; global using CoreEx; +global using CoreEx.Caching; global using CoreEx.Cosmos; global using CoreEx.Cosmos.Provisioning; +global using CoreEx.EntityFrameworkCore; global using CoreEx.Json; +global using CoreEx.RefData; global using DbEx; global using DbEx.Migration; global using DbEx.Postgres.Migration; global using DbEx.SqlServer.Migration; +global using Microsoft.EntityFrameworkCore; global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Options; global using Spectre.Console; +global using StackExchange.Redis; global using System.Collections.Concurrent; global using System.Diagnostics; global using System.Net.Http.Json; global using System.Reflection; global using System.Text; +global using System.Text.Json; +global using ZiggyCreatures.Caching.Fusion; +global using ZiggyCreatures.Caching.Fusion.Backplane.StackExchangeRedis; diff --git a/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogBootstrap.cs b/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogBootstrap.cs new file mode 100644 index 000000000..ba62042c7 --- /dev/null +++ b/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogBootstrap.cs @@ -0,0 +1,254 @@ +namespace Contoso.E2E.Runner.Infrastructure; + +/// Loads the shared demo snapshot and prepares both domains before hosts accept demo traffic. +/// This composition-only bridge uses the same wire contract and sync adapter as event consumption; it does not exercise the broker or outbox. +public static class DemoCatalogBootstrap +{ + /// Resets only Products and Shopping, seeds replicas before baskets, and verifies search readiness. + public static async Task RunAsync(IConfiguration configuration, CancellationToken cancellationToken = default) + { + var productsConnection = RequiredConnection(configuration, "Products"); + var shoppingConnection = RequiredConnection(configuration, "Shopping"); + cancellationToken.ThrowIfCancellationRequested(); + + var productsArgs = Products.Database.Program.ConfigureMigrationArgs(new MigrationArgs(MigrationCommand.ResetAndDatabase, productsConnection)); + productsArgs.DataParserArgs.NamedResources.Clear(); + productsArgs.DataParserArgs.AddNamed("ref-data.seed.yaml", "demo-catalog.seed.yaml"); + + using (var migration = new PostgresMigration(productsArgs)) + await MigrateAsync(migration, cancellationToken).ConfigureAwait(false); + + var shoppingArgs = Shopping.Database.Program.ConfigureMigrationArgs(new MigrationArgs(MigrationCommand.ResetAndDatabase, shoppingConnection)); + using (var migration = new SqlServerMigration(shoppingArgs)) + await MigrateAsync(migration, cancellationToken).ConfigureAwait(false); + + using var products = CreateProductsHost(configuration, productsConnection); + using var shopping = CreateShoppingHost(configuration, shoppingConnection); + var count = await SynchronizeAsync(products, shopping, cancellationToken).ConfigureAwait(false); + + var basketArgs = Shopping.Database.Program.ConfigureMigrationArgs(new MigrationArgs(MigrationCommand.Data, shoppingConnection)); + basketArgs.AddAssembly().DataParserArgs.AddNamed("demo-baskets.seed.yaml"); + using (var migration = new SqlServerMigration(basketArgs)) + await MigrateAsync(migration, cancellationToken).ConfigureAwait(false); + + using (var cache = CreateServingCacheHost(configuration, "Products")) + await InvalidateReferenceDataAsync(cache, [typeof(BrandCollection), typeof(CategoryCollection), typeof(SubCategoryCollection), + typeof(Products.Contracts.UnitOfMeasureCollection), typeof(MovementKindCollection), typeof(MovementStatusCollection)], cancellationToken).ConfigureAwait(false); + + using (var cache = CreateServingCacheHost(configuration, "Shopping")) + await InvalidateReferenceDataAsync(cache, [typeof(Shopping.Contracts.UnitOfMeasureCollection), + typeof(Shopping.Contracts.BasketStatusCollection), typeof(Shopping.Contracts.DiscountCouponCollection)], cancellationToken).ConfigureAwait(false); + + await VerifyAsync(products, shopping, cancellationToken).ConfigureAwait(false); + await PrepareSearchAsync(products, cancellationToken).ConfigureAwait(false); + return count; + } + + /// Creates an isolated Products composition using the real configured embedding client and fresh local reference-data cache. + public static IHost CreateProductsHost(IConfiguration configuration, string connectionString) + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.Sources.Clear(); + builder.Configuration.AddConfiguration(configuration); + builder.AddProductEmbeddings(); + builder.Services.AddSingleton(_ => Npgsql.NpgsqlDataSource.Create(connectionString)); + + builder.Services.AddExecutionContext().AddMemoryCache().AddMemoryOnlyHybridCache().AddDefaultCacheKeyProvider() + .AddReferenceDataOrchestrator() + .AddScoped() + .AddScoped() + .AddScoped() + .AddPostgresDatabase().AddPostgresUnitOfWork().AddDbContext().AddEfDb() + .AddScoped().AddScoped(); + + return builder.Build(); + } + + /// Creates a separate Shopping scope; Products reference data is never registered in this domain. + public static IHost CreateShoppingHost(IConfiguration configuration, string connectionString) + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.Sources.Clear(); + builder.Configuration.AddConfiguration(configuration); + + builder.Services.AddExecutionContext().AddMemoryCache().AddMemoryOnlyHybridCache().AddDefaultCacheKeyProvider() + .AddReferenceDataOrchestrator() + .AddScoped() + .AddScoped() + .AddScoped(_ => new Microsoft.Data.SqlClient.SqlConnection(connectionString)) + .AddSqlServerDatabase().AddDbContext().AddEfDb() + .AddScoped(); + + return builder.Build(); + } + + /// Uses the serving hosts' existing Redis/FusionCache integration for targeted reference-data eviction, never a database flush. + public static IHost CreateServingCacheHost(IConfiguration configuration, string domainName) + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.Sources.Clear(); + builder.Configuration.AddConfiguration(configuration); + builder.AddHostSettings(solutionName: "Contoso", domainName: domainName); + builder.AddRedisDistributedCache("redis"); + builder.Services.AddExecutionContext().AddMemoryCache().AddDefaultCacheKeyProvider().AddFusionHybridCache(); + + builder.Services.AddFusionCache() + .WithRegisteredMemoryCache() + .WithRegisteredDistributedCache() + .WithBackplane(sp => new RedisBackplane(new RedisBackplaneOptions { Configuration = sp.GetRequiredService>().Value.ToString() })) + .WithSystemTextJsonSerializer(JsonDefaults.SerializerOptions) + .WithDefaultEntryOptions(options => + { + options.AllowBackgroundDistributedCacheOperations = false; + options.AllowBackgroundBackplaneOperations = false; + options.ReThrowDistributedCacheExceptions = true; + options.ReThrowBackplaneExceptions = true; + }); + + builder.Services.AddScoped(); + return builder.Build(); + } + + /// Reconciles actual non-deleted source rows through Shopping's normal upsert/delete adapter. + public static async Task SynchronizeAsync(IHost products, IHost shopping, CancellationToken cancellationToken = default) + { + var snapshot = await ReadSnapshotAsync(products, cancellationToken).ConfigureAwait(false); + return await InScopeAsync(shopping, async services => + { + var ef = services.GetRequiredService(); + var adapter = services.GetRequiredService(); + var ids = snapshot.Select(p => p.Id).ToHashSet(StringComparer.Ordinal); + var existing = await ef.Products.Query().AsNoTracking().Select(p => p.Id).ToListAsync(cancellationToken).ConfigureAwait(false); + + foreach (var id in existing.Where(id => !ids.Contains(id))) + (await adapter.DeleteAsync(id, cancellationToken).ConfigureAwait(false)).ThrowOnError(); + + foreach (var product in snapshot) + (await adapter.ModifyAsync(product, cancellationToken).ConfigureAwait(false)).ThrowOnError(); + return snapshot.Count; + }).ConfigureAwait(false); + } + + /// Verifies exact replica fields, no stale rows, local reference data, and basket product integrity. + public static async Task VerifyAsync(IHost products, IHost shopping, CancellationToken cancellationToken = default) + { + var snapshot = await ReadSnapshotAsync(products, cancellationToken).ConfigureAwait(false); + await InScopeAsync(shopping, async services => + { + var ef = services.GetRequiredService(); + var replicas = await ef.Products.Query().AsNoTracking().OrderBy(p => p.Id).ToListAsync(cancellationToken).ConfigureAwait(false); + var expected = snapshot.OrderBy(p => p.Id).Select(p => (p.Id.ThrowIfNullOrEmpty(), p.Sku.ThrowIfNullOrEmpty(), p.Text.ThrowIfNullOrEmpty(), p.UnitOfMeasureCode.ThrowIfNullOrEmpty(), p.Price, p.IsInactive, p.IsNonStocked)); + var actual = replicas.Select(p => (p.Id, p.Sku, p.Text, p.UnitOfMeasureCode, p.Price, p.IsInactive, p.IsNonStocked)); + if (!expected.SequenceEqual(actual)) + throw new InvalidOperationException("Demo Products/Shopping replica mismatch (including stale or deleted products)."); + + var units = await ef.UnitsOfMeasure.Query().AsNoTracking().Select(u => u.Code).ToListAsync(cancellationToken).ConfigureAwait(false); + if (replicas.Any(p => !units.Contains(p.UnitOfMeasureCode))) + throw new InvalidOperationException("Demo Shopping unit-of-measure reference data is missing."); + + var ids = replicas.Select(p => p.Id).ToHashSet(StringComparer.Ordinal); + var items = await ef.BasketItems.Query().AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false); + var baskets = await ef.Baskets.Query().AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false); + if (baskets.Count == 0 || items.Count == 0) + throw new InvalidOperationException("Demo basket seed was not loaded."); + + var basketIds = baskets.Select(b => b.Id).ToHashSet(StringComparer.Ordinal); + if (items.Any(i => !ids.Contains(i.ProductId) || !basketIds.Contains(i.BasketId))) + throw new InvalidOperationException("Demo basket item references a missing product or basket."); + + var statuses = await ef.BasketStatuses.Query().Select(s => s.Code).ToListAsync(cancellationToken).ConfigureAwait(false); + var coupons = await ef.DiscountCoupons.Query().Select(c => c.Code).ToListAsync(cancellationToken).ConfigureAwait(false); + if (items.Any(i => !units.Contains(i.UnitOfMeasureCode)) + || baskets.Any(b => !statuses.Contains(b.BasketStatusCode) || b.DiscountCouponCode is not null && !coupons.Contains(b.DiscountCouponCode))) + throw new InvalidOperationException("Demo Shopping basket reference data is missing."); + + return true; + }).ConfigureAwait(false); + } + + /// Builds lexical and semantic projections with the configured real provider; failures are not swallowed or replaced by test vectors. + public static Task PrepareSearchAsync(IHost products, CancellationToken cancellationToken = default) + => InScopeAsync(products, async services => + { + var profile = services.GetRequiredService().Profile; + try + { + var semantic = services.GetRequiredService(); + var count = await semantic.RebuildAsync(cancellationToken).ConfigureAwait(false); + await semantic.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + await services.GetRequiredService().EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + await semantic.EmbedQueryAsync("mountain bike", cancellationToken).ConfigureAwait(false); + + var expected = await services.GetRequiredService().Products.Model.Query().CountAsync(cancellationToken).ConfigureAwait(false); + if (count != expected) + throw new InvalidOperationException($"Demo search projection is incomplete: expected {expected}, indexed {count}."); + + return count; + } + catch (Exception ex) when (ex is not OperationCanceledException && !cancellationToken.IsCancellationRequested) + { + throw new InvalidOperationException($"Demo lexical/semantic search preparation failed for embedding profile '{profile}'. Verify the configured provider/model and rerun setup.", ex); + } + }); + + private static Task> ReadSnapshotAsync(IHost products, CancellationToken cancellationToken) + => InScopeAsync(products, async services => + { + await services.GetRequiredService() + .PrefetchAsync([typeof(Brand), typeof(Category), typeof(SubCategory), typeof(Products.Contracts.UnitOfMeasure)], cancellationToken).ConfigureAwait(false); + + var ef = services.GetRequiredService(); + var source = await ef.Products.Model.Query().AsNoTracking().OrderBy(p => p.Id) + .ToMappedItemsAsync(Products.Infrastructure.Mapping.ProductMapper.From, cancellationToken).ConfigureAwait(false); + + var brands = await ef.Brands.Query().Select(b => b.Code).ToListAsync(cancellationToken).ConfigureAwait(false); + var categories = await ef.Categories.Query().Select(c => c.Code).ToListAsync(cancellationToken).ConfigureAwait(false); + var subcategories = await ef.SubCategories.Query().Select(s => s.Code).ToListAsync(cancellationToken).ConfigureAwait(false); + var units = await ef.UnitsOfMeasure.Query().Select(u => u.Code).ToListAsync(cancellationToken).ConfigureAwait(false); + + if (source.Count == 0 || source.Any(p => p.BrandCode is null || !brands.Contains(p.BrandCode) + || p.CategoryCode is null || !categories.Contains(p.CategoryCode) + || p.SubCategoryCode is null || !subcategories.Contains(p.SubCategoryCode) + || p.UnitOfMeasureCode is null || !units.Contains(p.UnitOfMeasureCode))) + throw new InvalidOperationException("Demo Products catalog is empty or reference data is missing."); + + // Match event consumption's JSON contract boundary instead of maintaining another field mapper. + return source.Select(p => JsonSerializer.Deserialize( + JsonSerializer.Serialize(p, JsonDefaults.SerializerOptions), JsonDefaults.SerializerOptions) + ?? throw new InvalidOperationException("Demo product snapshot deserialization failed.")).ToList(); + }); + + private static async Task InScopeAsync(IHost host, Func> action) + { + await using var scope = host.Services.CreateAsyncScope(); + scope.ServiceProvider.GetRequiredService(); + try + { + return await action(scope.ServiceProvider).ConfigureAwait(false); + } + finally + { + CoreEx.ExecutionContext.Reset(); + } + } + + private static string RequiredConnection(IConfiguration configuration, string domain) + => configuration[$"E2E:{domain}:ConnectionString"].ThrowIfNullOrEmpty($"E2E:{domain}:ConnectionString"); + + private static Task InvalidateReferenceDataAsync(IHost host, Type[] collections, CancellationToken cancellationToken) + => InScopeAsync(host, async services => + { + var cache = services.GetRequiredService(); + foreach (var type in collections) + await cache.RemoveAsync(type, cancellationToken).ConfigureAwait(false); + return true; + }); + + private static async Task MigrateAsync(DatabaseMigrationBase migration, CancellationToken cancellationToken) + { + var (success, output) = await migration.MigrateAndLogAsync(cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + if (!success) + throw new InvalidOperationException("Demo database migration failed:" + Environment.NewLine + output); + } +} diff --git a/samples/tests/Contoso.E2E.Runner/Program.cs b/samples/tests/Contoso.E2E.Runner/Program.cs index f12e861ae..c23e1db00 100644 --- a/samples/tests/Contoso.E2E.Runner/Program.cs +++ b/samples/tests/Contoso.E2E.Runner/Program.cs @@ -3,11 +3,21 @@ // Build configuration var config = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: false) + .SetBasePath(AppContext.BaseDirectory) + .AddJsonFile("products-settings.json", optional: false) + .AddJsonFile("e2e-settings.json", optional: false) .AddEnvironmentVariables() .Build(); +if (args.Contains("--demo-catalog-only", StringComparer.Ordinal)) +{ + if (!args.Contains("--accept-demo-reset", StringComparer.Ordinal)) + throw new InvalidOperationException("Demo setup resets the configured Products and Shopping schemas. Stop their writers and explicitly pass --accept-demo-reset after reviewing E2E connection strings."); + var count = await DemoCatalogBootstrap.RunAsync(config).ConfigureAwait(false); + Console.WriteLine($"Demo bootstrap complete: {count} products replicated; basket integrity and lexical/semantic search readiness verified."); + return; +} + var context = new TestContext(config); (bool ProductApiOk, bool ShoppingApiOk, bool CustomersApiOk, bool OrdersApiOk) status = (false, false, false, false); diff --git a/samples/tests/Contoso.E2E.Runner/Scenarios/DatabaseMigrationSetup.cs b/samples/tests/Contoso.E2E.Runner/Scenarios/DatabaseMigrationSetup.cs index 88c84b992..a9c0e5fd6 100644 --- a/samples/tests/Contoso.E2E.Runner/Scenarios/DatabaseMigrationSetup.cs +++ b/samples/tests/Contoso.E2E.Runner/Scenarios/DatabaseMigrationSetup.cs @@ -9,33 +9,8 @@ public sealed class DatabaseMigrationSetup : IScenario /// public async Task RunAsync(ScenarioContext context) { - // Step 1: Products database migration. - await context.StepAsync("Products database migration.", async () => - { - var cs = context.TestContext.Config.GetValue("E2E:Products:ConnectionString") ?? throw new InvalidOperationException("E2E:Products:ConnectionString configuration value is missing."); - var ma = new MigrationArgs(MigrationCommand.ResetAndDatabase, cs); - Contoso.Products.Database.Program.ConfigureMigrationArgs(ma); - ma.AddAssembly().DataParserArgs.AddNamed("mutate-data.seed.yaml"); - - using var m = new PostgresMigration(ma); - var (Success, Output) = await m.MigrateAndLogAsync().ConfigureAwait(false); - if (!Success) - throw new Exception("Database migration failed:" + Environment.NewLine + Output); - }, "Successfully migrated; base data refreshed.").ConfigureAwait(false); - - // Step 2: Shopping database migration. - await context.StepAsync("Shopping database migration.", async () => - { - var cs = context.TestContext.Config.GetValue("E2E:Shopping:ConnectionString") ?? throw new InvalidOperationException("E2E:Shopping:ConnectionString configuration value is missing."); - var ma = new MigrationArgs(MigrationCommand.ResetAndDatabase, cs); - Contoso.Shopping.Database.Program.ConfigureMigrationArgs(ma); - ma.AddAssembly().DataParserArgs.AddNamed("mutate-data.seed.yaml"); - - using var m = new SqlServerMigration(ma); - var (Success, Output) = await m.MigrateAndLogAsync().ConfigureAwait(false); - if (!Success) - throw new Exception("Database migration failed:" + Environment.NewLine + Output); - }, "Successfully migrated; base data refreshed.").ConfigureAwait(false); + await context.StepAsync("Products and Shopping demo catalog bootstrap.", () => DemoCatalogBootstrap.RunAsync(context.TestContext.Config), + count => $"{count} non-deleted products replicated; baskets and search projections verified.").ConfigureAwait(false); // Step 3: Customers database migration. await context.StepAsync("Customers database migration.", async () => diff --git a/samples/tests/Contoso.Products.Test.Api/GlobalUsing.cs b/samples/tests/Contoso.Products.Test.Api/GlobalUsing.cs index 9fbbabb09..229debcfc 100644 --- a/samples/tests/Contoso.Products.Test.Api/GlobalUsing.cs +++ b/samples/tests/Contoso.Products.Test.Api/GlobalUsing.cs @@ -1,6 +1,7 @@ global using AwesomeAssertions; global using Contoso.Products.Application.Adapters.Embeddings; global using Contoso.Products.Application.Repositories; +global using Contoso.Products.Application.Search; global using Contoso.Products.Contracts; global using Contoso.Products.Infrastructure.Clients.Ollama; global using Contoso.Products.Infrastructure.Repositories; diff --git a/samples/tests/Contoso.Products.Test.Api/ProductHybridSearchTests.cs b/samples/tests/Contoso.Products.Test.Api/ProductHybridSearchTests.cs new file mode 100644 index 000000000..036c78629 --- /dev/null +++ b/samples/tests/Contoso.Products.Test.Api/ProductHybridSearchTests.cs @@ -0,0 +1,254 @@ +namespace Contoso.Products.Test.Api; + +[NonParallelizable] +public class ProductHybridSearchTests : WithApiTester +{ + private readonly TestEmbeddingAdapter _embeddings = new(); + + [OneTimeSetUp] + public async Task OneTimeSetUpAsync() + { + Test.UseAdditionalConfiguration("Products:Search:Mode", "Hybrid"); + Test.ReplaceScoped(_ => _embeddings); + await Test.MigratePostgresDataAsync(["read-data.seed.yaml"], DbMigration.ConfigureMigrationArgs).ConfigureAwait(false); + await Test.ClearFusionCacheAsync().ConfigureAwait(false); + RebuildSearch(); + } + + [Test] + public void ReciprocalRankFusion_UsesOneBasedRanksAndRewardsSharedCandidates() + { + var fused = ProductSearchFusion.Fuse( + [new("shared", false), new("lexical-only", false)], + ["shared", "semantic-only"]); + + fused[0].Id.Should().Be("shared"); + fused[0].LexicalRank.Should().Be(1); + fused[0].SemanticRank.Should().Be(1); + fused[0].Score.Should().Be(2d / 61); + fused.Single(candidate => candidate.Id == "lexical-only").Score.Should().Be(1d / 62); + fused.Single(candidate => candidate.Id == "semantic-only").Score.Should().Be(1d / 62); + fused.Select(candidate => candidate.Id).Should().OnlyHaveUniqueItems(); + } + + [Test] + public void ReciprocalRankFusion_PrioritizesExactSkuAndKeepsBoundedBranchCandidates() + { + var lexical = Enumerable.Range(0, ProductSearchFusion.CandidateLimit) + .Select(i => new ProductSearchFusion.LexicalCandidate($"L{i:D3}", i == 0)) + .ToArray(); + var semantic = Enumerable.Range(0, ProductSearchFusion.CandidateLimit) + .Select(i => $"S{i:D3}") + .ToArray(); + var fused = ProductSearchFusion.Fuse(lexical, semantic); + + fused.Should().HaveCount(ProductSearchFusion.MaximumCandidateCount); + fused[0].Id.Should().Be("L000"); + fused[0].ExactSku.Should().BeTrue(); + fused.Select(candidate => candidate.Id).Should().OnlyHaveUniqueItems(); + } + + [Test] + public void ReciprocalRankFusion_HandlesEmptyAndDuplicateBranchCandidates() + { + ProductSearchFusion.Fuse([], []).Should().BeEmpty(); + var fused = ProductSearchFusion.Fuse( + [new("same", false), new("same", false)], + ["same", "same", "other"]); + + fused.Should().HaveCount(2); + fused[0].Id.Should().Be("same"); + fused[0].LexicalRank.Should().Be(1); + fused[0].SemanticRank.Should().Be(1); + ProductSearchFusion.Fuse( + [new("same", false), new("same", false)], + ["same", "same", "other"]).Select(candidate => candidate.Id) + .Should().Equal(fused.Select(candidate => candidate.Id)); + } + + [Test] + public void Product_HybridSearch_ExactSkuPriorityFiltersAndPagingAreConsistent() + { + var exact = Search("SHIMANO-XT-M8100-RD", "&$take=5&$count=true"); + exact.Should().NotBeEmpty(); + exact[0].Sku.Should().Be("SHIMANO-XT-M8100-RD"); + exact.Select(product => product.Id).Should().OnlyHaveUniqueItems(); + + Search("SHIMANO-XT-M8100-RD", "&$filter=price lt 0&$count=true").Should().BeEmpty(); + Search("derailleur", "&$filter=brand eq 'SRAM' and price lt 160&$count=true") + .Should().ContainSingle().Which.Sku.Should().Be("SRAM-GX-EAGLE-RD"); + + Test.Scoped(test => test.Run(async _ => + { + var repository = test.Services.GetRequiredService(); + var query = new QueryArgs { Search = "a component for changing gears" }; + var all = await repository.QueryAsync(query, PagingArgs.CreateWithCount(take: ProductSearchFusion.MaximumCandidateCount)).ConfigureAwait(false); + var firstPage = await repository.QueryAsync(query, PagingArgs.CreateWithCount(take: 3)).ConfigureAwait(false); + var secondPage = await repository.QueryAsync(query, PagingArgs.Create(skip: 3, take: 3)).ConfigureAwait(false); + + all.Paging!.TotalCount.Should().Be(firstPage.Paging!.TotalCount); + all.Paging.TotalCount.Should().BeLessThanOrEqualTo(ProductSearchFusion.MaximumCandidateCount); + var allItems = all.Items!.ToArray(); + firstPage.Items!.Select(product => product.Id).Should().Equal(allItems.Take(3).Select(product => product.Id)); + secondPage.Items!.Select(product => product.Id).Should().Equal(allItems.Skip(3).Take(3).Select(product => product.Id)); + }).AssertSuccess()); + } + + [Test] + public void Product_HybridSearch_EligibilityAndStructuredFiltersApplyBeforeCandidateLimit() + => Test.Scoped(test => test.Run(async _ => + { + var ef = test.Services.GetRequiredService(); + var unfiltered = await ProductSearchFusion.TakeCandidatesAsync( + ef.Products.Model.Query().OrderBy(product => product.Id).Select(product => product.Id), + 2, + CancellationToken.None).ConfigureAwait(false); + unfiltered.Should().HaveCount(2); + + var filtered = await ProductSearchFusion.TakeCandidatesAsync( + ef.Products.Model.Query() + .Where(product => !product.IsInactive && product.BrandCode == "SRAM" && product.Price < 160) + .OrderBy(product => product.Id) + .Select(product => product.Sku), + 2, + CancellationToken.None).ConfigureAwait(false); + filtered.Should().ContainSingle().Which.Should().Be("SRAM-GX-EAGLE-RD"); + }).AssertSuccess()); + + [Test] + public void Product_HybridSearch_InactiveAndDeletedEligibilityIsLive() + { + Search("SPEC-EPIC-8-PRO").Should().NotContain(product => product.Sku == "SPEC-EPIC-8-PRO"); + Search("SPEC-EPIC-8-PRO", "&$inactive").Should().Contain(product => product.Sku == "SPEC-EPIC-8-PRO"); + try + { + ExecuteSearchSql("UPDATE products.product SET is_deleted = true WHERE sku = 'MUCOFF-NANO-CLEANER'"); + Search("MUCOFF-NANO-CLEANER", "&$filter=sku eq 'MUCOFF-NANO-CLEANER'").Should().BeEmpty(); + } + finally + { + ExecuteSearchSql("UPDATE products.product SET is_deleted = false WHERE sku = 'MUCOFF-NANO-CLEANER'"); + } + } + + [Test] + public void Product_HybridSearch_RequiresBothReadyBranchesWithoutFallback() + { + try + { + ExecuteSearchSql("DELETE FROM products.product_lexical_search_readiness"); + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.ServiceUnavailable); + + RebuildSearch(); + ExecuteSearchSql("DELETE FROM products.product_search_readiness WHERE profile = 'tests:controlled-vectors:768'"); + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.ServiceUnavailable); + } + finally + { + RebuildSearch(); + } + } + + [Test] + public void Product_HybridSearch_EmbeddingFailureDoesNotReturnLexicalResults() + { + _embeddings.FailQuery = true; + try + { + Test.Http().Run(HttpMethod.Get, "/api/products?$search=cleaner").Assert(HttpStatusCode.ServiceUnavailable); + _embeddings.QueryCallCount.Should().BeGreaterThan(0); + } + finally + { + _embeddings.FailQuery = false; + } + } + + [Test] + public void Product_HybridSearch_CancellationDuringEmbeddingPropagates() + => Test.Scoped(test => test.Run(async _ => + { + var queryStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _embeddings.BeforeQuery = async cancellationToken => + { + queryStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); + }; + + try + { + using var cancellation = new CancellationTokenSource(); + var repository = test.Services.GetRequiredService(); + var pending = repository.QueryAsync(new QueryArgs { Search = "cleaner" }, null, cancellation.Token); + await queryStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); + cancellation.Cancel(); + Func wait = async () => await pending.ConfigureAwait(false); + await wait.Should().ThrowAsync().ConfigureAwait(false); + } + finally + { + _embeddings.BeforeQuery = null; + } + }).AssertSuccess()); + + [Test] + public void Product_HybridSearch_EnlistsCompatibleParentAndRejectsWeakParent() + { + Test.Scoped(test => test.Run(async _ => + { + var unitOfWork = test.Services.GetRequiredService(); + var repository = test.Services.GetRequiredService(); + await unitOfWork.TransactionAsync(new PostgresDatabaseArgs { IsolationLevel = IsolationLevel.RepeatableRead }, async token => + { + var parent = unitOfWork.Database.CurrentTransaction; + var result = await repository.QueryAsync(new QueryArgs { Search = "changing gears" }, PagingArgs.CreateWithCount(take: 1), token).ConfigureAwait(false); + result.Items.Should().ContainSingle(); + result.Paging!.TotalCount.Should().BeGreaterThan(0); + unitOfWork.Database.CurrentTransaction.Should().BeSameAs(parent); + }).ConfigureAwait(false); + + await unitOfWork.TransactionAsync(async token => + { + var calls = _embeddings.QueryCallCount; + Func search = () => repository.QueryAsync(new QueryArgs { Search = "cleaner" }, null, token); + await search.Should().ThrowAsync() + .WithMessage("Hybrid product search requires a RepeatableRead or Serializable parent transaction.") + .ConfigureAwait(false); + _embeddings.QueryCallCount.Should().Be(calls); + }).ConfigureAwait(false); + }).AssertSuccess()); + } + + [Test] + public void Product_HybridSearch_SchemaGraphQLAndOrdinaryQueryDescribeSelectedBehavior() + { + var schema = Test.Http().Run(HttpMethod.Get, "/api/products/$query").AssertOK().Value; + schema.GetProperty("search").GetProperty("mode").GetString().Should().Be("hybrid"); + schema.GetProperty("search").GetProperty("description").GetString().Should().Contain("100"); + schema.GetProperty("search").GetProperty("description").GetString().Should().Contain("deduplicated candidate union"); + + var graphql = Test.Http() + .Run(HttpMethod.Post, "/api/query", new { query = "{ products(search: \"changing gears\", first: 1) { edges { node { sku } } } }" }) + .AssertOK().Value; + graphql.TryGetProperty("errors", out _).Should().BeFalse(); + graphql.GetProperty("data").GetProperty("products").GetProperty("edges").GetArrayLength().Should().Be(1); + + Test.Http() + .Run(HttpMethod.Get, "/api/products?$search=%20%20&$take=50") + .AssertOK().Value.Should().HaveCount(29); + } + + private ProductLite[] Search(string query, string suffix = "") + => Test.Http() + .Run(HttpMethod.Get, $"/api/products?$search={Uri.EscapeDataString(query)}&$take=50{suffix}") + .AssertOK().Value!; + + private void RebuildSearch() => Test.Scoped(test => test.Run(async _ => + { + var count = await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false); + count.Should().Be(31); + }).AssertSuccess()); + + private void ExecuteSearchSql(string sql) => Test.Scoped(test => test.Run(async _ => + await test.Services.GetRequiredService().Database.ExecuteSqlRawAsync(sql).ConfigureAwait(false)).AssertSuccess()); +} diff --git a/samples/tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs b/samples/tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs index 6b319e308..6995b4e41 100644 --- a/samples/tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs +++ b/samples/tests/Contoso.Products.Test.Api/ProductLexicalSearchTests.cs @@ -110,9 +110,9 @@ public void InvalidConfiguredMode_HasNoSilentFallback() var configuration = Test.Configuration; try { - configuration["Products:Search:Mode"] = "Hybrid"; + configuration["Products:Search:Mode"] = "Vector"; Action resolve = () => ProductQueryArgsConfig.FromConfiguration(configuration); - resolve.Should().Throw().WithMessage("Unsupported Products:Search:Mode 'Hybrid'. Expected Semantic or FullText."); + resolve.Should().Throw().WithMessage("Unsupported Products:Search:Mode 'Vector'. Expected Semantic, FullText or Hybrid."); } finally { diff --git a/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs b/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs index 90b82e5c7..ace4559c5 100644 --- a/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs +++ b/samples/tests/Contoso.Products.Test.Api/ProductSearchRelevanceSetupTests.cs @@ -3,6 +3,8 @@ namespace Contoso.Products.Test.Api; [TestFixture, Explicit("Builds and verifies the selected real search branch in the isolated contoso_search_eval database.")] public class ProductSearchRelevanceSetupTests : WithApiTester { + private int _expectedEligibleCount; + [OneTimeSetUp] public async Task OneTimeSetUpAsync() { @@ -15,28 +17,69 @@ public async Task OneTimeSetUpAsync() Test.Scoped(test => test.Run(async _ => { - var lexical = Environment.GetEnvironmentVariable("Products__Search__Mode") == "FullText"; - var count = lexical - ? await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false) - : await test.Services.GetRequiredService().RebuildAsync().ConfigureAwait(false); - count.Should().Be(31); + var mode = Environment.GetEnvironmentVariable("Products__Search__Mode"); + var lexicalOnly = mode == "FullText"; + var hybrid = mode == "Hybrid"; + var database = test.Services.GetRequiredService().Database; + _expectedEligibleCount = await database + .SqlQuery($"SELECT COUNT(*)::integer AS \"Value\" FROM products.product WHERE is_inactive = false AND is_deleted = false AND sku <> 'LABOR'") + .SingleAsync().ConfigureAwait(false); + var expectedIndexedCount = await database + .SqlQuery($"SELECT COUNT(*)::integer AS \"Value\" FROM products.product WHERE is_deleted = false") + .SingleAsync().ConfigureAwait(false); + var semanticIndex = test.Services.GetRequiredService(); + var lexicalIndex = test.Services.GetRequiredService(); + var count = lexicalOnly + ? await lexicalIndex.RebuildAsync().ConfigureAwait(false) + : await semanticIndex.RebuildAsync().ConfigureAwait(false); + count.Should().Be(expectedIndexedCount); + + if (!lexicalOnly) + { + await semanticIndex.EnsureReadyAsync().ConfigureAwait(false); + var profile = test.Services.GetRequiredService().Profile; + var recipe = ProductSearchDocument.TemplateVersion; + var indexed = await test.Services.GetRequiredService().Database + .SqlQuery($"SELECT COUNT(*)::integer AS \"Value\" FROM products.product_search WHERE profile = {profile} AND recipe = {recipe}") + .SingleAsync().ConfigureAwait(false); + indexed.Should().Be(expectedIndexedCount); + } + + if (lexicalOnly || hybrid) + { + await lexicalIndex.EnsureReadyAsync().ConfigureAwait(false); + var indexed = await test.Services.GetRequiredService().Database + .SqlQuery($"SELECT COUNT(*)::integer AS \"Value\" FROM products.product_lexical_search WHERE recipe = {ProductLexicalSearchIndex.Recipe}") + .SingleAsync().ConfigureAwait(false); + indexed.Should().Be(expectedIndexedCount); + } }).AssertSuccess()); } [Test] public void Product_Search_RealModelProfileIsReady() { - var lexical = Environment.GetEnvironmentVariable("Products__Search__Mode") == "FullText"; + var mode = Environment.GetEnvironmentVariable("Products__Search__Mode"); + var lexical = mode == "FullText"; + var hybrid = mode == "Hybrid"; var response = Test.Http() .Run(HttpMethod.Get, lexical ? "/api/products?$search=SHIMANO-XT-M8100-RD&$take=1&$count=true" - : "/api/products?$search=something to clean mud off my bike&$take=1&$count=true") + : hybrid + ? "/api/products?$search=SHIMANO-XT-M8100-RD&$take=1&$count=true" + : "/api/products?$search=something to clean mud off my bike&$filter=sku ne 'LABOR'&$take=1&$count=true") .AssertOK(); if (lexical) response.Value.Should().ContainSingle().Which.Sku.Should().Be("SHIMANO-XT-M8100-RD"); + else if (hybrid) + { + response.Value.Should().ContainSingle().Which.Sku.Should().Be("SHIMANO-XT-M8100-RD"); + var candidateCount = int.Parse(response.Response.Headers.GetValues("X-Paging-Total-Count").Single()); + candidateCount.Should().BeLessThanOrEqualTo(ProductSearchFusion.MaximumCandidateCount); + } else response.Response.Headers.Should().ContainKey("X-Paging-Total-Count") - .WhoseValue.Should().ContainSingle().Which.Should().Be("29"); + .WhoseValue.Should().ContainSingle().Which.Should().Be(_expectedEligibleCount.ToString()); } } diff --git a/samples/tests/Contoso.Products.Test.Api/TestEmbeddingAdapter.cs b/samples/tests/Contoso.Products.Test.Api/TestEmbeddingAdapter.cs index 572d027b0..3241d9019 100644 --- a/samples/tests/Contoso.Products.Test.Api/TestEmbeddingAdapter.cs +++ b/samples/tests/Contoso.Products.Test.Api/TestEmbeddingAdapter.cs @@ -10,6 +10,7 @@ internal sealed class TestEmbeddingAdapter : IProductEmbeddingAdapter public float[]? QueryVector { get; set; } public bool FailQuery { get; set; } public Func? BeforeDocument { get; set; } + public Func? BeforeQuery { get; set; } public async Task> EmbedDocumentAsync(string text, CancellationToken ct = default) { @@ -19,12 +20,17 @@ public async Task> EmbedDocumentAsync(string text, CancellationT throw new HttpRequestException("Controlled embedding failure."); return await EmbedAsync(text, ct).ConfigureAwait(false); } - public Task> EmbedQueryAsync(string text, CancellationToken ct = default) + public async Task> EmbedQueryAsync(string text, CancellationToken ct = default) { QueryCallCount++; + if (BeforeQuery is not null) + await BeforeQuery(ct).ConfigureAwait(false); + ct.ThrowIfCancellationRequested(); if (FailQuery) - return Task.FromException>(new HttpRequestException("Controlled query embedding failure.")); - return QueryVector is null ? EmbedAsync(text, ct) : Task.FromResult(Result.Ok(QueryVector)); + throw new HttpRequestException("Controlled query embedding failure."); + + var vector = QueryVector; + return vector is null ? await EmbedAsync(text, ct).ConfigureAwait(false) : Result.Ok(vector); } private static Task> EmbedAsync(string text, CancellationToken ct) diff --git a/tools/product-search-relevance.dataset.json b/tools/product-search-relevance.dataset.json index 9b6cb4e6c..a014836a0 100644 --- a/tools/product-search-relevance.dataset.json +++ b/tools/product-search-relevance.dataset.json @@ -1,5 +1,5 @@ { - "datasetVersion": "product-search-relevance-v1", + "datasetVersion": "product-search-relevance-v2-mtb", "profile": "ollama:nomic-embed-text:v1.5:768:retrieval-prefix-v1", "model": "nomic-embed-text:v1.5", "modelDigest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f", @@ -7,386 +7,4407 @@ "documentRecipe": "product-document-v2", "documentInstruction": "search_document: ", "queryInstruction": "search_query: ", - "seedFile": "tools/product-search-relevance.seed.yaml", - "seedFileSha256": "F18614A4520837867BF9187D812ECC5BAF0559D0E652FC1175CDBECFC74ADF44", - "catalogEligibility": "Default Products API catalog: exclude inactive and logically deleted products; non-stocked service rows remain eligible.", + "seedFile": "samples/data/demo-catalog.seed.yaml", + "seedFileSha256": "58E9FBD244DB262ACDAC769DFACED6D8F8CD9E17BFF73CF4A35F7F317724FAA4", + "referenceDataFile": "samples/src/Contoso.Products.Database/Data/ref-data.seed.yaml", + "referenceDataFileSha256": "DD6709BEA71EB7AB5490B6038010CB349147F7B36AF52FBE0CD97ED0AF556323", + "catalogEligibility": "Shared demo catalog MTB subset: exclude inactive and logically deleted rows; sku ne 'LABOR' excludes the workshop service without removing reviewed Miscellaneous products. All 170 reviewed MTB products retain their identity.", "catalogReviewed": true, "catalog": [ - { "sku": "YETI-ASR-C2-2025", "text": "Yeti ASR C2", "price": 5800, "description": "A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack." }, - { "sku": "YETI-SB120-C2", "text": "Yeti SB120 C2", "price": 6200, "description": "A carbon full-suspension trail mountain bike designed for efficient climbs, responsive handling and flowing singletrack descents." }, - { "sku": "YETI-SB140-LR-C2", "text": "Yeti SB140 LR C2", "price": 6600, "description": "A carbon full-suspension trail mountain bike in the Lunch Ride configuration, built for technical climbs and rough, fast descents." }, - { "sku": "YETI-SB165-C3", "text": "Yeti SB165 C3", "price": 7200, "description": "A carbon full-suspension mountain bike for gravity-focused riding, steep descents and demanding bike-park terrain." }, - { "sku": "YETI-MTE-C2", "text": "Yeti MTe C2 eMTB", "price": 9850, "description": "An electric-assist mountain bike with a carbon full-suspension frame, designed for technical trail riding and assisted climbs." }, - { "sku": "CANYON-LUXTRAIL-CF6", "text": "Canyon Lux Trail CF 6", "price": 3899, "description": "A carbon full-suspension mountain bike combining cross-country pedalling efficiency with trail capability for long rides and technical singletrack." }, - { "sku": "CANYON-SPECTRAL-CF8", "text": "Canyon Spectral CF 8", "price": 4999, "description": "A carbon full-suspension trail mountain bike for varied singletrack, balancing climbing capability with control on rough, technical descents." }, - { "sku": "CANYON-NEURON-CF7", "text": "Canyon Neuron CF 7", "price": 3499, "description": "A carbon full-suspension trail mountain bike for long rides over varied terrain, combining efficient climbing with stable descending." }, - { "sku": "CANYON-TORQUE-MULLET-CF8", "text": "Canyon Torque Mullet CF 8", "price": 3599, "description": "A carbon mixed-wheel mountain bike designed for freeride and bike-park use, including jumps, drops and steep descents." }, - { "sku": "CANYON-SENDER-CFR", "text": "Canyon Sender CFR", "price": 7799, "description": "A carbon downhill mountain bike designed for racing and fast descending on steep, demanding tracks and bike-park terrain." }, - { "sku": "SPEC-STUMPJUMPER-15-COMP", "text": "Specialized Stumpjumper 15 Comp", "price": 5500, "description": "A full-suspension trail mountain bike with adjustable geometry, designed to balance climbing efficiency and control on technical descents." }, - { "sku": "SPEC-ENDURO-PRO", "text": "Specialized Enduro Pro", "price": 9500, "description": "A carbon full-suspension enduro mountain bike built for steep, rough descents and enduro racing while retaining the ability to pedal uphill." }, - { "sku": "SPEC-CHISEL-COMP", "text": "Specialized Chisel Comp", "price": 3400, "description": "An aluminium cross-country mountain bike designed for efficient pedalling, racing and fast singletrack riding." }, - { "sku": "SPEC-STATUS-140", "text": "Specialized Status 140", "price": 2499, "description": "An aluminium mixed-wheel full-suspension mountain bike designed for playful trail riding, jumps and technical descents." }, - { "sku": "GIANT-TRANCE-X-ADVPRO29", "text": "Giant Trance X Advanced Pro 29", "price": 6200, "description": "A carbon full-suspension trail mountain bike with 29-inch wheels and adjustable geometry, designed for climbing and technical descending." }, - { "sku": "GIANT-ANTHEM-ADV29-1", "text": "Giant Anthem Advanced 29 1", "price": 7800, "description": "A carbon full-suspension cross-country mountain bike with 29-inch wheels, designed for efficient pedalling and fast riding on demanding race courses." }, - { "sku": "SHIMANO-XT-M8100-RD", "text": "Shimano Deore XT M8100 Rear Derailleur", "price": 129, "description": "A 12-speed mountain bike rear derailleur that moves the chain between cassette sprockets to change gears. Its clutch helps control chain movement on rough trails." }, - { "sku": "SHIMANO-SLX-M7100-CASSETTE", "text": "Shimano SLX M7100 12-speed Cassette", "price": 99, "description": "A 12-speed mountain bike cassette that provides a wide range of gears for climbing and descending. It uses Shimano Micro Spline mounting and Hyperglide Plus shifting." }, - { "sku": "SHIMANO-XTR-M9100-CRANK", "text": "Shimano XTR M9100 Crankset", "price": 499, "description": "A mountain bike crankset that transfers pedalling power to the chain through a direct-mount chainring, designed for lightweight cross-country and trail use." }, - { "sku": "SRAM-GX-EAGLE-RD", "text": "SRAM GX Eagle Rear Derailleur", "price": 135, "description": "A cable-operated 12-speed mountain bike rear derailleur for SRAM Eagle drivetrains. It shifts the chain between rear sprockets and uses a clutch to control chain movement." }, - { "sku": "SRAM-X0-TRANSMISSION-RD", "text": "SRAM X0 Transmission Rear Derailleur", "price": 400, "description": "A wireless electronic mountain bike rear derailleur for SRAM Eagle Transmission drivetrains. Its hangerless Full Mount design attaches around the rear axle on compatible frames." }, - { "sku": "SRAM-XX1-EAGLE-CASSETTE", "text": "SRAM XX1 Eagle Cassette", "price": 449, "description": "A lightweight 12-speed mountain bike cassette with a wide gear range for climbing and descending, designed for SRAM Eagle drivetrains and XD freehub drivers." }, - { "sku": "ONEUP-V3-DROPPER-180", "text": "OneUp Dropper Post V3 180mm", "price": 269, "description": "A mountain bike dropper seatpost with 180 mm of travel. Riders can lower the saddle for descending and raise it for pedalling using a compatible remote." }, - { "sku": "ONEUP-CARBON-HANDLEBAR", "text": "OneUp Carbon Handlebar", "price": 159, "description": "A carbon mountain bike handlebar with a shaped profile designed to absorb vibration and reduce hand fatigue while maintaining steering control." }, - { "sku": "ONEUP-EDC-TOOL-V2", "text": "OneUp EDC Tool V2", "price": 59, "description": "A compact trailside repair multi-tool with hex keys, a Torx key, a chain breaker and a tyre lever, designed for storage in compatible OneUp steerer or pump systems." }, - { "sku": "MUCOFF-NANO-CLEANER", "text": "Muc-Off Nano Tech Bike Cleaner", "price": 16.99, "description": "A spray-on bicycle cleaner that breaks down dirt and grime on frames and components before rinsing. Suitable for routine washing after muddy rides." }, - { "sku": "MUCOFF-TUBELESS-SEALANT", "text": "Muc-Off MTB Tubeless Sealant", "price": 8.00, "description": "A tubeless mountain bike tyre sealant that helps seal punctures and maintain an airtight tyre system. Its UV-reactive dye helps locate leaks." }, - { "sku": "MUCOFF-DRIVETRAIN-CLEANER", "text": "Muc-Off Drivetrain Cleaner", "price": 25.00, "description": "A bicycle drivetrain cleaner that removes grease, oil and grime from chains, cassettes, chainrings and derailleurs. It can be sprayed on or used with a chain-cleaning device." }, - { "sku": "LABOR", "text": "Labor", "price": 80.00, "description": "Workshop labour charged by the hour for bicycle maintenance and repair. This is a service rather than a stocked physical product." } + { + "sku": "YETI-ASR-C2-2025", + "text": "Yeti ASR C2", + "price": 5800, + "description": "A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack.", + "sourceUrl": "https://yeticycles.com/en-us/bikes/asr", + "sourceType": "manufacturer", + "subCategoryCode": "XC", + "brandCode": "YETI", + "tags": [ + "cross-country", + "full-suspension", + "mountain-bike", + "dentist-bike" + ], + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "YETI-SB120-C2", + "text": "Yeti SB120 C2", + "price": 6200, + "description": "A carbon full-suspension trail mountain bike designed for efficient climbs, responsive handling and flowing singletrack descents.", + "sourceUrl": "https://yeticycles.com/en-us/bikes/sb120", + "sourceType": "manufacturer", + "subCategoryCode": "TR", + "brandCode": "YETI", + "tags": [ + "mountain-bike", + "dentist-bike" + ], + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "YETI-SB140-LR-C2", + "text": "Yeti SB140 LR C2", + "price": 6600, + "description": "A carbon full-suspension trail mountain bike in the Lunch Ride configuration, built for technical climbs and rough, fast descents.", + "sourceUrl": "https://yeticycles.com/en-us/bikes/sb140", + "sourceType": "manufacturer", + "subCategoryCode": "TR", + "brandCode": "YETI", + "tags": [ + "mountain-bike", + "dentist-bike" + ], + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "YETI-SB165-C3", + "text": "Yeti SB165 C3", + "price": 7200, + "description": "A carbon full-suspension mountain bike for gravity-focused riding, steep descents and demanding bike-park terrain.", + "sourceUrl": "https://support.yeticycles.com/hc/en-us/articles/56816935325211-SB165-specs-and-FAQ", + "sourceType": "manufacturer", + "subCategoryCode": "EN", + "brandCode": "YETI", + "tags": [ + "mountain-bike", + "dentist-bike" + ], + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "YETI-MTE-C2", + "text": "Yeti MTe C2 eMTB", + "price": 9850, + "description": "An electric-assist mountain bike with a carbon full-suspension frame, designed for technical trail riding and assisted climbs.", + "sourceUrl": "https://yeticycles.com/en-us/bikes/mte", + "sourceType": "manufacturer", + "subCategoryCode": "EM", + "brandCode": "YETI", + "tags": [ + "mountain-bike", + "dentist-bike", + "electric", + "e-bike", + "emtb" + ], + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + }, + { + "tag": "e-bike", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + }, + { + "tag": "emtb", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + } + ] + }, + { + "sku": "CANYON-LUXTRAIL-CF6", + "text": "Canyon Lux Trail CF 6", + "price": 3899, + "description": "A carbon full-suspension mountain bike combining cross-country pedalling efficiency with trail capability for long rides and technical singletrack.", + "sourceUrl": "https://media-centre.canyon.com/en-INT/231333-the-brand-new-lux-trail-canyon-s-ultra-capable-xc-bike/", + "sourceType": "manufacturer", + "subCategoryCode": "XC", + "brandCode": "CANYON", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "CANYON-SPECTRAL-CF8", + "text": "Canyon Spectral CF 8", + "price": 4999, + "description": "A carbon full-suspension trail mountain bike for varied singletrack, balancing climbing capability with control on rough, technical descents.", + "sourceUrl": "https://media-centre.canyon.com/en-INT/234670-canyon-launch-all-new-trail-taming-spectral-cf/", + "sourceType": "manufacturer", + "subCategoryCode": "TR", + "brandCode": "CANYON", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "CANYON-NEURON-CF7", + "text": "Canyon Neuron CF 7", + "price": 3499, + "description": "A carbon full-suspension trail mountain bike for long rides over varied terrain, combining efficient climbing with stable descending.", + "sourceUrl": "https://www.canyon.com/en-us/mountain-bikes/trail-bikes/neuron/", + "sourceType": "manufacturer", + "subCategoryCode": "TR", + "brandCode": "CANYON", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "CANYON-TORQUE-MULLET-CF8", + "text": "Canyon Torque Mullet CF 8", + "price": 3599, + "description": "A carbon mixed-wheel mountain bike designed for freeride and bike-park use, including jumps, drops and steep descents.", + "sourceUrl": "https://mtbdatabase.com/bikes/2024/canyon/torque/2024-canyon-torque-mullet-cf-8/", + "sourceType": "retailer", + "subCategoryCode": "EN", + "brandCode": "CANYON", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "CANYON-SENDER-CFR", + "text": "Canyon Sender CFR", + "price": 7799, + "description": "A carbon downhill mountain bike designed for racing and fast descending on steep, demanding tracks and bike-park terrain.", + "sourceUrl": "https://www.canyon.com/en-gb/mountain-bikes/downhill-bikes/sender/", + "sourceType": "manufacturer", + "subCategoryCode": "DH", + "brandCode": "CANYON", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SPEC-STUMPJUMPER-15-COMP", + "text": "Specialized Stumpjumper 15 Comp", + "price": 5500, + "description": "A full-suspension trail mountain bike with adjustable geometry, designed to balance climbing efficiency and control on technical descents.", + "sourceUrl": "https://www.specialized.com/us/en/stumpjumper-15-comp-sram-s-1000-axs-fox-performance/p/4221395", + "sourceType": "manufacturer", + "subCategoryCode": "TR", + "brandCode": "SPEC", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SPEC-ENDURO-PRO", + "text": "Specialized Enduro Pro", + "price": 9500, + "description": "A carbon full-suspension enduro mountain bike built for steep, rough descents and enduro racing while retaining the ability to pedal uphill.", + "sourceUrl": "https://www.specialized.com/us/en/enduro-pro/p/4276010", + "sourceType": "manufacturer", + "subCategoryCode": "EN", + "brandCode": "SPEC", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SPEC-CHISEL-COMP", + "text": "Specialized Chisel Comp", + "price": 3400, + "description": "An aluminium cross-country mountain bike designed for efficient pedalling, racing and fast singletrack riding.", + "sourceUrl": "https://www.specialized.com/us/en/chisel-hardtail-comp/p/4221772", + "sourceType": "manufacturer", + "subCategoryCode": "HT", + "brandCode": "SPEC", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SPEC-STATUS-140", + "text": "Specialized Status 140", + "price": 2499, + "description": "An aluminium mixed-wheel full-suspension mountain bike designed for playful trail riding, jumps and technical descents.", + "sourceUrl": "https://www.specialized.com/us/en/status-140/p/199765", + "sourceType": "manufacturer", + "subCategoryCode": "FR", + "brandCode": "SPEC", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "GIANT-TRANCE-X-ADVPRO29", + "text": "Giant Trance X Advanced Pro 29", + "price": 6200, + "description": "A carbon full-suspension trail mountain bike with 29-inch wheels and adjustable geometry, designed for climbing and technical descending.", + "sourceUrl": "https://www.giant-bicycles.com/global/showcase/trance-x-advanced", + "sourceType": "manufacturer", + "subCategoryCode": "TR", + "brandCode": "GIANT", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "GIANT-ANTHEM-ADV29-1", + "text": "Giant Anthem Advanced 29 1", + "price": 7800, + "description": "A carbon full-suspension cross-country mountain bike with 29-inch wheels, designed for efficient pedalling and fast riding on demanding race courses.", + "sourceUrl": "https://www.giant-bicycles.com/global/showcase/anthem-advanced-29", + "sourceType": "manufacturer", + "subCategoryCode": "XC", + "brandCode": "GIANT", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SHIMANO-XT-M8100-RD", + "text": "Shimano Deore XT M8100 Rear Derailleur", + "price": 129, + "description": "A 12-speed mountain bike rear derailleur that moves the chain between cassette sprockets to change gears. Its clutch helps control chain movement on rough trails.", + "sourceUrl": "https://bike.shimano.com/en-NA/products/components/pdp.P-RD-M8100-SGS.html", + "sourceType": "manufacturer", + "subCategoryCode": "DR", + "brandCode": "SHIMANO", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SHIMANO-SLX-M7100-CASSETTE", + "text": "Shimano SLX M7100 12-speed Cassette", + "price": 99, + "description": "A 12-speed mountain bike cassette that provides a wide range of gears for climbing and descending. It uses Shimano Micro Spline mounting and Hyperglide Plus shifting.", + "sourceUrl": "https://bike.shimano.com/products/components/pdp.P-CS-M7100-12.html", + "sourceType": "manufacturer", + "subCategoryCode": "CA", + "brandCode": "SHIMANO", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SHIMANO-XTR-M9100-CRANK", + "text": "Shimano XTR M9100 Crankset", + "price": 499, + "description": "A mountain bike crankset that transfers pedalling power to the chain through a direct-mount chainring, designed for lightweight cross-country and trail use.", + "sourceUrl": "https://bike.shimano.com/products/components/pdp.P-FC-M9100-1.html", + "sourceType": "manufacturer", + "subCategoryCode": "CR", + "brandCode": "SHIMANO", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SRAM-GX-EAGLE-RD", + "text": "SRAM GX Eagle Rear Derailleur", + "price": 135, + "description": "A cable-operated 12-speed mountain bike rear derailleur for SRAM Eagle drivetrains. It shifts the chain between rear sprockets and uses a clutch to control chain movement.", + "sourceUrl": "https://www.sram.com/en/sram/models/rd-gx-1-b2", + "sourceType": "manufacturer", + "subCategoryCode": "DR", + "brandCode": "SRAM", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "text": "SRAM X0 Transmission Rear Derailleur", + "price": 400, + "description": "A wireless electronic mountain bike rear derailleur for SRAM Eagle Transmission drivetrains. Its hangerless Full Mount design attaches around the rear axle on compatible frames.", + "sourceUrl": "https://www.sram.com/en/sram/models/rd-x0-e-b1", + "sourceType": "manufacturer", + "subCategoryCode": "DR", + "brandCode": "SRAM", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "SRAM-XX1-EAGLE-CASSETTE", + "text": "SRAM XX1 Eagle Cassette", + "price": 449, + "description": "A lightweight 12-speed mountain bike cassette with a wide gear range for climbing and descending, designed for SRAM Eagle drivetrains and XD freehub drivers.", + "sourceUrl": "https://www.sram.com/en/sram/models/cs-xg-1299-a2", + "sourceType": "manufacturer", + "subCategoryCode": "CA", + "brandCode": "SRAM", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "ONEUP-V3-DROPPER-180", + "text": "OneUp Dropper Post V3 180mm", + "price": 269, + "description": "A mountain bike dropper seatpost with 180 mm of travel. Riders can lower the saddle for descending and raise it for pedalling using a compatible remote.", + "sourceUrl": "https://www.oneupcomponents.com/products/dropper-post-v3", + "sourceType": "manufacturer", + "subCategoryCode": "SP", + "brandCode": "ONEUP", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "ONEUP-CARBON-HANDLEBAR", + "text": "OneUp Carbon Handlebar", + "price": 159, + "description": "A carbon mountain bike handlebar with a shaped profile designed to absorb vibration and reduce hand fatigue while maintaining steering control.", + "sourceUrl": "https://www.oneupcomponents.com/products/v2-carbon-handlebar", + "sourceType": "manufacturer", + "subCategoryCode": "HB", + "brandCode": "ONEUP", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "ONEUP-EDC-TOOL-V2", + "text": "OneUp EDC Tool V2", + "price": 59, + "description": "A compact trailside repair multi-tool with hex keys, a Torx key, a chain breaker and a tyre lever, designed for storage in compatible OneUp steerer or pump systems.", + "sourceUrl": "https://www.oneupcomponents.com/products/edc-v2-tool", + "sourceType": "manufacturer", + "subCategoryCode": "TO", + "brandCode": "ONEUP", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "MUCOFF-NANO-CLEANER", + "text": "Muc-Off Nano Tech Bike Cleaner", + "price": 16.99, + "description": "A spray-on bicycle cleaner that breaks down dirt and grime on frames and components before rinsing. Suitable for routine washing after muddy rides.", + "sourceUrl": "https://muc-off.com/products/bike-cleaner", + "sourceType": "manufacturer", + "subCategoryCode": "CL", + "brandCode": "MUCOFF", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "MUCOFF-TUBELESS-SEALANT", + "text": "Muc-Off MTB Tubeless Sealant", + "price": 8.0, + "description": "A tubeless mountain bike tyre sealant that helps seal punctures and maintain an airtight tyre system. Its UV-reactive dye helps locate leaks.", + "sourceUrl": "https://muc-off.com/products/mtb-tubeless-sealant", + "sourceType": "manufacturer", + "subCategoryCode": "SE", + "brandCode": "MUCOFF", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "MUCOFF-DRIVETRAIN-CLEANER", + "text": "Muc-Off Drivetrain Cleaner", + "price": 25.0, + "description": "A bicycle drivetrain cleaner that removes grease, oil and grime from chains, cassettes, chainrings and derailleurs. It can be sprayed on or used with a chain-cleaning device.", + "sourceUrl": "https://www.bikefreeus.com/products/muc-off-drivetrain-cleaner", + "sourceType": "retailer", + "subCategoryCode": "CL", + "brandCode": "MUCOFF", + "tags": [ + "mountain-bike" + ] + }, + { + "sku": "YETI-ARC", + "text": "Yeti ARC Mountain Bike", + "price": 3200, + "description": "A hardtail mountain bike in Yeti Cycles' ARC model family for cross-country and trail riding.", + "sourceUrl": "https://yeticycles.com/bikes/arc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail", + "cross-country", + "dentist-bike" + ], + "subCategoryCode": "HT", + "brandCode": "YETI", + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "YETI-SB135", + "text": "Yeti SB135 Mountain Bike", + "price": 6400, + "description": "A full-suspension trail mountain bike in Yeti Cycles' SB135 model family.", + "sourceUrl": "https://yeticycles.com/bikes/sb135", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "trail", + "dentist-bike" + ], + "subCategoryCode": "TR", + "brandCode": "YETI", + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "YETI-SB160", + "text": "Yeti SB160 Mountain Bike", + "price": 7900, + "description": "A full-suspension enduro mountain bike in Yeti Cycles' SB160 model family.", + "sourceUrl": "https://yeticycles.com/bikes/sb160", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "enduro", + "dentist-bike" + ], + "subCategoryCode": "EN", + "brandCode": "YETI", + "tagProvenance": [ + { + "tag": "dentist-bike", + "origin": "user-provided-informal-search-alias", + "note": "Search alias only; no intended-rider or affordability claim." + } + ] + }, + { + "sku": "CANYON-STOIC", + "text": "Canyon Stoic Mountain Bike", + "price": 1899, + "description": "A hardtail mountain bike in Canyon's Stoic model family for trail riding.", + "sourceUrl": "https://www.canyon.com/en-us/mountain-bikes/trail-bikes/stoic/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail", + "trail" + ], + "subCategoryCode": "HT", + "brandCode": "CANYON" + }, + { + "sku": "CANYON-STRIVE", + "text": "Canyon Strive Mountain Bike", + "price": 5799, + "description": "A full-suspension enduro mountain bike in Canyon's Strive model family.", + "sourceUrl": "https://www.canyon.com/en-us/mountain-bikes/enduro-bikes/strive/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "enduro" + ], + "subCategoryCode": "EN", + "brandCode": "CANYON" + }, + { + "sku": "CANYON-GRAND-CANYON", + "text": "Canyon Grand Canyon Mountain Bike", + "price": 1499, + "description": "A hardtail mountain bike in Canyon's Grand Canyon model family for off-road riding.", + "sourceUrl": "https://www.canyon.com/en-us/mountain-bikes/hardtail-mountain-bikes/grand-canyon/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail" + ], + "subCategoryCode": "HT", + "brandCode": "CANYON" + }, + { + "sku": "CANYON-LUX-WORLD-CUP", + "text": "Canyon Lux World Cup Mountain Bike", + "price": 6299, + "description": "A full-suspension cross-country race mountain bike in Canyon's Lux World Cup family.", + "sourceUrl": "https://media-centre.canyon.com/en-INT/264438-no-limits-on-the-limit-canyon-launches-new-lux-world-cup-cfr/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cross-country", + "race" + ], + "subCategoryCode": "XC", + "brandCode": "CANYON" + }, + { + "sku": "SPECIALIZED-FUSE", + "text": "Specialized Fuse Mountain Bike", + "price": 2799, + "description": "A hardtail mountain bike in Specialized's Fuse model family for trail riding.", + "sourceUrl": "https://www.specialized.com/us/en/shop/bikes/mountain-bikes/trail-mountain-bikes/fuse", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail", + "trail" + ], + "subCategoryCode": "HT", + "brandCode": "SPEC" + }, + { + "sku": "SPECIALIZED-DEMO", + "text": "Specialized Demo Mountain Bike", + "price": 6499, + "description": "A downhill mountain bike in Specialized's Demo model family.", + "sourceUrl": "https://www.specialized.com/us/en/shop/bikes/mountain-bikes/downhill-mountain-bikes/demo", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "downhill" + ], + "subCategoryCode": "DH", + "brandCode": "SPEC" + }, + { + "sku": "SPECIALIZED-TURBO-LEVO", + "text": "Specialized Turbo Levo E-Mountain Bike", + "price": 8999, + "description": "An electric-assist mountain bike in Specialized's Turbo Levo model family.", + "sourceUrl": "https://www.specialized.com/us/en/turbo-levo/c/turbo-levo", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "electric", + "e-bike", + "emtb" + ], + "subCategoryCode": "EM", + "brandCode": "SPEC", + "tagProvenance": [ + { + "tag": "e-bike", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + }, + { + "tag": "emtb", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + } + ] + }, + { + "sku": "SPECIALIZED-TURBO-KENEVO", + "text": "Specialized Turbo Kenevo E-Mountain Bike", + "price": 9999, + "description": "An electric-assist mountain bike in Specialized's Turbo Kenevo model family.", + "sourceUrl": "https://www.specialized.com/us/en/shop/bikes/electric-bikes/electric-mountain-bikes/turbo-kenevo", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "electric", + "enduro", + "e-bike", + "emtb" + ], + "subCategoryCode": "EM", + "brandCode": "SPEC", + "tagProvenance": [ + { + "tag": "e-bike", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + }, + { + "tag": "emtb", + "origin": "user-provided-colloquial-search-alias", + "note": "Synonym for the verified electric-assist mountain-bike product type." + } + ] + }, + { + "sku": "SPECIALIZED-EPIC-WORLD-CUP", + "text": "Specialized Epic World Cup Mountain Bike", + "price": 8999, + "description": "A cross-country race mountain bike in Specialized's Epic World Cup family.", + "sourceUrl": "https://www.specialized.com/us/en/s-works-epic-world-cup-sram-xx-sl-axs-rockshox-ultimate/p/4221515", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cross-country", + "race" + ], + "subCategoryCode": "XC", + "brandCode": "SPEC" + }, + { + "sku": "GIANT-TRANCE", + "text": "Giant Trance Mountain Bike", + "price": 4599, + "description": "A full-suspension trail mountain bike in Giant's Trance model family.", + "sourceUrl": "https://www.giant-bicycles.com/us/trance", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "trail" + ], + "subCategoryCode": "TR", + "brandCode": "GIANT" + }, + { + "sku": "GIANT-REIGN", + "text": "Giant Reign Mountain Bike", + "price": 5499, + "description": "A full-suspension enduro mountain bike in Giant's Reign model family.", + "sourceUrl": "https://www.giant-bicycles.com/us/reign", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "enduro" + ], + "subCategoryCode": "EN", + "brandCode": "GIANT" + }, + { + "sku": "GIANT-FATHOM", + "text": "Giant Fathom Mountain Bike", + "price": 1699, + "description": "A hardtail mountain bike in Giant's Fathom model family for trail riding.", + "sourceUrl": "https://www.giant-bicycles.com/us/fathom", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "hardtail", + "trail" + ], + "subCategoryCode": "HT", + "brandCode": "GIANT" + }, + { + "sku": "GIANT-GLORY", + "text": "Giant Glory Mountain Bike", + "price": 7199, + "description": "A downhill mountain bike in Giant's Glory model family.", + "sourceUrl": "https://www.giant-bicycles.com/us/glory", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "downhill" + ], + "subCategoryCode": "DH", + "brandCode": "GIANT" + }, + { + "sku": "GIANT-STANCE", + "text": "Giant Stance Mountain Bike", + "price": 2499, + "description": "A full-suspension trail mountain bike in Giant's Stance model family.", + "sourceUrl": "https://www.giant-bicycles.com/us/stance", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "trail" + ], + "subCategoryCode": "TR", + "brandCode": "GIANT" + }, + { + "sku": "SPECIALIZED-STATUS-2", + "text": "Specialized Status 2 Mountain Bike", + "price": 3299, + "description": "A mixed-wheel mountain bike in Specialized's Status model family for trail and freeride use.", + "sourceUrl": "https://www.specialized.com/us/en/status-140/p/199765", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "freeride" + ], + "subCategoryCode": "FR", + "brandCode": "SPEC" + }, + { + "sku": "GIANT-ANTHEM", + "text": "Giant Anthem Mountain Bike", + "price": 6399, + "description": "A full-suspension cross-country mountain bike in Giant's Anthem model family.", + "sourceUrl": "https://www.giant-bicycles.com/global/anthem", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cross-country" + ], + "subCategoryCode": "XC", + "brandCode": "GIANT" + }, + { + "sku": "SHIMANO-SLX-M7100-DERAILLEUR", + "text": "Shimano SLX M7100 Rear Derailleur", + "price": 89, + "description": "A rear derailleur from Shimano's SLX M7100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur" + ], + "subCategoryCode": "DR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-SLX-M7100-SHIFTER", + "text": "Shimano SLX M7100 Shifter", + "price": 49, + "description": "A mountain bike gear shifter from Shimano's SLX M7100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "shifter" + ], + "subCategoryCode": "DR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-SLX-M7100-CHAIN", + "text": "Shimano SLX M7100 Chain", + "price": 39, + "description": "A bicycle chain from Shimano's SLX M7100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ], + "subCategoryCode": "CH", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-SLX-M7100-CRANKSET", + "text": "Shimano SLX M7100 Crankset", + "price": 149, + "description": "A mountain bike crankset from Shimano's SLX M7100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ], + "subCategoryCode": "CR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XT-M8100-SHIFTER", + "text": "Shimano Deore XT M8100 Shifter", + "price": 69, + "description": "A mountain bike gear shifter from Shimano's Deore XT M8100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "shifter" + ], + "subCategoryCode": "DR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XT-M8100-CHAIN", + "text": "Shimano Deore XT M8100 Chain", + "price": 54, + "description": "A bicycle chain from Shimano's Deore XT M8100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ], + "subCategoryCode": "CH", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XT-M8100-CRANKSET", + "text": "Shimano Deore XT M8100 Crankset", + "price": 199, + "description": "A mountain bike crankset from Shimano's Deore XT M8100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ], + "subCategoryCode": "CR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XT-M8100-CASSETTE", + "text": "Shimano Deore XT M8100 Cassette", + "price": 149, + "description": "A mountain bike cassette from Shimano's Deore XT M8100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ], + "subCategoryCode": "CA", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XT-M8120-BRAKE", + "text": "Shimano Deore XT M8120 Disc Brake", + "price": 219, + "description": "A hydraulic disc brake from Shimano's Deore XT M8120 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ], + "subCategoryCode": "BR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XT-M8100-BRAKE-LEVER", + "text": "Shimano Deore XT M8100 Brake Lever", + "price": 89, + "description": "A hydraulic brake lever from Shimano's Deore XT M8100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes", + "lever" + ], + "subCategoryCode": "BR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XTR-M9100-DERAILLEUR", + "text": "Shimano XTR M9100 Rear Derailleur", + "price": 259, + "description": "A rear derailleur from Shimano's XTR M9100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur" + ], + "subCategoryCode": "DR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XTR-M9100-SHIFTER", + "text": "Shimano XTR M9100 Shifter", + "price": 129, + "description": "A mountain bike gear shifter from Shimano's XTR M9100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "shifter" + ], + "subCategoryCode": "DR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XTR-M9100-CHAIN", + "text": "Shimano XTR M9100 Chain", + "price": 79, + "description": "A bicycle chain from Shimano's XTR M9100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ], + "subCategoryCode": "CH", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XTR-M9100-CASSETTE", + "text": "Shimano XTR M9100 Cassette", + "price": 399, + "description": "A mountain bike cassette from Shimano's XTR M9100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ], + "subCategoryCode": "CA", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XTR-M9100-CRANKSET", + "text": "Shimano XTR M9100 Crankset", + "price": 499, + "description": "A mountain bike crankset from Shimano's XTR M9100 component series.", + "sourceUrl": "https://bike.shimano.com/products/components/pdp.P-FC-M9100-1.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ], + "subCategoryCode": "CR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-XTR-M9100-BRAKE", + "text": "Shimano XTR M9100 Disc Brake", + "price": 399, + "description": "A hydraulic disc brake from Shimano's XTR M9100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ], + "subCategoryCode": "BR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-DEORE-M6100-DERAILLEUR", + "text": "Shimano Deore M6100 Rear Derailleur", + "price": 64, + "description": "A rear derailleur from Shimano's Deore M6100 mountain bike component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur" + ], + "subCategoryCode": "DR", + "brandCode": "SHIMANO" + }, + { + "sku": "SHIMANO-DEORE-M6100-CASSETTE", + "text": "Shimano Deore M6100 Cassette", + "price": 119, + "description": "A mountain bike cassette from Shimano's Deore M6100 component series.", + "sourceUrl": "https://bike.shimano.com/category/mtb.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ], + "subCategoryCode": "CA", + "brandCode": "SHIMANO" + }, + { + "sku": "SRAM-GX-EAGLE-CASSETTE", + "text": "SRAM GX Eagle Cassette", + "price": 235, + "description": "A mountain bike cassette from SRAM's GX Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle/gx-eagle?page=1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ], + "subCategoryCode": "CA", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-X01-EAGLE-CASSETTE", + "text": "SRAM X01 Eagle Cassette", + "price": 349, + "description": "A mountain bike cassette from SRAM's X01 Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/x01-eagle", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ], + "subCategoryCode": "CA", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-XX1-EAGLE-CHAIN", + "text": "SRAM XX1 Eagle Chain", + "price": 89, + "description": "A bicycle chain from SRAM's XX1 Eagle mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/models/cn-eagle-xx1-a1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ], + "subCategoryCode": "CH", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-GX-EAGLE-CHAIN", + "text": "SRAM GX Eagle Chain", + "price": 42, + "description": "A bicycle chain from SRAM's GX Eagle mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle/gx-eagle?page=1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ], + "subCategoryCode": "CH", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-GX-EAGLE-CRANKSET", + "text": "SRAM GX Eagle Crankset", + "price": 159, + "description": "A mountain bike crankset from SRAM's GX Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle/gx-eagle?page=1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ], + "subCategoryCode": "CR", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-GX-EAGLE-SHIFTER", + "text": "SRAM GX Eagle Trigger Shifter", + "price": 49, + "description": "A mountain bike gear shifter from SRAM's GX Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle/gx-eagle?page=1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "shifter" + ], + "subCategoryCode": "DR", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-X01-EAGLE-AXS-DERAILLEUR", + "text": "SRAM X01 Eagle AXS Rear Derailleur", + "price": 499, + "description": "A wireless electronic rear derailleur from SRAM's X01 Eagle AXS mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/models/rd-x0-1-e-a1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur", + "electronic" + ], + "subCategoryCode": "DR", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-XX1-EAGLE-AXS-DERAILLEUR", + "text": "SRAM XX1 Eagle AXS Rear Derailleur", + "price": 699, + "description": "A wireless electronic rear derailleur from SRAM's XX1 Eagle AXS mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/models/rd-xx-1-e-a1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur", + "electronic" + ], + "subCategoryCode": "DR", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-GX-TRANSMISSION-DERAILLEUR", + "text": "SRAM GX Eagle Transmission Rear Derailleur", + "price": 599, + "description": "A rear derailleur from SRAM's GX Eagle Transmission mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle-transmission/gx-eagle-transmission", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "derailleur", + "electronic" + ], + "subCategoryCode": "DR", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-CODE-RSC-BRAKE", + "text": "SRAM Code RSC Disc Brake", + "price": 349, + "description": "A hydraulic disc brake from SRAM's Code mountain bike brake family.", + "sourceUrl": "https://www.sram.com/en/sram/models/db-codr-d1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ], + "subCategoryCode": "BR", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-MAVEN-ULTIMATE-BRAKE", + "text": "SRAM Maven Ultimate Disc Brake", + "price": 699, + "description": "A hydraulic disc brake from SRAM's Maven mountain bike brake family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/maven/maven", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ], + "subCategoryCode": "BR", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-MAVEN-SILVER-BRAKE", + "text": "SRAM Maven Silver Disc Brake", + "price": 499, + "description": "A hydraulic disc brake from SRAM's Maven mountain bike brake family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/maven/maven", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ], + "subCategoryCode": "BR", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-LEVEL-ULTIMATE-BRAKE", + "text": "SRAM Level Ultimate Disc Brake", + "price": 399, + "description": "A hydraulic disc brake from SRAM's Level mountain bike brake family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/level/level", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "brakes" + ], + "subCategoryCode": "BR", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-XX-TRANSMISSION-CASSETTE", + "text": "SRAM XX Eagle Transmission Cassette", + "price": 699, + "description": "A cassette from SRAM's XX Eagle Transmission mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle-transmission/gx-eagle-transmission", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "cassette" + ], + "subCategoryCode": "CA", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-XX-TRANSMISSION-CHAIN", + "text": "SRAM XX Eagle Transmission Chain", + "price": 89, + "description": "A chain from SRAM's XX Eagle Transmission mountain bike drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/gx-eagle-transmission/gx-eagle-transmission", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain" + ], + "subCategoryCode": "CH", + "brandCode": "SRAM" + }, + { + "sku": "SRAM-X01-EAGLE-CRANKSET", + "text": "SRAM X01 Eagle Crankset", + "price": 329, + "description": "A mountain bike crankset from SRAM's X01 Eagle drivetrain family.", + "sourceUrl": "https://www.sram.com/en/sram/mountain/series/x01-eagle", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "crankset" + ], + "subCategoryCode": "CR", + "brandCode": "SRAM" + }, + { + "sku": "FOX-32-SC-FACTORY", + "text": "FOX 32 Step-Cast Factory Fork", + "price": 1099, + "description": "A mountain bike suspension fork in FOX's 32 Step-Cast Factory series for cross-country riding.", + "sourceUrl": "https://www.ridefox.com/family.php?m=bike&family=32", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "cross-country" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-34-FACTORY", + "text": "FOX 34 Factory Fork", + "price": 1199, + "description": "A mountain bike suspension fork in FOX's 34 Factory series for trail and downcountry riding.", + "sourceUrl": "https://ridefox.com/collections/fox-34", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "trail" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-34-FACTORY-GRIP-X", + "text": "FOX 34 Factory GRIP X Fork", + "price": 1199, + "description": "A mountain bike suspension fork in FOX's 34 Factory GRIP X series for trail riding.", + "sourceUrl": "https://ridefox.com/collections/fox-34", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "trail" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-34-PERFORMANCE-ELITE", + "text": "FOX 34 Performance Elite Fork", + "price": 899, + "description": "A mountain bike suspension fork in FOX's 34 Performance Elite series.", + "sourceUrl": "https://ridefox.com/collections/fox-34", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-36-SL-FACTORY", + "text": "FOX 36 SL Factory GRIP X Fork", + "price": 1199, + "description": "A mountain bike suspension fork in FOX's 36 SL Factory series for trail and downcountry riding.", + "sourceUrl": "https://ridefox.com/products/my27-fox-36-sl-factory-gripx", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "trail" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-36-FACTORY-GRIP-X", + "text": "FOX 36 Factory GRIP X Fork", + "price": 1299, + "description": "A mountain bike suspension fork in FOX's 36 Factory GRIP X series for trail riding.", + "sourceUrl": "https://ridefox.com/products/my27-fox-36-factory-gripx", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "trail" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-36-FACTORY-GRIP-X2", + "text": "FOX 36 Factory GRIP X2 Fork", + "price": 1399, + "description": "A mountain bike suspension fork in FOX's 36 Factory GRIP X2 series for all-mountain riding.", + "sourceUrl": "https://ridefox.com/products/my27-fox-36-factory-gripx2", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "all-mountain" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-36-PERFORMANCE-ELITE", + "text": "FOX 36 Performance Elite GRIP X Fork", + "price": 999, + "description": "A mountain bike suspension fork in FOX's 36 Performance Elite series.", + "sourceUrl": "https://ridefox.com/products/my27-fox-36-pse-gripx-fcb6ecad", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-38-FACTORY", + "text": "FOX 38 Factory Fork", + "price": 1399, + "description": "A mountain bike suspension fork in FOX's 38 Factory series for enduro riding.", + "sourceUrl": "https://ridefox.com/products/my27-fox-38-factory", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "enduro" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-38-PERFORMANCE-ELITE", + "text": "FOX 38 Performance Elite Fork", + "price": 1199, + "description": "A mountain bike suspension fork in FOX's 38 Performance Elite series.", + "sourceUrl": "https://ridefox.com/products/my27-fox-38-pse-d795211b", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "enduro" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-40-FACTORY", + "text": "FOX 40 Factory Downhill Fork", + "price": 1899, + "description": "A dual-crown suspension fork in FOX's 40 Factory downhill series.", + "sourceUrl": "https://ridefox.com/products/fox-40-factory-grip-x2-dh-fork-for-downhill-riding", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "fork", + "downhill" + ], + "subCategoryCode": "FS", + "brandCode": "FOX" + }, + { + "sku": "FOX-FLOAT-DPS-FACTORY", + "text": "FOX Float DPS Factory Rear Shock", + "price": 599, + "description": "An air-sprung rear shock in FOX's Float DPS Factory mountain bike suspension series.", + "sourceUrl": "https://www.ridefox.com/family.php?m=bike&family=floatdps", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "rear-shock" + ], + "subCategoryCode": "RS", + "brandCode": "FOX" + }, + { + "sku": "FOX-FLOAT-X-FACTORY", + "text": "FOX Float X Factory Rear Shock", + "price": 699, + "description": "An air-sprung rear shock in FOX's Float X Factory mountain bike suspension series.", + "sourceUrl": "https://ridefox.com/pages/fox-float-x", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "rear-shock" + ], + "subCategoryCode": "RS", + "brandCode": "FOX" + }, + { + "sku": "FOX-FLOAT-X2-FACTORY", + "text": "FOX Float X2 Factory Rear Shock", + "price": 799, + "description": "An air-sprung rear shock in FOX's Float X2 Factory mountain bike suspension series.", + "sourceUrl": "https://www.ridefox.com/family.php?m=bike&family=floatx2", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "rear-shock" + ], + "subCategoryCode": "RS", + "brandCode": "FOX" + }, + { + "sku": "FOX-DHX-FACTORY", + "text": "FOX DHX Factory Coil Shock", + "price": 699, + "description": "A coil-sprung rear shock in FOX's DHX Factory mountain bike suspension series.", + "sourceUrl": "https://ca.ridefox.com/products/fox-dhx-factory", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "suspension", + "rear-shock", + "coil" + ], + "subCategoryCode": "RS", + "brandCode": "FOX" + }, + { + "sku": "MAXXIS-ASSEGAI", + "text": "Maxxis Assegai Mountain Bike Tire", + "price": 99, + "description": "A mountain bike tire in Maxxis's Assegai tread family for enduro and downhill riding.", + "sourceUrl": "https://shop.maxxis.com/products/assegai", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "enduro", + "downhill" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-MINION-DHF", + "text": "Maxxis Minion DHF Mountain Bike Tire", + "price": 95, + "description": "A mountain bike tire in Maxxis's Minion DHF tread family, commonly used at the front for trail, enduro, and downhill riding.", + "sourceUrl": "https://shop.maxxis.com/products/minion-dhf-1", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "front", + "enduro" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-MINION-DHR-II", + "text": "Maxxis Minion DHR II Mountain Bike Tire", + "price": 95, + "description": "A mountain bike tire in Maxxis's Minion DHR II tread family with a braking-oriented tread pattern.", + "sourceUrl": "https://shop.maxxis.com/products/minion-dhr-ii", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "rear", + "braking" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-DISSECTOR", + "text": "Maxxis Dissector Mountain Bike Tire", + "price": 89, + "description": "A mountain bike tire in Maxxis's Dissector tread family for trail and enduro riding.", + "sourceUrl": "https://shop.maxxis.com/products/dissector", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "trail", + "enduro" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-REKON", + "text": "Maxxis Rekon Mountain Bike Tire", + "price": 79, + "description": "A mountain bike tire in Maxxis's Rekon tread family for cross-country and trail riding.", + "sourceUrl": "https://shop.maxxis.com/products/rekon", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "trail" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-REKON-RACE", + "text": "Maxxis Rekon Race Mountain Bike Tire", + "price": 79, + "description": "A low-profile mountain bike tire in Maxxis's Rekon Race family for cross-country racing.", + "sourceUrl": "https://www.maxxis.com/us/tire/rekon-race/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "race" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-FOREKASTER", + "text": "Maxxis Forekaster Mountain Bike Tire", + "price": 84, + "description": "A mountain bike tire in Maxxis's Forekaster tread family for trail riding in mixed conditions.", + "sourceUrl": "https://www.maxxis.com/us/tire/forekaster/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "trail", + "mixed-conditions" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-HIGH-ROLLER-II", + "text": "Maxxis High Roller II Mountain Bike Tire", + "price": 89, + "description": "An aggressive mountain bike tire in Maxxis's High Roller II tread family for loose and varied terrain.", + "sourceUrl": "https://www.maxxis.com/us/tire/high-roller/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "loose-terrain" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-SHORTY", + "text": "Maxxis Shorty Mountain Bike Tire", + "price": 99, + "description": "A mountain bike tire in Maxxis's Shorty tread family for muddy and loose conditions.", + "sourceUrl": "https://www.maxxis.com/us/tire/shorty/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "mud", + "downhill" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-AGGRESSOR", + "text": "Maxxis Aggressor Mountain Bike Tire", + "price": 89, + "description": "A mountain bike tire in Maxxis's Aggressor tread family, often selected for the rear on dry, rocky trails.", + "sourceUrl": "https://www.maxxis.com/us/tire/aggressor/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "rear", + "dry-trail" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-ARDENT", + "text": "Maxxis Ardent Mountain Bike Tire", + "price": 75, + "description": "A mountain bike tire in Maxxis's Ardent tread family for cross-country and trail riding.", + "sourceUrl": "https://www.maxxis.com/us/tire/ardent/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "trail" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-ARDENT-RACE", + "text": "Maxxis Ardent Race Mountain Bike Tire", + "price": 79, + "description": "A mountain bike tire in Maxxis's Ardent Race tread family for cross-country riding.", + "sourceUrl": "https://www.maxxis.com/us/tire/ardent-race/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "race" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-IKON", + "text": "Maxxis Ikon Mountain Bike Tire", + "price": 79, + "description": "A mountain bike tire in Maxxis's Ikon tread family for cross-country riding.", + "sourceUrl": "https://www.maxxis.com/us/tire/ikon/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-ASPEN", + "text": "Maxxis Aspen Mountain Bike Tire", + "price": 89, + "description": "A mountain bike tire in Maxxis's Aspen tread family for cross-country riding and racing.", + "sourceUrl": "https://shop.maxxis.com/products/aspen", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "race" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-ASPEN-ST", + "text": "Maxxis Aspen ST Mountain Bike Tire", + "price": 99, + "description": "A low-profile mountain bike tire in Maxxis's Aspen ST family for short-track cross-country racing.", + "sourceUrl": "https://www.maxxis.com/us/tire/aspen-st/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "race" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-ASPEN-AT", + "text": "Maxxis Aspen AT Mountain Bike Tire", + "price": 99, + "description": "A mountain bike tire in Maxxis's Aspen AT family for cross-country riding across varied terrain.", + "sourceUrl": "https://www.maxxis.com/us/tire/aspen-at/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "all-terrain" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-CROSSMARK-II", + "text": "Maxxis Crossmark II Mountain Bike Tire", + "price": 69, + "description": "A mountain bike tire in Maxxis's Crossmark II tread family with a fast-rolling center ridge for dry conditions.", + "sourceUrl": "https://www.maxxis.com/us/tire/crossmark-ii/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "cross-country", + "fast-rolling" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-MINION-SS", + "text": "Maxxis Minion SS Mountain Bike Tire", + "price": 89, + "description": "A semi-slick mountain bike tire in Maxxis's Minion SS family for fast-rolling conditions.", + "sourceUrl": "https://www.maxxis.com/us/tire/minion-ss/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "semi-slick", + "fast-rolling" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-MINION-FBF", + "text": "Maxxis Minion FBF Fat Bike Front Tire", + "price": 109, + "description": "A front tire in Maxxis's Minion FBF fat-bike tread family.", + "sourceUrl": "https://www.maxxis.com/us/tire/minion-fbf/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "fat-bike", + "front" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "MAXXIS-MINION-FBR", + "text": "Maxxis Minion FBR Fat Bike Rear Tire", + "price": 109, + "description": "A rear tire in Maxxis's Minion FBR fat-bike tread family.", + "sourceUrl": "https://www.maxxis.com/us/tire/minion-fbr/", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tire", + "fat-bike", + "rear" + ], + "subCategoryCode": "TY", + "brandCode": "MAXXIS" + }, + { + "sku": "ONEUP-V3-DROPPER-120", + "text": "OneUp Dropper Post V3 120mm", + "price": 269, + "description": "A mountain bike dropper seatpost with 120 mm of travel from OneUp Components' V3 series.", + "sourceUrl": "https://www.oneupcomponents.com/products/dropper-post-v3", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "dropper", + "seatpost" + ], + "subCategoryCode": "SP", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-V3-DROPPER-210", + "text": "OneUp Dropper Post V3 210mm", + "price": 269, + "description": "A mountain bike dropper seatpost with 210 mm of travel from OneUp Components' V3 series.", + "sourceUrl": "https://www.oneupcomponents.com/products/dropper-post-v3", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "dropper", + "seatpost" + ], + "subCategoryCode": "SP", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-V3-DROPPER-240", + "text": "OneUp Dropper Post V3 240mm", + "price": 269, + "description": "A mountain bike dropper seatpost with 240 mm of travel from OneUp Components' V3 series.", + "sourceUrl": "https://www.oneupcomponents.com/products/dropper-post-v3", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "dropper", + "seatpost" + ], + "subCategoryCode": "SP", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-V2-CARBON-HANDLEBAR", + "text": "OneUp V2 Carbon Handlebar", + "price": 159, + "description": "A carbon mountain bike handlebar from OneUp Components' V2 series.", + "sourceUrl": "https://www.oneupcomponents.com/products/v2-carbon-handlebar", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "handlebar" + ], + "subCategoryCode": "HB", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-ALUMINUM-HANDLEBAR", + "text": "OneUp Aluminum Handlebar", + "price": 89, + "description": "An aluminum handlebar for mountain bikes from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/handlebars", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "handlebar" + ], + "subCategoryCode": "HB", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-ALUMINUM-PEDAL", + "text": "OneUp Aluminum Pedals", + "price": 149, + "description": "Flat platform pedals for mountain bikes from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/pedals", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "pedals", + "flat-pedal" + ], + "subCategoryCode": "PE", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-EDC-V2-TOOL", + "text": "OneUp EDC Tool V2", + "price": 59, + "description": "A compact trailside multi-tool from OneUp Components' EDC V2 family.", + "sourceUrl": "https://www.oneupcomponents.com/products/edc-v2-tool", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tool", + "repair" + ], + "subCategoryCode": "TO", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-EDC-LITE-TOOL", + "text": "OneUp EDC Lite Tool", + "price": 39, + "description": "A compact multi-tool in OneUp Components' EDC Lite tool family for mountain bike repairs.", + "sourceUrl": "https://www.oneupcomponents.com/collections/edc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tool", + "repair" + ], + "subCategoryCode": "TO", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-EDC-PUMP-100", + "text": "OneUp EDC Pump 100cc", + "price": 59, + "description": "A compact hand pump in OneUp Components' EDC pump family for carrying on mountain bike rides.", + "sourceUrl": "https://www.oneupcomponents.com/collections/edc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "pump", + "repair" + ], + "subCategoryCode": "TO", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-EDC-PUMP-70", + "text": "OneUp EDC Pump 70cc", + "price": 49, + "description": "A compact hand pump in OneUp Components' EDC pump family for carrying on mountain bike rides.", + "sourceUrl": "https://www.oneupcomponents.com/collections/edc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "pump", + "repair" + ], + "subCategoryCode": "TO", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-EDC-TOP-CAP", + "text": "OneUp EDC Top Cap", + "price": 29, + "description": "A top cap in OneUp Components' EDC system for mountain bike tool storage.", + "sourceUrl": "https://www.oneupcomponents.com/collections/edc", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "tool-storage" + ], + "subCategoryCode": "TO", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-CHAIN-GUIDE", + "text": "OneUp Chain Guide", + "price": 79, + "description": "A chain guide for mountain bikes from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/components", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "chain-guide" + ], + "subCategoryCode": "CG", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-BASH-GUIDE", + "text": "OneUp Bash Guide", + "price": 89, + "description": "A bash guide for mountain bikes from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/components", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "drivetrain", + "bash-guide" + ], + "subCategoryCode": "BG", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-V3-REMOTE", + "text": "OneUp V3 Dropper Remote", + "price": 59, + "description": "A handlebar remote control in OneUp Components' V3 dropper post system.", + "sourceUrl": "https://www.oneupcomponents.com/products/dropper-post-v3", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "dropper", + "remote" + ], + "subCategoryCode": "SP", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-THICK-GRIP", + "text": "OneUp Thick Lock-On Grips", + "price": 29, + "description": "Lock-on grips for mountain bike handlebars from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/grips", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "grips" + ], + "subCategoryCode": "HB", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-DIRECT-MOUNT-STEM", + "text": "OneUp Direct Mount Stem", + "price": 109, + "description": "A direct-mount stem for mountain bike handlebars from OneUp Components.", + "sourceUrl": "https://www.oneupcomponents.com/collections/components", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "stem" + ], + "subCategoryCode": "HB", + "brandCode": "ONEUP" + }, + { + "sku": "ONEUP-CARBON-EBAR", + "text": "OneUp Carbon E-Bar", + "price": 169, + "description": "A carbon handlebar from OneUp Components' E-Bar series for electric mountain bikes.", + "sourceUrl": "https://www.oneupcomponents.com/collections/handlebars", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "cockpit", + "handlebar", + "electric" + ], + "subCategoryCode": "HB", + "brandCode": "ONEUP" + }, + { + "sku": "FOXRACING-UNION-FLAT-SHOE", + "text": "Fox Racing Union Flat Mountain Bike Shoe", + "price": 179, + "description": "A flat-pedal mountain bike shoe in Fox Racing's Union footwear family.", + "sourceUrl": "https://www.foxracing.com/product/fox-union-flat-shoes/29354.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "shoe", + "flat-pedal" + ], + "subCategoryCode": "SH", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-UNION-BOA-FLAT", + "text": "Fox Racing Union BOA Flat Mountain Bike Shoe", + "price": 249, + "description": "A flat-pedal mountain bike shoe with BOA closure in Fox Racing's Union footwear family.", + "sourceUrl": "https://www.foxracing.com/products/union-boa-flat-shoes/32347.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "shoe", + "flat-pedal" + ], + "subCategoryCode": "SH", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-UNION-CLIPLESS", + "text": "Fox Racing Union Clipless Mountain Bike Shoe", + "price": 229, + "description": "A clipless mountain bike shoe in Fox Racing's Union footwear family.", + "sourceUrl": "https://www.foxracing.com/products/union-clipless-shoes/31948.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "shoe", + "clipless" + ], + "subCategoryCode": "SH", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-UNION-BOA-CLIPLESS", + "text": "Fox Racing Union BOA Clipless Mountain Bike Shoe", + "price": 279, + "description": "A clipless mountain bike shoe with BOA closure in Fox Racing's Union footwear family.", + "sourceUrl": "https://www.foxracing.com/product/fox-union-boa-clipless-shoes/29353.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "shoe", + "clipless" + ], + "subCategoryCode": "SH", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-RANGER-SS-JERSEY", + "text": "Fox Racing Ranger TruDri Short Sleeve Jersey", + "price": 69, + "description": "A short-sleeve mountain bike jersey from Fox Racing's Ranger TruDri apparel family.", + "sourceUrl": "https://www.foxracing.com/product/ranger-trudri-short-sleeve-jersey/28874.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-RANGER-LS-JERSEY", + "text": "Fox Racing Ranger Long Sleeve Jersey", + "price": 79, + "description": "A long-sleeve mountain bike jersey from Fox Racing's Ranger apparel family.", + "sourceUrl": "https://www.foxracing.com/products/ranger-long-sleeve-jersey/27362.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-FLEXAIR-PRO-JERSEY", + "text": "Fox Racing Flexair Pro Jersey", + "price": 99, + "description": "A mountain bike jersey from Fox Racing's Flexair Pro apparel family.", + "sourceUrl": "https://www.foxracing.com/product/flexair-pro-jersey/31296.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-FLEXAIR-SS-JERSEY", + "text": "Fox Racing Flexair Short Sleeve Jersey", + "price": 89, + "description": "A short-sleeve mountain bike jersey from Fox Racing's Flexair apparel family.", + "sourceUrl": "https://www.foxracing.com/products/flexair-short-sleeve-jersey/28863.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-DEFEND-LS-JERSEY", + "text": "Fox Racing Defend Long Sleeve Jersey", + "price": 89, + "description": "A long-sleeve mountain bike jersey from Fox Racing's Defend apparel family.", + "sourceUrl": "https://www.foxracing.com/product/defend-long-sleeve-jersey/28864.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "jersey" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-RANGER-PANTS", + "text": "Fox Racing Ranger Mountain Bike Pants", + "price": 119, + "description": "Mountain bike riding pants in Fox Racing's Ranger apparel family.", + "sourceUrl": "https://www.foxracing.com/product/ranger-pants/28891.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "pants" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-FLEXAIR-PANTS", + "text": "Fox Racing Flexair Mountain Bike Pants", + "price": 169, + "description": "Mountain bike riding pants in Fox Racing's Flexair apparel family.", + "sourceUrl": "https://www.foxracing.com/product/flexair-pant/28886.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "pants" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-DEFEND-PANTS", + "text": "Fox Racing Defend Mountain Bike Pants", + "price": 159, + "description": "Mountain bike riding pants in Fox Racing's Defend apparel family.", + "sourceUrl": "https://www.foxracing.com/product/defend-pant/25929.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "pants" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-RANGER-SHORTS", + "text": "Fox Racing Ranger Mountain Bike Shorts", + "price": 89, + "description": "Mountain bike riding shorts in Fox Racing's Ranger apparel family.", + "sourceUrl": "https://www.foxracing.com/products/ranger-shorts/28882.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "shorts" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-FLEXAIR-SHORTS", + "text": "Fox Racing Flexair Mountain Bike Shorts", + "price": 129, + "description": "Mountain bike riding shorts in Fox Racing's Flexair apparel family.", + "sourceUrl": "https://www.foxracing.com/products/flexair-shorts/28881.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "shorts" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-DEFEND-SHORTS", + "text": "Fox Racing Defend Mountain Bike Shorts", + "price": 139, + "description": "Mountain bike riding shorts in Fox Racing's Defend apparel family.", + "sourceUrl": "https://www.foxracing.com/product/defend-shorts/28881.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "apparel", + "shorts" + ], + "subCategoryCode": "AP", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-RANGER-GLOVES", + "text": "Fox Racing Ranger Mountain Bike Gloves", + "price": 39, + "description": "Mountain bike riding gloves from Fox Racing's Ranger gear family.", + "sourceUrl": "https://www.foxracing.com/product/ranger-gloves/27162.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "gloves" + ], + "subCategoryCode": "GL", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-RANGER-GEL-GLOVES", + "text": "Fox Racing Ranger Gel Mountain Bike Gloves", + "price": 49, + "description": "Mountain bike riding gloves in Fox Racing's Ranger Gel family.", + "sourceUrl": "https://www.foxracing.com/products/ranger-gel-gloves/33607.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "gloves" + ], + "subCategoryCode": "GL", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-FLEXAIR-GLOVES", + "text": "Fox Racing Flexair Mountain Bike Gloves", + "price": 45, + "description": "Mountain bike riding gloves from Fox Racing's Flexair gear family.", + "sourceUrl": "https://www.foxracing.com/products/flexair-gloves/31308.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "gloves" + ], + "subCategoryCode": "GL", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-DEFEND-GLOVES", + "text": "Fox Racing Defend Mountain Bike Gloves", + "price": 49, + "description": "Mountain bike riding gloves from Fox Racing's Defend gear family.", + "sourceUrl": "https://www.foxracing.com/products/defend-gloves/33792.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "gloves" + ], + "subCategoryCode": "GL", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-SPEEDFRAME-HELMET", + "text": "Fox Racing Speedframe Mountain Bike Helmet", + "price": 179, + "description": "An open-face mountain bike helmet in Fox Racing's Speedframe family.", + "sourceUrl": "https://www.foxracing.com/product/speedframe-helmet/26712.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "helmet" + ], + "subCategoryCode": "HE", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-SPEEDFRAME-PRO", + "text": "Fox Racing Speedframe Pro Mountain Bike Helmet", + "price": 279, + "description": "An open-face mountain bike helmet in Fox Racing's Speedframe Pro family.", + "sourceUrl": "https://www.foxracing.com/product/speedframe-pro-helmet/25102.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "helmet" + ], + "subCategoryCode": "HE", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-DROPFRAME-PRO", + "text": "Fox Racing Dropframe Pro Mountain Bike Helmet", + "price": 299, + "description": "A mountain bike helmet in Fox Racing's Dropframe Pro family.", + "sourceUrl": "https://www.foxracing.com/products/dropframe-pro-helmet/31459.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "helmet" + ], + "subCategoryCode": "HE", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-PROFRAME-RS", + "text": "Fox Racing Proframe RS Mountain Bike Helmet", + "price": 449, + "description": "A full-face mountain bike helmet in Fox Racing's Proframe RS family.", + "sourceUrl": "https://www.foxracing.com/products/proframe-rs-helmet/32497.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "helmet", + "full-face" + ], + "subCategoryCode": "HE", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-LAUNCH-D3O-KNEE", + "text": "Fox Racing Launch D3O Knee Guard", + "price": 99, + "description": "A knee guard in Fox Racing's Launch D3O mountain bike protection family.", + "sourceUrl": "https://www.foxracing.com/product/launch-d3o-knee-guard/26430.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "protection", + "knee-guard" + ], + "subCategoryCode": "PR", + "brandCode": "FOXRACING" + }, + { + "sku": "FOXRACING-LAUNCH-PRO-D3O-ELBOW", + "text": "Fox Racing Launch Pro D3O Elbow Guard", + "price": 89, + "description": "An elbow guard in Fox Racing's Launch Pro D3O mountain bike protection range.", + "sourceUrl": "https://www.foxracing.com/products/launch-pro-d3o-elbow-guards/18495.html", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "protection", + "elbow-guard" + ], + "subCategoryCode": "PR", + "brandCode": "FOXRACING" + }, + { + "sku": "MUCOFF-DRY-CHAIN-LUBE", + "text": "Muc-Off Dry Chain Lube", + "price": 14, + "description": "A bicycle chain lubricant from Muc-Off for dry riding conditions.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "chain-lube", + "dry" + ], + "subCategoryCode": "LU", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-WET-CHAIN-LUBE", + "text": "Muc-Off Wet Chain Lube", + "price": 14, + "description": "A bicycle chain lubricant from Muc-Off for wet riding conditions.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "chain-lube", + "wet" + ], + "subCategoryCode": "LU", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-C3-CERAMIC-DRY-LUBE", + "text": "Muc-Off C3 Ceramic Dry Chain Lube", + "price": 28, + "description": "A ceramic bicycle chain lubricant from Muc-Off for dry conditions.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "chain-lube", + "dry" + ], + "subCategoryCode": "LU", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-C3-CERAMIC-WET-LUBE", + "text": "Muc-Off C3 Ceramic Wet Chain Lube", + "price": 28, + "description": "A ceramic bicycle chain lubricant from Muc-Off for wet conditions.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "chain-lube", + "wet" + ], + "subCategoryCode": "LU", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-BIO-DRIVETRAIN-CLEANER", + "text": "Muc-Off Bio Drivetrain Cleaner", + "price": 22, + "description": "A bicycle drivetrain cleaner from Muc-Off for cleaning grease and grime from drivetrain parts.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "cleaner", + "drivetrain" + ], + "subCategoryCode": "CL", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-DISC-BRAKE-CLEANER", + "text": "Muc-Off Disc Brake Cleaner", + "price": 16, + "description": "A cleaner from Muc-Off formulated for bicycle disc brake components.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "cleaner", + "brakes" + ], + "subCategoryCode": "CL", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-BIKE-PROTECT", + "text": "Muc-Off Bike Protect", + "price": 22, + "description": "A post-wash protection spray for bicycles from Muc-Off.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "protection-spray" + ], + "subCategoryCode": "MS", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-MO94", + "text": "Muc-Off MO-94 Maintenance Spray", + "price": 12, + "description": "A multi-purpose maintenance spray from Muc-Off for bicycle care.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "spray" + ], + "subCategoryCode": "MS", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-PREMIUM-BRUSH-SET", + "text": "Muc-Off Premium Brush Set", + "price": 35, + "description": "A bicycle cleaning brush set from Muc-Off for washing frames and components.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "brush" + ], + "subCategoryCode": "MS", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-CLAW-BRUSH", + "text": "Muc-Off Claw Brush", + "price": 12, + "description": "A bicycle cleaning brush from Muc-Off for drivetrain maintenance.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "brush", + "drivetrain" + ], + "subCategoryCode": "MS", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-NANO-TECH-CONCENTRATE", + "text": "Muc-Off Nano Tech Bike Cleaner Concentrate", + "price": 29, + "description": "A concentrated bicycle cleaner from Muc-Off for preparing bike wash solution.", + "sourceUrl": "https://muc-off.com/products/bike-cleaner", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "cleaner" + ], + "subCategoryCode": "CL", + "brandCode": "MUCOFF" + }, + { + "sku": "MUCOFF-TUBELESS-VALVE-KIT", + "text": "Muc-Off Tubeless Valve Kit", + "price": 34, + "description": "A tubeless valve kit from Muc-Off for bicycle wheel setups.", + "sourceUrl": "https://us.muc-off.com/collections/all-bicycle-products", + "sourceType": "manufacturer", + "tags": [ + "mountain-bike", + "maintenance", + "tubeless" + ], + "subCategoryCode": "MS", + "brandCode": "MUCOFF" + } ], "queries": [ { - "id": "tune-exact-yeti-asr", + "id": "tune-exact-yeti-asr", + "split": "tuning", + "kind": "exact-sku", + "query": "YETI-ASR-C2-2025", + "intent": "Exact evaluation-catalog SKU lookup for Yeti ASR.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "YETI-ASR-C2-2025", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "YETI-ASR-C2-2025" + }, + { + "id": "tune-exact-shimano-xt-derailleur", + "split": "tuning", + "kind": "exact-sku", + "query": "SHIMANO-XT-M8100-RD", + "intent": "Exact evaluation-catalog SKU lookup for Shimano XT rear derailleur.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "SHIMANO-XT-M8100-RD", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "SHIMANO-XT-M8100-RD" + }, + { + "id": "tune-exact-fox-fork", + "split": "tuning", + "kind": "exact-sku", + "query": "FOX-36-FACTORY-GRIP-X2", + "intent": "Exact evaluation-catalog SKU lookup for FOX 36 Factory fork.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOX-36-FACTORY-GRIP-X2", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "FOX-36-FACTORY-GRIP-X2" + }, + { + "id": "tune-exact-maxxis-assegai", + "split": "tuning", + "kind": "exact-sku", + "query": "MAXXIS-ASSEGAI", + "intent": "Exact evaluation-catalog SKU lookup for Maxxis Assegai tyre.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "MAXXIS-ASSEGAI", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "MAXXIS-ASSEGAI" + }, + { + "id": "tune-exact-foxracing-helmet", + "split": "tuning", + "kind": "exact-sku", + "query": "FOXRACING-SPEEDFRAME-PRO", + "intent": "Exact evaluation-catalog SKU lookup for Fox Racing Speedframe Pro helmet.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-SPEEDFRAME-PRO", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "FOXRACING-SPEEDFRAME-PRO" + }, + { + "id": "tune-exact-shoe-flat", + "split": "tuning", + "kind": "exact-sku", + "query": "FOXRACING-UNION-FLAT-SHOE", + "intent": "Exact evaluation-catalog SKU lookup for Fox Racing Union flat shoe.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-UNION-FLAT-SHOE", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "FOXRACING-UNION-FLAT-SHOE" + }, + { + "id": "tune-trail-bikes-singletrack", + "split": "tuning", + "kind": "terminology", + "query": "full-suspension trail bikes for varied singletrack", + "intent": "Full-suspension trail bikes named or described for varied trails, climbing or technical descending.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "YETI-SB120-C2", + "grade": 3, + "rationale": "Catalog identifies a full-suspension trail bike for singletrack or varied trail riding." + }, + { + "sku": "YETI-SB140-LR-C2", + "grade": 3, + "rationale": "Catalog identifies a full-suspension trail bike for singletrack or varied trail riding." + }, + { + "sku": "CANYON-SPECTRAL-CF8", + "grade": 3, + "rationale": "Catalog identifies a full-suspension trail bike for singletrack or varied trail riding." + }, + { + "sku": "CANYON-NEURON-CF7", + "grade": 3, + "rationale": "Catalog identifies a full-suspension trail bike for singletrack or varied trail riding." + }, + { + "sku": "SPEC-STUMPJUMPER-15-COMP", + "grade": 3, + "rationale": "Catalog identifies a full-suspension trail bike for singletrack or varied trail riding." + }, + { + "sku": "GIANT-TRANCE-X-ADVPRO29", + "grade": 3, + "rationale": "Catalog identifies a full-suspension trail bike for singletrack or varied trail riding." + }, + { + "sku": "YETI-SB135", + "grade": 3, + "rationale": "Catalog identifies a full-suspension trail bike for singletrack or varied trail riding." + }, + { + "sku": "GIANT-TRANCE", + "grade": 3, + "rationale": "Catalog identifies a full-suspension trail bike for singletrack or varied trail riding." + }, + { + "sku": "GIANT-STANCE", + "grade": 3, + "rationale": "Catalog identifies a full-suspension trail bike for singletrack or varied trail riding." + } + ] + }, + { + "id": "tune-cross-country-race-bikes", + "split": "tuning", + "kind": "terminology", + "query": "cross-country mountain bikes for fast race riding", + "intent": "Cross-country/race intent based on model family and catalog descriptions.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "YETI-ASR-C2-2025", + "grade": 3, + "rationale": "Catalog identifies this as a cross-country model or describes cross-country use." + }, + { + "sku": "CANYON-LUXTRAIL-CF6", + "grade": 3, + "rationale": "Catalog identifies this as a cross-country model or describes cross-country use." + }, + { + "sku": "GIANT-ANTHEM-ADV29-1", + "grade": 3, + "rationale": "Catalog identifies this as a cross-country model or describes cross-country use." + }, + { + "sku": "SPEC-CHISEL-COMP", + "grade": 3, + "rationale": "Catalog identifies this as a cross-country model or describes cross-country use." + }, + { + "sku": "YETI-ARC", + "grade": 3, + "rationale": "Catalog identifies this as a cross-country model or describes cross-country use." + }, + { + "sku": "CANYON-LUX-WORLD-CUP", + "grade": 3, + "rationale": "Catalog identifies this as a cross-country model or describes cross-country use." + }, + { + "sku": "SPECIALIZED-EPIC-WORLD-CUP", + "grade": 3, + "rationale": "Catalog identifies this as a cross-country model or describes cross-country use." + }, + { + "sku": "GIANT-ANTHEM", + "grade": 3, + "rationale": "Catalog identifies this as a cross-country model or describes cross-country use." + }, + { + "sku": "GIANT-FATHOM", + "grade": 3, + "rationale": "Catalog identifies this as a cross-country model or describes cross-country use." + } + ] + }, + { + "id": "tune-downhill-bikes", "split": "tuning", - "kind": "exact-sku", - "query": "YETI-ASR-C2-2025", - "expectedTop1Sku": "YETI-ASR-C2-2025", + "kind": "terminology", + "query": "downhill mountain bike for steep race tracks", + "intent": "Downhill-specific mountain bike request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Exact SKU lookup for the Yeti ASR C2 2025.", "judgments": [ - { "sku": "YETI-ASR-C2-2025", "grade": 3, "rationale": "Exact identifier match." } + { + "sku": "CANYON-SENDER-CFR", + "grade": 3, + "rationale": "Catalog identifies this model as a downhill mountain bike." + }, + { + "sku": "SPECIALIZED-DEMO", + "grade": 3, + "rationale": "Catalog identifies this model as a downhill mountain bike." + }, + { + "sku": "GIANT-GLORY", + "grade": 3, + "rationale": "Catalog identifies this model as a downhill mountain bike." + } ] }, { - "id": "tune-exact-shimano-derailleur", + "id": "tune-electric-mountain-bikes", "split": "tuning", - "kind": "exact-sku", - "query": "SHIMANO-XT-M8100-RD", - "expectedTop1Sku": "SHIMANO-XT-M8100-RD", + "kind": "terminology", + "query": "electric assist mountain bike for technical trails", + "intent": "Explicit request for electric-assist mountain bikes.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Exact SKU lookup for the Shimano XT rear derailleur.", "judgments": [ - { "sku": "SHIMANO-XT-M8100-RD", "grade": 3, "rationale": "Exact identifier match." } + { + "sku": "YETI-MTE-C2", + "grade": 3, + "rationale": "Catalog identifies this as an electric-assist mountain bike." + }, + { + "sku": "SPECIALIZED-TURBO-LEVO", + "grade": 3, + "rationale": "Catalog identifies this as an electric-assist mountain bike." + }, + { + "sku": "SPECIALIZED-TURBO-KENEVO", + "grade": 3, + "rationale": "Catalog identifies this as an electric-assist mountain bike." + } ] }, { - "id": "tune-exact-mucoff-cleaner", + "id": "tune-hardtail-mountain-bikes", "split": "tuning", - "kind": "exact-sku", - "query": "MUCOFF-NANO-CLEANER", - "expectedTop1Sku": "MUCOFF-NANO-CLEANER", + "kind": "terminology", + "query": "hardtail mountain bike for off-road trails", + "intent": "Hardtail mountain bike request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Exact SKU lookup for the Nano Tech bicycle cleaner.", "judgments": [ - { "sku": "MUCOFF-NANO-CLEANER", "grade": 3, "rationale": "Exact identifier match." } + { + "sku": "SPEC-CHISEL-COMP", + "grade": 3, + "rationale": "Catalog identifies this model as a hardtail mountain bike." + }, + { + "sku": "YETI-ARC", + "grade": 3, + "rationale": "Catalog identifies this model as a hardtail mountain bike." + }, + { + "sku": "CANYON-STOIC", + "grade": 3, + "rationale": "Catalog identifies this model as a hardtail mountain bike." + }, + { + "sku": "CANYON-GRAND-CANYON", + "grade": 3, + "rationale": "Catalog identifies this model as a hardtail mountain bike." + }, + { + "sku": "SPECIALIZED-FUSE", + "grade": 3, + "rationale": "Catalog identifies this model as a hardtail mountain bike." + }, + { + "sku": "GIANT-FATHOM", + "grade": 3, + "rationale": "Catalog identifies this model as a hardtail mountain bike." + } ] }, { - "id": "tune-paraphrase-muddy-bike-cleaner", + "id": "tune-freeride-bike-park", "split": "tuning", - "kind": "paraphrase", - "query": "something to clean mud off my bike", + "kind": "terminology", + "query": "mountain bike for freeride and bike park riding", + "intent": "Freeride/park intent matched to named model-family use.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "A user describes the cleaning task rather than the product name.", "judgments": [ - { "sku": "MUCOFF-NANO-CLEANER", "grade": 3, "rationale": "Specifically cleans bicycle dirt and grime after muddy rides." }, - { "sku": "MUCOFF-DRIVETRAIN-CLEANER", "grade": 2, "rationale": "Cleans bicycle grime, but is specialized for drivetrain parts rather than the whole bike." } + { + "sku": "CANYON-TORQUE-MULLET-CF8", + "grade": 3, + "rationale": "Catalog identifies this model with freeride, park or jump-oriented use." + }, + { + "sku": "SPEC-STATUS-140", + "grade": 3, + "rationale": "Catalog identifies this model with freeride, park or jump-oriented use." + }, + { + "sku": "SPECIALIZED-STATUS-2", + "grade": 3, + "rationale": "Catalog identifies this model with freeride, park or jump-oriented use." + } ] }, { - "id": "tune-terminology-changing-gears", + "id": "tune-rear-derailleurs", "split": "tuning", "kind": "terminology", - "query": "a component for changing gears", + "query": "mountain bike rear derailleur", + "intent": "Direct component-type request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Broad functional terminology for drivetrain components.", "judgments": [ - { "sku": "SHIMANO-XT-M8100-RD", "grade": 3, "rationale": "A rear derailleur directly moves the chain to change gears." }, - { "sku": "SRAM-GX-EAGLE-RD", "grade": 3, "rationale": "A rear derailleur directly moves the chain to change gears." }, - { "sku": "SRAM-X0-TRANSMISSION-RD", "grade": 3, "rationale": "An electronic rear derailleur directly changes gears." }, - { "sku": "SHIMANO-SLX-M7100-CASSETTE", "grade": 2, "rationale": "The cassette supplies the selectable sprockets used when changing gears." }, - { "sku": "SRAM-XX1-EAGLE-CASSETTE", "grade": 2, "rationale": "The cassette supplies the selectable sprockets used when changing gears." }, - { "sku": "SHIMANO-XTR-M9100-CRANK", "grade": 1, "rationale": "The crankset is a drivetrain component but does not itself select gears." } + { + "sku": "SHIMANO-XT-M8100-RD", + "grade": 3, + "rationale": "This catalog item is explicitly a mountain bike rear derailleur." + }, + { + "sku": "SRAM-GX-EAGLE-RD", + "grade": 3, + "rationale": "This catalog item is explicitly a mountain bike rear derailleur." + }, + { + "sku": "SRAM-X0-TRANSMISSION-RD", + "grade": 3, + "rationale": "This catalog item is explicitly a mountain bike rear derailleur." + }, + { + "sku": "SHIMANO-SLX-M7100-DERAILLEUR", + "grade": 3, + "rationale": "This catalog item is explicitly a mountain bike rear derailleur." + }, + { + "sku": "SHIMANO-XTR-M9100-DERAILLEUR", + "grade": 3, + "rationale": "This catalog item is explicitly a mountain bike rear derailleur." + }, + { + "sku": "SHIMANO-DEORE-M6100-DERAILLEUR", + "grade": 3, + "rationale": "This catalog item is explicitly a mountain bike rear derailleur." + }, + { + "sku": "SRAM-X01-EAGLE-AXS-DERAILLEUR", + "grade": 3, + "rationale": "This catalog item is explicitly a mountain bike rear derailleur." + }, + { + "sku": "SRAM-XX1-EAGLE-AXS-DERAILLEUR", + "grade": 3, + "rationale": "This catalog item is explicitly a mountain bike rear derailleur." + }, + { + "sku": "SRAM-GX-TRANSMISSION-DERAILLEUR", + "grade": 3, + "rationale": "This catalog item is explicitly a mountain bike rear derailleur." + } ] }, { - "id": "tune-paraphrase-climbing-gears", + "id": "tune-mtb-shifters", "split": "tuning", - "kind": "paraphrase", - "query": "a wide range of low gears for climbing", + "kind": "terminology", + "query": "mountain bike trigger shifter for changing gears", + "intent": "Specific gear shifter request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "A climbing use case described without naming a cassette.", "judgments": [ - { "sku": "SHIMANO-SLX-M7100-CASSETTE", "grade": 3, "rationale": "Explicitly offers a wide range of gears for climbing." }, - { "sku": "SRAM-XX1-EAGLE-CASSETTE", "grade": 3, "rationale": "Explicitly offers a wide gear range for climbing." }, - { "sku": "SHIMANO-XT-M8100-RD", "grade": 1, "rationale": "A derailleur is part of a drivetrain, but its description does not establish a low climbing range." } + { + "sku": "SHIMANO-SLX-M7100-SHIFTER", + "grade": 3, + "rationale": "Catalog names this component as a mountain bike gear shifter." + }, + { + "sku": "SHIMANO-XT-M8100-SHIFTER", + "grade": 3, + "rationale": "Catalog names this component as a mountain bike gear shifter." + }, + { + "sku": "SHIMANO-XTR-M9100-SHIFTER", + "grade": 3, + "rationale": "Catalog names this component as a mountain bike gear shifter." + }, + { + "sku": "SRAM-GX-EAGLE-SHIFTER", + "grade": 3, + "rationale": "Catalog names this component as a mountain bike gear shifter." + } ] }, { - "id": "tune-paraphrase-tubeless-punctures", + "id": "tune-mountain-bike-cassettes", "split": "tuning", - "kind": "paraphrase", - "query": "liquid to seal small punctures in tubeless tires", + "kind": "terminology", + "query": "mountain bike replacement cassette", + "intent": "Generic cassette component request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Describes the purpose of a tubeless tire sealant without its product name.", "judgments": [ - { "sku": "MUCOFF-TUBELESS-SEALANT", "grade": 3, "rationale": "The product is a tubeless tyre sealant that helps seal punctures." } + { + "sku": "SHIMANO-SLX-M7100-CASSETTE", + "grade": 3, + "rationale": "Catalog identifies this as a mountain bike drivetrain cassette." + }, + { + "sku": "SRAM-XX1-EAGLE-CASSETTE", + "grade": 3, + "rationale": "Catalog identifies this as a mountain bike drivetrain cassette." + }, + { + "sku": "SHIMANO-XT-M8100-CASSETTE", + "grade": 3, + "rationale": "Catalog identifies this as a mountain bike drivetrain cassette." + }, + { + "sku": "SHIMANO-XTR-M9100-CASSETTE", + "grade": 3, + "rationale": "Catalog identifies this as a mountain bike drivetrain cassette." + }, + { + "sku": "SHIMANO-DEORE-M6100-CASSETTE", + "grade": 3, + "rationale": "Catalog identifies this as a mountain bike drivetrain cassette." + }, + { + "sku": "SRAM-GX-EAGLE-CASSETTE", + "grade": 3, + "rationale": "Catalog identifies this as a mountain bike drivetrain cassette." + }, + { + "sku": "SRAM-X01-EAGLE-CASSETTE", + "grade": 3, + "rationale": "Catalog identifies this as a mountain bike drivetrain cassette." + }, + { + "sku": "SRAM-XX-TRANSMISSION-CASSETTE", + "grade": 3, + "rationale": "Catalog identifies this as a mountain bike drivetrain cassette." + } ] }, { - "id": "tune-paraphrase-lower-saddle", + "id": "tune-hydraulic-disc-brakes", "split": "tuning", - "kind": "paraphrase", - "query": "a seat post that lowers the saddle for descents", + "kind": "terminology", + "query": "hydraulic disc brake for a mountain bike", + "intent": "Complete brake request; separate levers are not complete brakes.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "SHIMANO-XT-M8120-BRAKE", + "grade": 3, + "rationale": "Catalog identifies this as a hydraulic disc brake." + }, + { + "sku": "SHIMANO-XTR-M9100-BRAKE", + "grade": 3, + "rationale": "Catalog identifies this as a hydraulic disc brake." + }, + { + "sku": "SRAM-CODE-RSC-BRAKE", + "grade": 3, + "rationale": "Catalog identifies this as a hydraulic disc brake." + }, + { + "sku": "SRAM-MAVEN-ULTIMATE-BRAKE", + "grade": 3, + "rationale": "Catalog identifies this as a hydraulic disc brake." + }, + { + "sku": "SRAM-MAVEN-SILVER-BRAKE", + "grade": 3, + "rationale": "Catalog identifies this as a hydraulic disc brake." + }, + { + "sku": "SRAM-LEVEL-ULTIMATE-BRAKE", + "grade": 3, + "rationale": "Catalog identifies this as a hydraulic disc brake." + } + ] + }, + { + "id": "tune-mtb-chain-replacement", + "split": "tuning", + "kind": "terminology", + "query": "replacement bicycle chain for mountain bike drivetrain", + "intent": "Catalogued mountain bike chain request.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "SHIMANO-SLX-M7100-CHAIN", + "grade": 3, + "rationale": "Catalog identifies this as a chain in a mountain bike component family." + }, + { + "sku": "SHIMANO-XT-M8100-CHAIN", + "grade": 3, + "rationale": "Catalog identifies this as a chain in a mountain bike component family." + }, + { + "sku": "SHIMANO-XTR-M9100-CHAIN", + "grade": 3, + "rationale": "Catalog identifies this as a chain in a mountain bike component family." + }, + { + "sku": "SRAM-XX1-EAGLE-CHAIN", + "grade": 3, + "rationale": "Catalog identifies this as a chain in a mountain bike component family." + }, + { + "sku": "SRAM-GX-EAGLE-CHAIN", + "grade": 3, + "rationale": "Catalog identifies this as a chain in a mountain bike component family." + }, + { + "sku": "SRAM-XX-TRANSMISSION-CHAIN", + "grade": 3, + "rationale": "Catalog identifies this as a chain in a mountain bike component family." + } + ] + }, + { + "id": "tune-shimano-xt-components", + "split": "tuning", + "kind": "terminology", + "query": "Shimano Deore XT mountain bike components", + "intent": "Brand and component series request.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "SHIMANO-XT-M8100-RD", + "grade": 3, + "rationale": "Item is named as a Shimano Deore XT mountain bike component." + }, + { + "sku": "SHIMANO-XT-M8100-SHIFTER", + "grade": 3, + "rationale": "Item is named as a Shimano Deore XT mountain bike component." + }, + { + "sku": "SHIMANO-XT-M8100-CHAIN", + "grade": 3, + "rationale": "Item is named as a Shimano Deore XT mountain bike component." + }, + { + "sku": "SHIMANO-XT-M8100-CRANKSET", + "grade": 3, + "rationale": "Item is named as a Shimano Deore XT mountain bike component." + }, + { + "sku": "SHIMANO-XT-M8100-CASSETTE", + "grade": 3, + "rationale": "Item is named as a Shimano Deore XT mountain bike component." + }, + { + "sku": "SHIMANO-XT-M8120-BRAKE", + "grade": 3, + "rationale": "Item is named as a Shimano Deore XT mountain bike component." + }, + { + "sku": "SHIMANO-XT-M8100-BRAKE-LEVER", + "grade": 3, + "rationale": "Item is named as a Shimano Deore XT mountain bike component." + } + ] + }, + { + "id": "tune-fox-34-forks", + "split": "tuning", + "kind": "terminology", + "query": "FOX 34 mountain bike fork", + "intent": "Specific FOX 34 fork-family query.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOX-34-FACTORY", + "grade": 3, + "rationale": "Catalog names this as a FOX 34 suspension fork." + }, + { + "sku": "FOX-34-FACTORY-GRIP-X", + "grade": 3, + "rationale": "Catalog names this as a FOX 34 suspension fork." + }, + { + "sku": "FOX-34-PERFORMANCE-ELITE", + "grade": 3, + "rationale": "Catalog names this as a FOX 34 suspension fork." + } + ] + }, + { + "id": "tune-fox-gravity-forks", + "split": "tuning", + "kind": "terminology", + "query": "FOX fork for downhill and gravity riding", + "intent": "FOX gravity fork request; product labels limit claims to the model family.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOX-38-FACTORY", + "grade": 2, + "rationale": "FOX 38 is a related gravity-oriented fork family." + }, + { + "sku": "FOX-38-PERFORMANCE-ELITE", + "grade": 2, + "rationale": "FOX 38 is a related gravity-oriented fork family." + }, + { + "sku": "FOX-40-FACTORY", + "grade": 3, + "rationale": "This is explicitly named as a FOX 40 Factory Downhill Fork." + } + ] + }, + { + "id": "tune-fox-rear-shocks", + "split": "tuning", + "kind": "terminology", + "query": "FOX rear shock for mountain bike suspension", + "intent": "Specific FOX rear suspension component request.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOX-FLOAT-DPS-FACTORY", + "grade": 3, + "rationale": "Catalog identifies this as a FOX rear shock model." + }, + { + "sku": "FOX-FLOAT-X-FACTORY", + "grade": 3, + "rationale": "Catalog identifies this as a FOX rear shock model." + }, + { + "sku": "FOX-FLOAT-X2-FACTORY", + "grade": 3, + "rationale": "Catalog identifies this as a FOX rear shock model." + }, + { + "sku": "FOX-DHX-FACTORY", + "grade": 3, + "rationale": "Catalog identifies this as a FOX rear shock model." + } + ] + }, + { + "id": "tune-maxxis-mtb-tyres", + "split": "tuning", + "kind": "terminology", + "query": "Maxxis mountain bike tyres", + "intent": "Brand and product-type request; all listed Maxxis entries are MTB tyres.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "MAXXIS-ASSEGAI", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-MINION-DHF", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-MINION-DHR-II", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-DISSECTOR", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-REKON", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-REKON-RACE", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-FOREKASTER", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-HIGH-ROLLER-II", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-SHORTY", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-AGGRESSOR", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-ARDENT", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-ARDENT-RACE", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-IKON", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-ASPEN", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-ASPEN-ST", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-ASPEN-AT", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-CROSSMARK-II", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-MINION-SS", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-MINION-FBF", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + }, + { + "sku": "MAXXIS-MINION-FBR", + "grade": 3, + "rationale": "Catalog identifies this as a Maxxis mountain bike tyre." + } + ] + }, + { + "id": "tune-maxxis-fat-bike-tyres", + "split": "tuning", + "kind": "terminology", + "query": "Maxxis Minion fat bike tyres", + "intent": "Specific fat-bike tyre family. ", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "MAXXIS-MINION-FBF", + "grade": 3, + "rationale": "Name identifies a Minion FBF/FBR fat-bike tyre." + }, + { + "sku": "MAXXIS-MINION-FBR", + "grade": 3, + "rationale": "Name identifies a Minion FBF/FBR fat-bike tyre." + } + ] + }, + { + "id": "tune-oneup-v3-droppers", + "split": "tuning", + "kind": "terminology", + "query": "OneUp V3 dropper seatpost", + "intent": "Brand, series and type request.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "ONEUP-V3-DROPPER-180", + "grade": 3, + "rationale": "This is a OneUp V3 dropper post; travel variant is in its name." + }, + { + "sku": "ONEUP-V3-DROPPER-120", + "grade": 3, + "rationale": "This is a OneUp V3 dropper post; travel variant is in its name." + }, + { + "sku": "ONEUP-V3-DROPPER-210", + "grade": 3, + "rationale": "This is a OneUp V3 dropper post; travel variant is in its name." + }, + { + "sku": "ONEUP-V3-DROPPER-240", + "grade": 3, + "rationale": "This is a OneUp V3 dropper post; travel variant is in its name." + } + ] + }, + { + "id": "tune-oneup-handlebars", + "split": "tuning", + "kind": "terminology", + "query": "OneUp mountain bike handlebar", + "intent": "Brand and component request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Describes dropper-post behavior in everyday language.", "judgments": [ - { "sku": "ONEUP-V3-DROPPER-180", "grade": 3, "rationale": "The dropper post lowers the saddle for descending." }, - { "sku": "YETI-SB165-C3", "grade": 1, "rationale": "A downhill-oriented bike relates to the use case but is not the requested seat post." } + { + "sku": "ONEUP-CARBON-HANDLEBAR", + "grade": 3, + "rationale": "Catalog identifies this as a OneUp handlebar." + }, + { + "sku": "ONEUP-V2-CARBON-HANDLEBAR", + "grade": 3, + "rationale": "Catalog identifies this as a OneUp handlebar." + }, + { + "sku": "ONEUP-ALUMINUM-HANDLEBAR", + "grade": 3, + "rationale": "Catalog identifies this as a OneUp handlebar." + }, + { + "sku": "ONEUP-CARBON-EBAR", + "grade": 3, + "rationale": "Catalog identifies this as a OneUp handlebar." + } ] }, { - "id": "tune-terminology-reduce-hand-vibration", + "id": "tune-flat-pedals", "split": "tuning", "kind": "terminology", - "query": "handlebars that reduce hand fatigue on rough trails", + "query": "platform flat pedals for mountain biking", + "intent": "Direct platform-pedal request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Matches rider language to a product's vibration and fatigue benefit.", "judgments": [ - { "sku": "ONEUP-CARBON-HANDLEBAR", "grade": 3, "rationale": "Its shaped carbon profile is designed to absorb vibration and reduce hand fatigue." } + { + "sku": "ONEUP-ALUMINUM-PEDAL", + "grade": 3, + "rationale": "This is a OneUp aluminum mountain bike pedal." + } ] }, { - "id": "tune-paraphrase-trailside-repair", + "id": "tune-trailside-tools", "split": "tuning", "kind": "paraphrase", - "query": "a compact tool for fixing my bike on the trail", + "query": "compact multi tool for trailside bike repairs", + "intent": "Portable repair multi-tool request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "A use-case paraphrase for a portable repair multi-tool.", "judgments": [ - { "sku": "ONEUP-EDC-TOOL-V2", "grade": 3, "rationale": "A compact trailside repair tool with a chain breaker and tyre lever." } + { + "sku": "ONEUP-EDC-TOOL-V2", + "grade": 3, + "rationale": "Catalog identifies this as a compact OneUp EDC repair tool." + }, + { + "sku": "ONEUP-EDC-V2-TOOL", + "grade": 3, + "rationale": "Catalog identifies this as a compact OneUp EDC repair tool." + }, + { + "sku": "ONEUP-EDC-LITE-TOOL", + "grade": 3, + "rationale": "Catalog identifies this as a compact OneUp EDC repair tool." + } ] }, { - "id": "tune-no-current-bike-under-1000", + "id": "tune-oneup-mini-pumps", "split": "tuning", - "kind": "price-constraint", - "query": "a mountain bike under $1000", - "constraints": { "price": { "operator": "lt", "value": 1000 } }, + "kind": "terminology", + "query": "compact OneUp pump for trailside tyre inflation", + "intent": "Brand and portable pump request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "No currently eligible seeded mountain bike is priced below $1000; the deleted Giant Talon is not eligible.", - "judgments": [] + "judgments": [ + { + "sku": "ONEUP-EDC-PUMP-100", + "grade": 3, + "rationale": "Catalog names this as a OneUp EDC pump; size variant is in the name." + }, + { + "sku": "ONEUP-EDC-PUMP-70", + "grade": 3, + "rationale": "Catalog names this as a OneUp EDC pump; size variant is in the name." + } + ] + }, + { + "id": "tune-mtb-gloves", + "split": "tuning", + "kind": "terminology", + "query": "Fox Racing mountain bike gloves", + "intent": "Brand, activity and apparel type. ", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-RANGER-GLOVES", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike glove." + }, + { + "sku": "FOXRACING-RANGER-GEL-GLOVES", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike glove." + }, + { + "sku": "FOXRACING-FLEXAIR-GLOVES", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike glove." + }, + { + "sku": "FOXRACING-DEFEND-GLOVES", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike glove." + } + ] + }, + { + "id": "tune-mtb-jerseys", + "split": "tuning", + "kind": "terminology", + "query": "Fox Racing mountain bike riding jersey", + "intent": "Brand, activity and apparel type. ", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-RANGER-SS-JERSEY", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike jersey." + }, + { + "sku": "FOXRACING-RANGER-LS-JERSEY", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike jersey." + }, + { + "sku": "FOXRACING-FLEXAIR-PRO-JERSEY", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike jersey." + }, + { + "sku": "FOXRACING-FLEXAIR-SS-JERSEY", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike jersey." + }, + { + "sku": "FOXRACING-DEFEND-LS-JERSEY", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike jersey." + } + ] + }, + { + "id": "tune-mtb-riding-bottoms", + "split": "tuning", + "kind": "terminology", + "query": "Fox Racing mountain bike riding pants or shorts", + "intent": "Brand and MTB apparel request.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-RANGER-PANTS", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike pant or short." + }, + { + "sku": "FOXRACING-FLEXAIR-PANTS", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike pant or short." + }, + { + "sku": "FOXRACING-DEFEND-PANTS", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike pant or short." + }, + { + "sku": "FOXRACING-RANGER-SHORTS", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike pant or short." + }, + { + "sku": "FOXRACING-FLEXAIR-SHORTS", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike pant or short." + }, + { + "sku": "FOXRACING-DEFEND-SHORTS", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike pant or short." + } + ] + }, + { + "id": "tune-mtb-helmets", + "split": "tuning", + "kind": "terminology", + "query": "Fox Racing mountain bike helmet", + "intent": "Brand and protective-equipment type request.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-SPEEDFRAME-HELMET", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike helmet." + }, + { + "sku": "FOXRACING-SPEEDFRAME-PRO", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike helmet." + }, + { + "sku": "FOXRACING-DROPFRAME-PRO", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike helmet." + }, + { + "sku": "FOXRACING-PROFRAME-RS", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike helmet." + } + ] + }, + { + "id": "tune-knee-elbow-protection", + "split": "tuning", + "kind": "terminology", + "query": "mountain bike knee and elbow guards", + "intent": "Direct request for MTB body protection.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-LAUNCH-D3O-KNEE", + "grade": 3, + "rationale": "Catalog names this as a Fox Racing MTB knee or elbow guard." + }, + { + "sku": "FOXRACING-LAUNCH-PRO-D3O-ELBOW", + "grade": 3, + "rationale": "Catalog names this as a Fox Racing MTB knee or elbow guard." + } + ] + }, + { + "id": "tune-flat-pedal-shoes", + "split": "tuning", + "kind": "terminology", + "query": "flat pedal mountain bike shoes", + "intent": "Flat-pedal shoes; clipless models are excluded.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-UNION-FLAT-SHOE", + "grade": 3, + "rationale": "Catalog identifies this Union shoe as a flat-pedal shoe." + }, + { + "sku": "FOXRACING-UNION-BOA-FLAT", + "grade": 3, + "rationale": "Catalog identifies this Union shoe as a flat-pedal shoe." + } + ] + }, + { + "id": "tune-clipless-shoes", + "split": "tuning", + "kind": "terminology", + "query": "clipless mountain bike shoes for pedal cleats", + "intent": "Clipless shoes; flat-pedal models are excluded.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-UNION-CLIPLESS", + "grade": 3, + "rationale": "Catalog identifies this Union shoe as a clipless model." + }, + { + "sku": "FOXRACING-UNION-BOA-CLIPLESS", + "grade": 3, + "rationale": "Catalog identifies this Union shoe as a clipless model." + } + ] }, { - "id": "tune-terminology-cross-country-race", + "id": "tune-dry-condition-chain-lube", "split": "tuning", "kind": "terminology", - "query": "a fast cross-country bike for racing", + "query": "dry chain lubricant for mountain biking", + "intent": "Dry-condition chain lubricant request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Cross-country race terminology matched to seeded bike descriptions.", "judgments": [ - { "sku": "YETI-ASR-C2-2025", "grade": 3, "rationale": "Built for efficient climbing and fast technical singletrack." }, - { "sku": "GIANT-ANTHEM-ADV29-1", "grade": 3, "rationale": "Designed for efficient pedalling and demanding race courses." }, - { "sku": "SPEC-CHISEL-COMP", "grade": 2, "rationale": "An efficient cross-country race bike, though aluminium rather than carbon." }, - { "sku": "CANYON-LUXTRAIL-CF6", "grade": 2, "rationale": "Combines cross-country efficiency with trail capability, but is framed for long rides rather than racing." } + { + "sku": "MUCOFF-DRY-CHAIN-LUBE", + "grade": 3, + "rationale": "Product name identifies this as dry chain lubricant." + }, + { + "sku": "MUCOFF-C3-CERAMIC-DRY-LUBE", + "grade": 3, + "rationale": "Product name identifies this as dry chain lubricant." + } ] }, { - "id": "tune-broad-steep-bike-park", + "id": "tune-wet-condition-chain-lube", "split": "tuning", - "kind": "broad", - "query": "a bike for steep rough descents and bike-park riding", + "kind": "terminology", + "query": "wet chain lubricant for muddy rides", + "intent": "Wet-condition chain lubricant request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Broad descending intent with several suitable bike categories.", "judgments": [ - { "sku": "YETI-SB165-C3", "grade": 3, "rationale": "Designed for gravity riding, steep descents and bike-park terrain." }, - { "sku": "CANYON-SENDER-CFR", "grade": 3, "rationale": "A downhill bike for steep tracks and bike-park terrain." }, - { "sku": "SPEC-ENDURO-PRO", "grade": 3, "rationale": "Built for steep, rough descents and enduro racing." }, - { "sku": "CANYON-TORQUE-MULLET-CF8", "grade": 2, "rationale": "Designed for freeride and bike-park use, but not specifically downhill racing." }, - { "sku": "SPEC-STATUS-140", "grade": 2, "rationale": "Designed for jumps and technical descents, but is a more playful trail bike." }, - { "sku": "YETI-SB140-LR-C2", "grade": 2, "rationale": "Built for rough, fast descents, though its stated use is trail riding." } + { + "sku": "MUCOFF-WET-CHAIN-LUBE", + "grade": 3, + "rationale": "Product name identifies this as wet chain lubricant." + }, + { + "sku": "MUCOFF-C3-CERAMIC-WET-LUBE", + "grade": 3, + "rationale": "Product name identifies this as wet chain lubricant." + } ] }, { - "id": "tune-paraphrase-drivetrain-degreaser", + "id": "tune-whole-bike-cleaner", "split": "tuning", "kind": "paraphrase", - "query": "degreaser for a greasy chain and cassette", + "query": "clean dirt and mud from the whole bicycle", + "intent": "General bicycle cleaner, not a drivetrain-only degreaser.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Names drivetrain parts and soil rather than the cleaner product name.", "judgments": [ - { "sku": "MUCOFF-DRIVETRAIN-CLEANER", "grade": 3, "rationale": "Removes grease and grime from chains, cassettes and other drivetrain parts." }, - { "sku": "MUCOFF-NANO-CLEANER", "grade": 1, "rationale": "A general bicycle cleaner, but not specifically a drivetrain degreaser." } + { + "sku": "MUCOFF-NANO-CLEANER", + "grade": 3, + "rationale": "Whole-bike spray cleaner for dirt and grime." + }, + { + "sku": "MUCOFF-NANO-TECH-CONCENTRATE", + "grade": 3, + "rationale": "Concentrate for Muc-Off Nano Tech bicycle cleaner." + }, + { + "sku": "MUCOFF-BIO-DRIVETRAIN-CLEANER", + "grade": 1, + "rationale": "Drivetrain-specific cleaner, not general whole-bike washing." + } ] }, { - "id": "tune-price-derailleur-under-150", + "id": "tune-drivetrain-degreaser", "split": "tuning", - "kind": "price-constraint", - "query": "a rear derailleur for a 12-speed mountain bike under $150", - "constraints": { "price": { "operator": "lt", "value": 150 } }, + "kind": "paraphrase", + "query": "degreaser for grease on the bicycle chain and cassette", + "intent": "Drivetrain-specific cleaner request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Hard price ceiling and product type; two seeded derailleurs satisfy both.", "judgments": [ - { "sku": "SHIMANO-XT-M8100-RD", "grade": 3, "rationale": "A 12-speed mountain bike rear derailleur priced at $129." }, - { "sku": "SRAM-GX-EAGLE-RD", "grade": 3, "rationale": "A 12-speed mountain bike rear derailleur priced at $135." } + { + "sku": "MUCOFF-DRIVETRAIN-CLEANER", + "grade": 3, + "rationale": "Removes grease and grime from chains, cassettes and drivetrain parts." + }, + { + "sku": "MUCOFF-BIO-DRIVETRAIN-CLEANER", + "grade": 3, + "rationale": "Muc-Off drivetrain cleaner; direct product-type match." + } ] }, { - "id": "tune-price-trail-bike-under-5000", + "id": "tune-tubeless-puncture-sealant", "split": "tuning", - "kind": "price-constraint", - "query": "a trail bike for technical descents under $5000", - "constraints": { "price": { "operator": "lt", "value": 5000 } }, + "kind": "paraphrase", + "query": "liquid to seal punctures in a tubeless mountain bike tyre", + "intent": "Tubeless tyre puncture sealant request.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "MUCOFF-TUBELESS-SEALANT", + "grade": 3, + "rationale": "This is a tubeless MTB tyre sealant." + } + ] + }, + { + "id": "tune-disc-brake-cleaner", + "split": "tuning", + "kind": "terminology", + "query": "cleaner made for bicycle disc brake rotors", + "intent": "Specific bicycle disc-brake cleaning product.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Filters to affordable seeded bikes while preferring trail models suitable for technical descents.", "judgments": [ - { "sku": "CANYON-SPECTRAL-CF8", "grade": 3, "rationale": "A trail bike balancing climbing with control on rough technical descents, priced at $4999." }, - { "sku": "CANYON-NEURON-CF7", "grade": 3, "rationale": "A trail bike with stable descending over varied terrain, priced at $3499." }, - { "sku": "CANYON-LUXTRAIL-CF6", "grade": 2, "rationale": "A trail-capable cross-country bike for technical singletrack, priced at $3899." }, - { "sku": "CANYON-TORQUE-MULLET-CF8", "grade": 1, "rationale": "Capable on steep descents but oriented to freeride and bike parks rather than trail riding." } + { + "sku": "MUCOFF-DISC-BRAKE-CLEANER", + "grade": 3, + "rationale": "The catalog names this as a disc brake cleaner." + } ] }, { - "id": "tune-broad-everyday-singletrack", + "id": "tune-bike-protect-spray", "split": "tuning", - "kind": "broad", - "query": "a versatile mountain bike for after-work singletrack", + "kind": "terminology", + "query": "spray to protect a mountain bike after washing", + "intent": "Protection or maintenance spray rather than a cleaner.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Ambiguous everyday trail request without a specific price, discipline or component.", "judgments": [ - { "sku": "YETI-SB120-C2", "grade": 2, "rationale": "A trail bike for efficient climbs and flowing singletrack descents." }, - { "sku": "CANYON-SPECTRAL-CF8", "grade": 2, "rationale": "A versatile trail bike for varied singletrack." }, - { "sku": "CANYON-NEURON-CF7", "grade": 2, "rationale": "A trail bike for long rides over varied terrain." }, - { "sku": "SPEC-STUMPJUMPER-15-COMP", "grade": 2, "rationale": "Balances climbing efficiency and technical trail control." }, - { "sku": "GIANT-TRANCE-X-ADVPRO29", "grade": 2, "rationale": "A trail bike designed for climbing and technical descending." }, - { "sku": "SPEC-CHISEL-COMP", "grade": 1, "rationale": "A cross-country bike suitable for fast singletrack but less versatile on technical trails." }, - { "sku": "YETI-ASR-C2-2025", "grade": 1, "rationale": "An efficient cross-country bike, narrower in purpose than the broad trail request." } + { + "sku": "MUCOFF-BIKE-PROTECT", + "grade": 3, + "rationale": "The catalog identifies this as Muc-Off Bike Protect." + }, + { + "sku": "MUCOFF-MO94", + "grade": 2, + "rationale": "MO-94 is a related maintenance spray, less specific than Bike Protect." + } ] }, { - "id": "tune-broad-weekend-ride-kit", + "id": "tune-tubeless-valve-kit", "split": "tuning", - "kind": "broad", - "query": "useful gear for a weekend ride on rough trails", + "kind": "terminology", + "query": "replacement tubeless tyre valve kit", + "intent": "Direct request for the tubeless valve kit.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Intentionally broad request that could mean a trail accessory or repair item.", "judgments": [ - { "sku": "ONEUP-EDC-TOOL-V2", "grade": 2, "rationale": "A compact trailside tool is useful for repairs during a ride." }, - { "sku": "ONEUP-V3-DROPPER-180", "grade": 1, "rationale": "A dropper supports descending, but the request does not specifically ask for a bike component." }, - { "sku": "ONEUP-CARBON-HANDLEBAR", "grade": 1, "rationale": "A trail riding component, but not an obvious general-purpose ride kit item." }, - { "sku": "MUCOFF-TUBELESS-SEALANT", "grade": 1, "rationale": "A repair-related consumable, though it is installed in a tyre rather than carried as general kit." } + { + "sku": "MUCOFF-TUBELESS-VALVE-KIT", + "grade": 3, + "rationale": "The catalog identifies this as a tubeless valve kit." + } ] }, { - "id": "holdout-exact-sram-x0-derailleur", + "id": "tune-no-bike-under-1000", + "split": "tuning", + "kind": "price-constraint", + "query": "mountain bike under $1000", + "intent": "No eligible MTB listing is priced below $1000; bike entries exceed the ceiling.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [], + "constraints": { + "price": { + "operator": "lt", + "value": 1000 + } + } + }, + { + "id": "tune-hardtail-under-2000", + "split": "tuning", + "kind": "price-constraint", + "query": "hardtail mountain bike under $2000", + "intent": "Price-constrained hardtail request; other bike styles and hardtails over the ceiling are excluded.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "CANYON-STOIC", + "grade": 3, + "rationale": "Catalogued hardtail below $2000 (illustrative USD)." + }, + { + "sku": "CANYON-GRAND-CANYON", + "grade": 3, + "rationale": "Catalogued hardtail below $2000 (illustrative USD)." + }, + { + "sku": "GIANT-FATHOM", + "grade": 3, + "rationale": "Catalogued hardtail below $2000 (illustrative USD)." + } + ], + "constraints": { + "price": { + "operator": "lt", + "value": 2000 + } + } + }, + { + "id": "tune-derailleur-under-150", + "split": "tuning", + "kind": "price-constraint", + "query": "mountain bike rear derailleur under $150", + "intent": "Type and hard price ceiling; unrelated drivetrain parts are excluded.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "SHIMANO-XT-M8100-RD", + "grade": 3, + "rationale": "Catalogued rear derailleur below $150 (illustrative USD)." + }, + { + "sku": "SRAM-GX-EAGLE-RD", + "grade": 3, + "rationale": "Catalogued rear derailleur below $150 (illustrative USD)." + }, + { + "sku": "SHIMANO-SLX-M7100-DERAILLEUR", + "grade": 3, + "rationale": "Catalogued rear derailleur below $150 (illustrative USD)." + }, + { + "sku": "SHIMANO-DEORE-M6100-DERAILLEUR", + "grade": 3, + "rationale": "Catalogued rear derailleur below $150 (illustrative USD)." + } + ], + "constraints": { + "price": { + "operator": "lt", + "value": 150 + } + } + }, + { + "id": "tune-no-e-bike-charger", + "split": "tuning", + "kind": "out-of-catalog", + "query": "charger for an electric mountain bike battery", + "intent": "No battery chargers are catalogued; do not substitute an e-bike or unrelated part.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [] + }, + { + "id": "holdout-exact-sram-xx1-axs", "split": "held-out", "kind": "exact-sku", - "query": "SRAM-X0-TRANSMISSION-RD", - "expectedTop1Sku": "SRAM-X0-TRANSMISSION-RD", + "query": "SRAM-XX1-EAGLE-AXS-DERAILLEUR", + "intent": "Exact evaluation-catalog SKU lookup for SRAM XX1 Eagle AXS rear derailleur.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Held-out exact SKU lookup for the SRAM X0 electronic rear derailleur.", "judgments": [ - { "sku": "SRAM-X0-TRANSMISSION-RD", "grade": 3, "rationale": "Exact identifier match." } - ] + { + "sku": "SRAM-XX1-EAGLE-AXS-DERAILLEUR", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "SRAM-XX1-EAGLE-AXS-DERAILLEUR" + }, + { + "id": "holdout-exact-maxxis-dhr-ii", + "split": "held-out", + "kind": "exact-sku", + "query": "MAXXIS-MINION-DHR-II", + "intent": "Exact evaluation-catalog SKU lookup for Maxxis Minion DHR II tyre.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "MAXXIS-MINION-DHR-II", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "MAXXIS-MINION-DHR-II" + }, + { + "id": "holdout-exact-fox-shock", + "split": "held-out", + "kind": "exact-sku", + "query": "FOX-FLOAT-X2-FACTORY", + "intent": "Exact evaluation-catalog SKU lookup for FOX Float X2 rear shock.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOX-FLOAT-X2-FACTORY", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "FOX-FLOAT-X2-FACTORY" + }, + { + "id": "holdout-exact-clipless-shoe", + "split": "held-out", + "kind": "exact-sku", + "query": "FOXRACING-UNION-CLIPLESS", + "intent": "Exact evaluation-catalog SKU lookup for Fox Racing Union clipless shoe.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "FOXRACING-UNION-CLIPLESS", + "grade": 3, + "rationale": "Exact identifier match to the catalog entry." + } + ], + "expectedTop1Sku": "FOXRACING-UNION-CLIPLESS" }, { - "id": "holdout-terminology-workshop-labour", + "id": "holdout-helmet-for-trail-riding", "split": "held-out", "kind": "terminology", - "query": "an hourly workshop charge for a bicycle tune-up", + "query": "mountain bike helmet for trail riding", + "intent": "Request for MTB helmets, not other protective equipment.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Held-out service request for workshop labour rather than a stocked physical product.", "judgments": [ - { "sku": "LABOR", "grade": 3, "rationale": "The catalog entry is hourly workshop labour for bicycle maintenance and repair." } + { + "sku": "FOXRACING-SPEEDFRAME-HELMET", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike helmet." + }, + { + "sku": "FOXRACING-SPEEDFRAME-PRO", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike helmet." + }, + { + "sku": "FOXRACING-DROPFRAME-PRO", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike helmet." + }, + { + "sku": "FOXRACING-PROFRAME-RS", + "grade": 3, + "rationale": "This is a Fox Racing mountain bike helmet." + } ] }, { - "id": "holdout-price-carbon-bike-under-4000", + "id": "holdout-entry-hardtail-under-2000", "split": "held-out", "kind": "price-constraint", - "query": "a carbon mountain bike for long rides under $4000", - "constraints": { "price": { "operator": "lt", "value": 4000 } }, + "query": "entry-level hardtail mountain bike below $2000", + "intent": "Illustrative prices are not market-position claims; judged only by hardtail type and price ceiling.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "CANYON-STOIC", + "grade": 3, + "rationale": "Catalogued hardtail below $2000 (illustrative USD)." + }, + { + "sku": "CANYON-GRAND-CANYON", + "grade": 3, + "rationale": "Catalogued hardtail below $2000 (illustrative USD)." + }, + { + "sku": "GIANT-FATHOM", + "grade": 3, + "rationale": "Catalogued hardtail below $2000 (illustrative USD)." + } + ], + "constraints": { + "price": { + "operator": "lt", + "value": 2000 + } + } + }, + { + "id": "holdout-hydraulic-brake-parts", + "split": "held-out", + "kind": "terminology", + "query": "hydraulic mountain bike disc brakes and levers", + "intent": "Complete hydraulic brakes and the separately named lever match the parts request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "Held-out structured price ceiling with a long-ride carbon-bike intent.", "judgments": [ - { "sku": "CANYON-LUXTRAIL-CF6", "grade": 3, "rationale": "A carbon full-suspension bike explicitly described for long rides, priced at $3899." }, - { "sku": "CANYON-NEURON-CF7", "grade": 3, "rationale": "A carbon full-suspension trail bike explicitly designed for long rides, priced at $3499." }, - { "sku": "CANYON-TORQUE-MULLET-CF8", "grade": 1, "rationale": "A carbon bike under the price limit, but its freeride and bike-park focus is a poor long-ride fit." } + { + "sku": "SHIMANO-XT-M8120-BRAKE", + "grade": 3, + "rationale": "This is a hydraulic mountain bike disc brake." + }, + { + "sku": "SHIMANO-XTR-M9100-BRAKE", + "grade": 3, + "rationale": "This is a hydraulic mountain bike disc brake." + }, + { + "sku": "SRAM-CODE-RSC-BRAKE", + "grade": 3, + "rationale": "This is a hydraulic mountain bike disc brake." + }, + { + "sku": "SRAM-MAVEN-ULTIMATE-BRAKE", + "grade": 3, + "rationale": "This is a hydraulic mountain bike disc brake." + }, + { + "sku": "SRAM-MAVEN-SILVER-BRAKE", + "grade": 3, + "rationale": "This is a hydraulic mountain bike disc brake." + }, + { + "sku": "SRAM-LEVEL-ULTIMATE-BRAKE", + "grade": 3, + "rationale": "This is a hydraulic mountain bike disc brake." + }, + { + "sku": "SHIMANO-XT-M8100-BRAKE-LEVER", + "grade": 2, + "rationale": "Related XT hydraulic brake lever, not a complete brake." + } ] }, { - "id": "holdout-no-electric-bike-charger", + "id": "holdout-xt-brake-lever", "split": "held-out", - "kind": "out-of-catalog", - "query": "a charger for the battery on an electric mountain bike", + "kind": "terminology", + "query": "Shimano Deore XT M8100 brake lever", + "intent": "Specific brake-lever replacement request.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "The catalog includes an electric mountain bike but no battery charger accessory.", - "judgments": [] + "judgments": [ + { + "sku": "SHIMANO-XT-M8100-BRAKE-LEVER", + "grade": 3, + "rationale": "Product type and Deore XT M8100 series match." + } + ] + }, + { + "id": "holdout-wet-ride-maintenance", + "split": "held-out", + "kind": "paraphrase", + "query": "lubricant for a chain ridden in wet conditions", + "intent": "Wet-condition chain lubrication; cleaners and dry lubricants excluded.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "MUCOFF-WET-CHAIN-LUBE", + "grade": 3, + "rationale": "Product name identifies this as wet chain lubricant." + }, + { + "sku": "MUCOFF-C3-CERAMIC-WET-LUBE", + "grade": 3, + "rationale": "Product name identifies this as wet chain lubricant." + } + ] }, { - "id": "holdout-no-night-riding-lights", + "id": "holdout-no-gravel-tyres", "split": "held-out", "kind": "out-of-catalog", - "query": "USB rechargeable lights for night riding", + "query": "gravel bike tyres for mixed-surface road riding", + "intent": "The MTB-only catalog has no gravel tyres; MTB tyres do not satisfy the stated use.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "The reviewed eligible catalog contains no lights or night-riding electronics.", "judgments": [] }, { - "id": "holdout-no-city-bike-child-seat", + "id": "holdout-no-suspension-service-pump", "split": "held-out", "kind": "out-of-catalog", - "query": "a child seat for a city bicycle", + "query": "high pressure suspension shock pump", + "intent": "No suspension service pump is present; OneUp EDC mini-pumps are not suspension pumps.", "catalogReviewed": true, - "judgedCatalogCount": 29, + "judgedCatalogCount": 170, "omittedSkuGrade": 0, - "intent": "The reviewed eligible catalog contains no child seats or city-bike accessories.", "judgments": [] + }, + { + "id": "holdout-electric-mountain-bike-under-10000", + "split": "held-out", + "kind": "price-constraint", + "query": "electric mountain bike under $10000", + "intent": "All catalogued electric MTBs fall below this illustrative price ceiling.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "YETI-MTE-C2", + "grade": 3, + "rationale": "Electric-assist mountain bike under $10000 (illustrative USD)." + }, + { + "sku": "SPECIALIZED-TURBO-LEVO", + "grade": 3, + "rationale": "Electric-assist mountain bike under $10000 (illustrative USD)." + }, + { + "sku": "SPECIALIZED-TURBO-KENEVO", + "grade": 3, + "rationale": "Electric-assist mountain bike under $10000 (illustrative USD)." + } + ], + "constraints": { + "price": { + "operator": "lt", + "value": 10000 + } + } + }, + { + "id": "tune-informal-yeti-dentist-bike-alias", + "split": "tuning", + "kind": "terminology", + "query": "Yeti dentist bike", + "intent": "User-provided informal demo search alias for Yeti MTB listings; it does not assert intended riders or affordability.", + "catalogReviewed": true, + "judgedCatalogCount": 170, + "omittedSkuGrade": 0, + "judgments": [ + { + "sku": "YETI-ASR-C2-2025", + "grade": 3, + "rationale": "Yeti listing carries the explicitly curated informal dentist-bike search alias; this is not a claim about its riders or price." + }, + { + "sku": "YETI-SB120-C2", + "grade": 3, + "rationale": "Yeti listing carries the explicitly curated informal dentist-bike search alias; this is not a claim about its riders or price." + }, + { + "sku": "YETI-SB140-LR-C2", + "grade": 3, + "rationale": "Yeti listing carries the explicitly curated informal dentist-bike search alias; this is not a claim about its riders or price." + }, + { + "sku": "YETI-SB165-C3", + "grade": 3, + "rationale": "Yeti listing carries the explicitly curated informal dentist-bike search alias; this is not a claim about its riders or price." + }, + { + "sku": "YETI-MTE-C2", + "grade": 3, + "rationale": "Yeti listing carries the explicitly curated informal dentist-bike search alias; this is not a claim about its riders or price." + }, + { + "sku": "YETI-ARC", + "grade": 3, + "rationale": "Yeti listing carries the explicitly curated informal dentist-bike search alias; this is not a claim about its riders or price." + }, + { + "sku": "YETI-SB135", + "grade": 3, + "rationale": "Yeti listing carries the explicitly curated informal dentist-bike search alias; this is not a claim about its riders or price." + }, + { + "sku": "YETI-SB160", + "grade": 3, + "rationale": "Yeti listing carries the explicitly curated informal dentist-bike search alias; this is not a claim about its riders or price." + } + ] } - ] + ], + "priceCurrency": "USD", + "pricePolicy": "Illustrative demo prices only; not current retail prices.", + "sourcePolicy": "Manufacturer pages support all 142 expansion entries. The retained v1 catalog subset preserves two previously documented non-manufacturer references, explicitly marked retailer; BikesOnline is approved as a supplementary retailer source for additional research.", + "tagProvenancePolicy": { + "manufacturerFacts": "Product names, category tags and factual descriptions are supported by the per-product sourceUrl. Prices are illustrative USD.", + "colloquialAliases": "e-bike and emtb are user-requested colloquial search aliases for verified electric-assist mountain bikes; they are not extra specifications.", + "informalYetiAlias": "dentist-bike is a user-provided informal demo search alias on Yeti mountain-bike listings only. It is not a manufacturer fact, intended-rider claim, or affordability statement." + } } diff --git a/tools/product-search-relevance.ps1 b/tools/product-search-relevance.ps1 index 9f9df8c77..c7151e333 100644 --- a/tools/product-search-relevance.ps1 +++ b/tools/product-search-relevance.ps1 @@ -4,7 +4,7 @@ param( [uri] $OllamaAddress = 'http://localhost:11434/', [string] $DatasetPath = "$PSScriptRoot\product-search-relevance.dataset.json", [string] $OutputPath, - [ValidateSet('semantic', 'fulltext')][string] $Approach = 'semantic', + [ValidateSet('semantic', 'fulltext', 'hybrid')][string] $Approach = 'semantic', [ValidateSet('all', 'tuning', 'held-out')][string] $Split = 'all', [string] $BaselinePath ) @@ -21,7 +21,7 @@ function Get-ProductSearchDataset { function Test-ProductSearchDataset { param([Parameter(Mandatory)][object] $Dataset) - if ($Dataset.datasetVersion -ne 'product-search-relevance-v1') { throw 'Unsupported product-search relevance dataset version.' } + if ($Dataset.datasetVersion -ne 'product-search-relevance-v2-mtb') { throw 'Unsupported product-search relevance dataset version.' } if ($Dataset.profile -ne 'ollama:nomic-embed-text:v1.5:768:retrieval-prefix-v1' -or $Dataset.model -ne 'nomic-embed-text:v1.5' -or $Dataset.modelDigest -ne '0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f' -or @@ -34,18 +34,63 @@ function Test-ProductSearchDataset { if ($Dataset.catalogReviewed -ne $true -or [string]::IsNullOrWhiteSpace($Dataset.catalogEligibility)) { throw 'The eligible seeded catalog must be explicitly reviewed and described.' } - if (@($Dataset.queries).Count -lt 20 -or @($Dataset.queries).Count -gt 30) { - throw 'The curated evaluation must contain 20 to 30 queries.' + if ([string]::IsNullOrWhiteSpace($Dataset.referenceDataFile)) { throw 'The owning-domain reference-data seed must be pinned.' } + $referencePath = Join-Path $PSScriptRoot ("..\{0}" -f ($Dataset.referenceDataFile -replace '/', '\')) + if (!(Test-Path -LiteralPath $referencePath -PathType Leaf) -or + (Get-FileHash -LiteralPath $referencePath -Algorithm SHA256).Hash -ne $Dataset.referenceDataFileSha256) { + throw 'Reference-data seed drift: review canonical document labels and update the dataset pin intentionally.' + } + if ($Dataset.priceCurrency -ne 'USD' -or [string]::IsNullOrWhiteSpace($Dataset.pricePolicy) -or + [string]::IsNullOrWhiteSpace($Dataset.sourcePolicy) -or $null -eq $Dataset.tagProvenancePolicy) { + throw 'The catalog must document its illustrative price currency, source policy and tag provenance.' + } + if (@($Dataset.queries).Count -lt 50 -or @($Dataset.queries).Count -gt 60) { + throw 'The curated evaluation must contain 50 to 60 queries.' + } + if (@($Dataset.catalog).Count -lt 150 -or @($Dataset.catalog).Count -gt 200) { + throw 'The reviewed mountain-bike catalog must contain 150 to 200 products.' } - if (@($Dataset.catalog).Count -eq 0) { throw 'The reviewed catalog snapshot is empty.' } $catalogSkus = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) foreach ($product in $Dataset.catalog) { if ([string]::IsNullOrWhiteSpace($product.sku) -or [string]::IsNullOrWhiteSpace($product.text) -or $null -eq $product.price -or - [string]::IsNullOrWhiteSpace($product.description)) { - throw 'Every catalog snapshot entry requires a SKU, text, price and description.' + [string]::IsNullOrWhiteSpace($product.description) -or + [string]::IsNullOrWhiteSpace($product.brandCode) -or + [string]::IsNullOrWhiteSpace($product.subCategoryCode) -or + @($product.tags).Count -eq 0 -or 'mountain-bike' -notin @($product.tags) -or + [string]::IsNullOrWhiteSpace($product.sourceType) -or + $product.sourceType -notin @('manufacturer', 'retailer') -or + [string]::IsNullOrWhiteSpace($product.sourceUrl)) { + throw 'Every catalog item requires a mountain-bike SKU, category, brand, tags, source URL and product text.' + } + $sourceUri = $null + if (![uri]::TryCreate([string]$product.sourceUrl, [UriKind]::Absolute, [ref]$sourceUri) -or + $sourceUri.Scheme -notin @('http', 'https')) { throw "Catalog item '$($product.sku)' has an invalid source URL." } + if ($null -ne $product.tagProvenance) { + foreach ($alias in @($product.tagProvenance)) { + if ($alias.tag -notin @($product.tags) -or + $alias.origin -notin @('user-provided-colloquial-search-alias', 'user-provided-informal-search-alias') -or + [string]::IsNullOrWhiteSpace($alias.note)) { + throw "Catalog item '$($product.sku)' has invalid curated-tag provenance." + } + if ($alias.tag -eq 'dentist-bike' -and + ($product.brandCode -ne 'YETI' -or $alias.origin -ne 'user-provided-informal-search-alias')) { + throw "The dentist-bike alias is restricted to Yeti demo listings and must be marked informal." + } + if ($alias.tag -in @('e-bike', 'emtb') -and + ($product.subCategoryCode -ne 'EM' -or $alias.origin -ne 'user-provided-colloquial-search-alias')) { + throw "E-bike aliases are restricted to catalogued electric mountain bikes and must be marked colloquial." + } + } + } + if ('dentist-bike' -in @($product.tags) -and !(@($product.tagProvenance | Where-Object { $_.tag -eq 'dentist-bike' }).Count)) { + throw "Catalog item '$($product.sku)' must identify the dentist-bike alias as user-provided, not manufacturer fact." + } + if (@($product.tags | Where-Object { $_ -in @('e-bike', 'emtb') }).Count -gt 0 -and + @($product.tagProvenance | Where-Object { $_.tag -in @('e-bike', 'emtb') }).Count -lt 2) { + throw "Catalog item '$($product.sku)' must document both colloquial electric-MTB aliases." } if (!$catalogSkus.Add([string]$product.sku)) { throw "Duplicate catalog SKU '$($product.sku)'." } } @@ -168,10 +213,10 @@ function Get-ProductSearchEligibleCatalog { function Get-ProductSearchFilter { param([Parameter(Mandatory)][object] $Query) - if ($null -eq $Query.constraints -or $null -eq $Query.constraints.price) { return $null } + if ($null -eq $Query.constraints -or $null -eq $Query.constraints.price) { return "sku ne 'LABOR'" } $constraint = $Query.constraints.price $value = ([decimal]$constraint.value).ToString('0.############################', [Globalization.CultureInfo]::InvariantCulture) - return "price $($constraint.operator) $value" + return "sku ne 'LABOR' and price $($constraint.operator) $value" } function Get-ProductSearchMetrics { @@ -249,14 +294,14 @@ function Invoke-ProductSearchRelevanceEvaluation { [Parameter(Mandatory)][uri] $OllamaAddress, [Parameter(Mandatory)][scriptblock] $Request, [string] $DatasetSha256, - [ValidateSet('semantic', 'fulltext')][string] $Approach = 'semantic', + [ValidateSet('semantic', 'fulltext', 'hybrid')][string] $Approach = 'semantic', [ValidateSet('all', 'tuning', 'held-out')][string] $Split = 'all', [string] $BaselinePath ) if ($Approach -eq 'fulltext' -and $Split -ne 'tuning') { throw 'This lexical milestone is tuning-only. Pass -Split tuning; held-out queries must not execute during tuning.' } - if ($BaselinePath -and $Split -ne 'tuning') { throw 'Baseline comparison is tuning-only.' } + if ($BaselinePath -and $Split -eq 'all') { throw 'Baseline comparison requires one explicit split: tuning or held-out.' } $selectedQueries = @($Dataset.queries | Where-Object { $Split -eq 'all' -or $_.split -eq $Split }) $requestedSplit = $Split if (!$selectedQueries.Count) { throw "No queries selected for split '$Split'." } @@ -272,19 +317,22 @@ function Invoke-ProductSearchRelevanceEvaluation { $DatasetSha256 = (Get-FileHash -LiteralPath (Join-Path $PSScriptRoot 'product-search-relevance.dataset.json') -Algorithm SHA256).Hash } - if ($Approach -eq 'semantic') { + $lexicalRecipe = $null + $usesSemantic = $Approach -in @('semantic', 'hybrid') + $usesLexical = $Approach -in @('fulltext', 'hybrid') + if ($usesSemantic) { $models = (Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($OllamaAddress, 'api/tags'))).Data.models $model = @($models | Where-Object { $_.name -eq $Dataset.model -and $_.digest -eq $Dataset.modelDigest }) if ($model.Count -ne 1) { throw "Ollama model drift: expected $($Dataset.model) with digest $($Dataset.modelDigest)." } } - else { + if ($usesLexical) { $sqlPath = Join-Path $PSScriptRoot '..\samples\src\Contoso.Products.Infrastructure\Repositories\Sql' $repositoryPath = Split-Path $sqlPath $lexicalSource = Get-Content -LiteralPath (Join-Path $repositoryPath 'ProductLexicalSearchIndex.cs') -Raw if ($lexicalSource -notmatch 'Recipe\s*=\s*"([^"]+)"') { throw 'Cannot identify the lexical recipe.' } $lexicalRecipe = $Matches[1] $implementationIdentity | Add-Member -NotePropertyName lexicalImplementationSha256 -NotePropertyValue @( - @('ProductLexicalSearchIndex.cs', 'ProductRepository.cs', 'ProductQueryArgsConfig.cs') | ForEach-Object { + @('ProductLexicalSearchIndex.cs', 'ProductQueryRepository.cs', 'ProductSearchFusion.cs', 'ProductQueryArgsConfig.cs') | ForEach-Object { [pscustomobject]@{ file = $_; sha256 = (Get-FileHash -LiteralPath (Join-Path $repositoryPath $_) -Algorithm SHA256).Hash } }) $implementationIdentity | Add-Member -NotePropertyName lexicalSqlSha256 -NotePropertyValue @( @@ -298,9 +346,26 @@ function Invoke-ProductSearchRelevanceEvaluation { throw "Products API drift: this evaluation requires $Approach ranked search." } - $catalogResponse = Get-ProductSearchResponse -Request $Request -Uri ([uri]::new($BaseAddress, 'api/products?$take=100&$count=true')) - $liveCatalog = @($catalogResponse.Data) - $liveCatalogCount = Get-ProductSearchCountHeader $catalogResponse.Headers + $liveCatalog = [Collections.Generic.List[object]]::new() + $liveCatalogCount = $null + for ($offset = 0; $true; $offset += 100) { + $catalogUri = [uri]::new($BaseAddress, "api/products?`$skip=$offset&`$take=100&`$count=true&`$filter=$([uri]::EscapeDataString("sku ne 'LABOR'"))") + $catalogResponse = Get-ProductSearchResponse -Request $Request -Uri $catalogUri + $page = @($catalogResponse.Data) + $pageCount = Get-ProductSearchCountHeader $catalogResponse.Headers + if ($null -eq $liveCatalogCount) { $liveCatalogCount = $pageCount } + elseif ($pageCount -ne $liveCatalogCount) { throw 'Eligible catalog count changed during paged catalog verification.' } + if ($offset -ge $liveCatalogCount) { + if ($page.Count -ne 0) { throw "Products API returned rows beyond the reported catalog count at offset $offset." } + break + } + $expectedPageCount = [Math]::Min(100, $liveCatalogCount - $offset) + if ($page.Count -ne $expectedPageCount) { + throw "Incomplete catalog page at offset $offset`: expected $expectedPageCount rows, got $($page.Count)." + } + foreach ($product in $page) { $liveCatalog.Add($product) } + if ($offset + $page.Count -ge $liveCatalogCount) { break } + } if ($liveCatalogCount -ne @($Dataset.catalog).Count -or $liveCatalog.Count -ne @($Dataset.catalog).Count) { $expectedSkus = @($Dataset.catalog | ForEach-Object sku) $actualSkus = @($liveCatalog | ForEach-Object sku) @@ -323,6 +388,9 @@ function Invoke-ProductSearchRelevanceEvaluation { if ($Approach -eq 'semantic' -and $firstActualCount -ne $firstExpectedCount) { throw "Search backfill is not ready or is incomplete for profile '$($Dataset.profile)' and recipe '$($Dataset.documentRecipe)': expected $firstExpectedCount indexed candidates, got $firstActualCount." } + if ($Approach -eq 'hybrid' -and $firstActualCount -gt [Math]::Min($firstExpectedCount, 200)) { + throw "Hybrid candidate count exceeds the bounded eligible pool for query '$($firstQuery.id)': expected at most $([Math]::Min($firstExpectedCount, 200)), got $firstActualCount." + } $reports = [Collections.Generic.List[object]]::new() foreach ($query in $selectedQueries) { @@ -338,9 +406,10 @@ function Invoke-ProductSearchRelevanceEvaluation { $candidateCount = Get-ProductSearchCountHeader $response.Headers $expectedCandidateCount = @((Get-ProductSearchEligibleCatalog -Dataset $Dataset -Query $query)).Count + $maximumCandidateCount = if ($Approach -eq 'hybrid') { [Math]::Min($expectedCandidateCount, 200) } else { $expectedCandidateCount } if (($Approach -eq 'semantic' -and $candidateCount -ne $expectedCandidateCount) -or - $candidateCount -lt 0 -or $candidateCount -gt $expectedCandidateCount) { - throw "Search count drift for $Approach query '$($query.id)': $expectedCandidateCount eligible catalog rows, got $candidateCount candidates. Semantic must match the eligible count; lexical counts must be between zero and that count." + $candidateCount -lt 0 -or $candidateCount -gt $maximumCandidateCount) { + throw "Search count drift for $Approach query '$($query.id)': $expectedCandidateCount eligible catalog rows, got $candidateCount candidates. Semantic count must match indexed eligible rows; lexical and hybrid counts must remain within their documented candidate scope." } $judgments = @{} @@ -381,7 +450,7 @@ function Invoke-ProductSearchRelevanceEvaluation { noRelevant = !$metrics.HasRelevant ndcgAt5 = $metrics.NdcgAt5 precisionAt5 = $metrics.PrecisionAt5 - eligibleCandidateCount = $expectedCandidateCount + eligibleCatalogCount = $expectedCandidateCount matchedCandidateCount = $candidateCount constraintViolationCount = $constraintViolations.Count topFive = $topFive @@ -393,7 +462,7 @@ function Invoke-ProductSearchRelevanceEvaluation { $splitQueries = @($reports | Where-Object split -EQ $split) $relevantQueries = @($splitQueries | Where-Object { !$_.noRelevant }) $exactQueries = @($splitQueries | Where-Object { $null -ne $_.exactSkuTop1 }) - $constraintQueries = @($splitQueries | Where-Object { $null -ne $_.eligibleCandidateCount -and $null -ne $_.kind -and $_.kind -eq 'price-constraint' }) + $constraintQueries = @($splitQueries | Where-Object { $null -ne $_.eligibleCatalogCount -and $null -ne $_.kind -and $_.kind -eq 'price-constraint' }) $constraintResultCount = 0 foreach ($item in $constraintQueries) { $constraintResultCount += $item.topFive.Count } $violations = ($constraintQueries | Measure-Object -Property constraintViolationCount -Sum).Sum @@ -424,37 +493,57 @@ function Invoke-ProductSearchRelevanceEvaluation { $baseline.catalogIdentity.seedFileSha256 -ne $actualSeedHash) { throw 'Baseline dataset/catalog identity does not match this evaluation.' } - $baselineIds = @($baseline.queries | Where-Object split -EQ 'tuning' | ForEach-Object id | Sort-Object) + $baselineIds = @($baseline.queries | Where-Object split -EQ $Split | ForEach-Object id | Sort-Object) $actualIds = @($reports | ForEach-Object id | Sort-Object) - if (($baselineIds -join '|') -ne ($actualIds -join '|')) { throw 'Baseline tuning query IDs differ.' } + if (($baselineIds -join '|') -ne ($actualIds -join '|')) { throw "Baseline $Split query IDs differ." } + $baselineSplit = $baseline.splits.$Split + $currentSplit = $splitReports[$Split] $comparison = [pscustomobject]@{ baselineSha256 = (Get-FileHash -LiteralPath $BaselinePath -Algorithm SHA256).Hash - split = 'tuning' - baseline = $baseline.splits.tuning - meanNdcgAt5Delta = $splitReports.tuning.meanNdcgAt5 - $baseline.splits.tuning.meanNdcgAt5 - meanPrecisionAt5Delta = $splitReports.tuning.meanPrecisionAt5 - $baseline.splits.tuning.meanPrecisionAt5 - exactSkuTop1AccuracyDelta = $splitReports.tuning.exactSkuTop1Accuracy - $baseline.splits.tuning.exactSkuTop1Accuracy + split = $Split + baseline = $baselineSplit + meanNdcgAt5Delta = $currentSplit.meanNdcgAt5 - $baselineSplit.meanNdcgAt5 + meanPrecisionAt5Delta = $currentSplit.meanPrecisionAt5 - $baselineSplit.meanPrecisionAt5 + exactSkuTop1AccuracyDelta = $currentSplit.exactSkuTop1Accuracy - $baselineSplit.exactSkuTop1Accuracy } } return [pscustomobject]@{ - evaluationVersion = 'product-search-relevance-runner-v2' + evaluationVersion = 'product-search-relevance-runner-v3' approach = $Approach selectedSplit = $requestedSplit completedAtUtc = [DateTime]::UtcNow.ToString('o') status = 'completed' datasetVersion = [string]$Dataset.datasetVersion - model = if ($Approach -eq 'semantic') { [string]$Dataset.model } else { $null } - modelDigest = if ($Approach -eq 'semantic') { [string]$Dataset.modelDigest } else { $null } - profile = if ($Approach -eq 'semantic') { [string]$Dataset.profile } else { 'postgres:english:ts_rank_cd:exact-sku-first' } - dimensions = if ($Approach -eq 'semantic') { [int]$Dataset.dimensions } else { $null } - documentRecipe = if ($Approach -eq 'semantic') { [string]$Dataset.documentRecipe } else { $lexicalRecipe } - documentInstruction = if ($Approach -eq 'semantic') { [string]$Dataset.documentInstruction } else { $null } - queryInstruction = if ($Approach -eq 'semantic') { [string]$Dataset.queryInstruction } else { $null } + model = if ($usesSemantic) { [string]$Dataset.model } else { $null } + modelDigest = if ($usesSemantic) { [string]$Dataset.modelDigest } else { $null } + profile = if ($usesSemantic) { [string]$Dataset.profile } else { 'postgres:english:ts_rank_cd:exact-sku-first' } + dimensions = if ($usesSemantic) { [int]$Dataset.dimensions } else { $null } + documentRecipe = if ($usesSemantic) { [string]$Dataset.documentRecipe } else { $lexicalRecipe } + documentInstruction = if ($usesSemantic) { [string]$Dataset.documentInstruction } else { $null } + queryInstruction = if ($usesSemantic) { [string]$Dataset.queryInstruction } else { $null } + lexicalRecipe = if ($usesLexical) { $lexicalRecipe } else { $null } + candidatePool = if ($Approach -eq 'hybrid') { + [pscustomobject]@{ + candidatesPerBranch = 100 + maximumUniqueCandidates = 200 + rrfK = 60 + exactEligibleSkuPriority = $true + countSemantics = 'Deduplicated union of the bounded eligible branch candidates, not the full eligible catalog.' + pagingSemantics = 'Stable offset paging over the fused candidate union; offsets beyond the union return no items.' + } + } else { $null } + countSemantics = switch ($Approach) { + 'semantic' { 'Eligible indexed semantic candidates; expected to cover the reviewed eligible catalog after the guarded completeness fixture.' } + 'fulltext' { 'Eligible lexical matches; not the full eligible catalog.' } + 'hybrid' { 'Deduplicated bounded union of eligible lexical and semantic candidates; not the full eligible catalog.' } + } datasetSha256 = $DatasetSha256 implementationIdentity = $implementationIdentity catalogIdentity = [pscustomobject]@{ seedFile = [string]$Dataset.seedFile seedFileSha256 = $actualSeedHash + referenceDataFile = [string]$Dataset.referenceDataFile + referenceDataFileSha256 = [string]$Dataset.referenceDataFileSha256 eligibleCatalogCount = $liveCatalogCount reviewedCatalogSha256 = $catalogSignature liveApiCatalogSha256 = $liveCatalogSignature @@ -469,7 +558,7 @@ function Invoke-ProductSearchRelevanceEvaluation { constraints = 'Returned price-filtered results are validated against the declared hard price condition; no threshold is imposed on relevance scores.' } splits = $splitReports - tuningComparison = $comparison + baselineComparison = $comparison queries = @($reports) } } diff --git a/tools/product-search-relevance.seed.yaml b/tools/product-search-relevance.seed.yaml deleted file mode 100644 index 0edc5a1da..000000000 --- a/tools/product-search-relevance.seed.yaml +++ /dev/null @@ -1,45 +0,0 @@ -products: - - product: - - { product_id: ^1, sku: YETI-ASR-C2-2025, text: Yeti ASR C2, sub_category_code: XC, unit_of_measure_code: EA, price: 5800, brand_code: YETI, tags_json: '["cross-country","full-suspension"]', description: 'A carbon full-suspension cross-country mountain bike built for efficient climbing and fast riding on technical singletrack.' } - - { product_id: ^2, sku: YETI-SB120-C2, text: Yeti SB120 C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: YETI, description: 'A carbon full-suspension trail mountain bike designed for efficient climbs, responsive handling and flowing singletrack descents.' } - - { product_id: ^3, sku: YETI-SB140-LR-C2, text: Yeti SB140 LR C2, sub_category_code: TR, unit_of_measure_code: EA, price: 6600, brand_code: YETI, description: 'A carbon full-suspension trail mountain bike in the Lunch Ride configuration, built for technical climbs and rough, fast descents.' } - - { product_id: ^4, sku: YETI-SB165-C3, text: Yeti SB165 C3, sub_category_code: EN, unit_of_measure_code: EA, price: 7200, brand_code: YETI, description: 'A carbon full-suspension mountain bike for gravity-focused riding, steep descents and demanding bike-park terrain.' } - - { product_id: ^5, sku: YETI-MTE-C2, text: Yeti MTe C2 eMTB, sub_category_code: EM, unit_of_measure_code: EA, price: 9850, brand_code: YETI, description: 'An electric-assist mountain bike with a carbon full-suspension frame, designed for technical trail riding and assisted climbs.' } - - { product_id: ^6, sku: CANYON-LUXTRAIL-CF6, text: Canyon Lux Trail CF 6, sub_category_code: XC, unit_of_measure_code: EA, price: 3899, brand_code: CANYON, description: 'A carbon full-suspension mountain bike combining cross-country pedalling efficiency with trail capability for long rides and technical singletrack.' } - - { product_id: ^7, sku: CANYON-SPECTRAL-CF8, text: Canyon Spectral CF 8, sub_category_code: TR, unit_of_measure_code: EA, price: 4999, brand_code: CANYON, description: 'A carbon full-suspension trail mountain bike for varied singletrack, balancing climbing capability with control on rough, technical descents.' } - - { product_id: ^8, sku: CANYON-NEURON-CF7, text: Canyon Neuron CF 7, sub_category_code: TR, unit_of_measure_code: EA, price: 3499, brand_code: CANYON, description: 'A carbon full-suspension trail mountain bike for long rides over varied terrain, combining efficient climbing with stable descending.' } - - { product_id: ^9, sku: CANYON-TORQUE-MULLET-CF8, text: Canyon Torque Mullet CF 8, sub_category_code: EN, unit_of_measure_code: EA, price: 3599, brand_code: CANYON, description: 'A carbon mixed-wheel mountain bike designed for freeride and bike-park use, including jumps, drops and steep descents.' } - - { product_id: ^10, sku: CANYON-SENDER-CFR, text: Canyon Sender CFR, sub_category_code: DH, unit_of_measure_code: EA, price: 7799, brand_code: CANYON, description: 'A carbon downhill mountain bike designed for racing and fast descending on steep, demanding tracks and bike-park terrain.' } - - { product_id: ^11, sku: SPEC-STUMPJUMPER-15-COMP, text: Specialized Stumpjumper 15 Comp, sub_category_code: TR, unit_of_measure_code: EA, price: 5500, brand_code: SPEC, description: 'A full-suspension trail mountain bike with adjustable geometry, designed to balance climbing efficiency and control on technical descents.' } - - { product_id: ^12, sku: SPEC-ENDURO-PRO, text: Specialized Enduro Pro, sub_category_code: EN, unit_of_measure_code: EA, price: 9500, brand_code: SPEC, description: 'A carbon full-suspension enduro mountain bike built for steep, rough descents and enduro racing while retaining the ability to pedal uphill.' } - - { product_id: ^13, sku: SPEC-EPIC-8-PRO, text: Specialized Epic 8 Pro, sub_category_code: XC, unit_of_measure_code: EA, price: 9000, is_inactive: true, brand_code: SPEC, description: 'A carbon full-suspension cross-country mountain bike designed for efficient pedalling, competitive racing and technical singletrack.' } - - { product_id: ^14, sku: SPEC-CHISEL-COMP, text: Specialized Chisel Comp, sub_category_code: HT, unit_of_measure_code: EA, price: 3400, brand_code: SPEC, description: 'An aluminium cross-country mountain bike designed for efficient pedalling, racing and fast singletrack riding.' } - - { product_id: ^15, sku: SPEC-STATUS-140, text: Specialized Status 140, sub_category_code: FR, unit_of_measure_code: EA, price: 2499, brand_code: SPEC, description: 'An aluminium mixed-wheel full-suspension mountain bike designed for playful trail riding, jumps and technical descents.' } - - { product_id: ^16, sku: GIANT-TRANCE-X-ADVPRO29, text: Giant Trance X Advanced Pro 29, sub_category_code: TR, unit_of_measure_code: EA, price: 6200, brand_code: GIANT, description: 'A carbon full-suspension trail mountain bike with 29-inch wheels and adjustable geometry, designed for climbing and technical descending.' } - - { product_id: ^17, sku: GIANT-ANTHEM-ADV29-1, text: Giant Anthem Advanced 29 1, sub_category_code: XC, unit_of_measure_code: EA, price: 7800, brand_code: GIANT, description: 'A carbon full-suspension cross-country mountain bike with 29-inch wheels, designed for efficient pedalling and fast riding on demanding race courses.' } - - { product_id: ^18, sku: GIANT-TALON-2, text: Giant Talon 2, sub_category_code: HT, unit_of_measure_code: EA, price: 800, is_deleted: true, brand_code: GIANT, description: 'An aluminium hardtail mountain bike with a suspension fork, designed as an accessible introduction to off-road trails.' } - - { product_id: ^19, sku: SHIMANO-XT-M8100-RD, text: Shimano Deore XT M8100 Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 129, brand_code: SHIMANO, description: 'A 12-speed mountain bike rear derailleur that moves the chain between cassette sprockets to change gears. Its clutch helps control chain movement on rough trails.' } - - { product_id: ^20, sku: SHIMANO-SLX-M7100-CASSETTE, text: Shimano SLX M7100 12-speed Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 99, brand_code: SHIMANO, description: 'A 12-speed mountain bike cassette that provides a wide range of gears for climbing and descending. It uses Shimano Micro Spline mounting and Hyperglide Plus shifting.' } - - { product_id: ^21, sku: SHIMANO-XTR-M9100-CRANK, text: Shimano XTR M9100 Crankset, sub_category_code: CR, unit_of_measure_code: EA, price: 499, brand_code: SHIMANO, description: 'A mountain bike crankset that transfers pedalling power to the chain through a direct-mount chainring, designed for lightweight cross-country and trail use.' } - - { product_id: ^22, sku: SRAM-GX-EAGLE-RD, text: SRAM GX Eagle Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 135, brand_code: SRAM, description: 'A cable-operated 12-speed mountain bike rear derailleur for SRAM Eagle drivetrains. It shifts the chain between rear sprockets and uses a clutch to control chain movement.' } - - { product_id: ^23, sku: SRAM-X0-TRANSMISSION-RD, text: SRAM X0 Transmission Rear Derailleur, sub_category_code: DR, unit_of_measure_code: EA, price: 400, brand_code: SRAM, description: 'A wireless electronic mountain bike rear derailleur for SRAM Eagle Transmission drivetrains. Its hangerless Full Mount design attaches around the rear axle on compatible frames.' } - - { product_id: ^24, sku: SRAM-XX1-EAGLE-CASSETTE, text: SRAM XX1 Eagle Cassette, sub_category_code: CA, unit_of_measure_code: EA, price: 449, brand_code: SRAM, description: 'A lightweight 12-speed mountain bike cassette with a wide gear range for climbing and descending, designed for SRAM Eagle drivetrains and XD freehub drivers.' } - - { product_id: ^25, sku: ONEUP-V3-DROPPER-180, text: OneUp Dropper Post V3 180mm, sub_category_code: SP, unit_of_measure_code: EA, price: 269, brand_code: ONEUP, description: 'A mountain bike dropper seatpost with 180 mm of travel. Riders can lower the saddle for descending and raise it for pedalling using a compatible remote.' } - - { product_id: ^26, sku: ONEUP-CARBON-HANDLEBAR, text: OneUp Carbon Handlebar, sub_category_code: HB, unit_of_measure_code: EA, price: 159, brand_code: ONEUP, description: 'A carbon mountain bike handlebar with a shaped profile designed to absorb vibration and reduce hand fatigue while maintaining steering control.' } - - { product_id: ^27, sku: ONEUP-COMPOSITE-PEDALS, text: OneUp Composite Pedals, sub_category_code: PE, unit_of_measure_code: PR, price: 49, is_inactive: true, brand_code: ONEUP, description: 'Flat mountain bike pedals with composite bodies, broad platforms and replaceable traction pins, designed to grip ordinary flat-soled riding shoes.' } - - { product_id: ^28, sku: ONEUP-EDC-TOOL-V2, text: OneUp EDC Tool V2, sub_category_code: TO, unit_of_measure_code: EA, price: 59, brand_code: ONEUP, description: 'A compact trailside repair multi-tool with hex keys, a Torx key, a chain breaker and a tyre lever, designed for storage in compatible OneUp steerer or pump systems.' } - - { product_id: ^29, sku: MUCOFF-NANO-CLEANER, text: Muc-Off Nano Tech Bike Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 16.99, brand_code: MUCOFF, description: 'A spray-on bicycle cleaner that breaks down dirt and grime on frames and components before rinsing. Suitable for routine washing after muddy rides.' } - - { product_id: ^30, sku: MUCOFF-TUBELESS-SEALANT, text: Muc-Off MTB Tubeless Sealant, sub_category_code: SE, unit_of_measure_code: EA, price: 8.00, brand_code: MUCOFF, description: 'A tubeless mountain bike tyre sealant that helps seal punctures and maintain an airtight tyre system. Its UV-reactive dye helps locate leaks.' } - - { product_id: ^31, sku: MUCOFF-DRIVETRAIN-CLEANER, text: Muc-Off Drivetrain Cleaner, sub_category_code: CL, unit_of_measure_code: EA, price: 25.00, brand_code: MUCOFF, description: 'A bicycle drivetrain cleaner that removes grease, oil and grime from chains, cassettes, chainrings and derailleurs. It can be sprayed on or used with a chain-cleaning device.' } - - { product_id: ^32, sku: LABOR, text: Labor, sub_category_code: LA, unit_of_measure_code: HR, price: 80.00, brand_code: NONE, is_non_stocked: true, description: 'Workshop labour charged by the hour for bicycle maintenance and repair. This is a service rather than a stocked physical product.' } - - inventory: - - { inventory_id: ^1, qty_on_hand: 3 } - - { inventory_id: ^16, qty_on_hand: 1 } - - { inventory_id: ^24, qty_on_hand: 3 } - - { inventory_id: ^28, qty_on_hand: 5 } - - movement: - - { movement_id: ^1, product_id: ^1, quantity: -2, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - - { movement_id: ^2, product_id: ^6, quantity: -1, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - - { movement_id: ^3, product_id: ^31, quantity: -3, reference_id: ^1000, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - - { movement_id: ^4, product_id: ^24, quantity: -2, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } - - { movement_id: ^5, product_id: ^28, quantity: -1, reference_id: ^1001, movement_kind_code: I, movement_status_code: P, unit_of_measure_code: EA } diff --git a/tools/test-product-search-relevance.ps1 b/tools/test-product-search-relevance.ps1 index 00ed0c604..51ad48c2a 100644 --- a/tools/test-product-search-relevance.ps1 +++ b/tools/test-product-search-relevance.ps1 @@ -14,6 +14,10 @@ function Assert-Throws { $dataset = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" Test-ProductSearchDataset $dataset | Out-Null +Assert-Equal (Get-ProductSearchFilter $dataset.queries[0]) "sku ne 'LABOR'" 'The shared catalog benchmark must exclude only the workshop service, not reviewed Miscellaneous products.' +$invalidReference = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" +$invalidReference.referenceDataFileSha256 = 'drift' +Assert-Throws { Test-ProductSearchDataset $invalidReference } 'Canonical reference-data label drift must be rejected.' $metrics = Get-ProductSearchMetrics -RankedGrades @(3, 2, 0) -IdealGrades @(3, 2, 1) $expectedNdcg = (7 + 3 / [Math]::Log(3, 2)) / (7 + 3 / [Math]::Log(3, 2) + 1 / [Math]::Log(4, 2)) @@ -36,7 +40,7 @@ $invalid = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.data $invalid.queries[0].kind = 'unreviewed-kind' Assert-Throws { Test-ProductSearchDataset $invalid } 'Unsupported query kinds must be rejected.' $invalid = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" -$constrainedQuery = $invalid.queries | Where-Object id -EQ 'tune-price-derailleur-under-150' +$constrainedQuery = $invalid.queries | Where-Object id -EQ 'tune-derailleur-under-150' $constrainedQuery.judgments[0].sku = 'SHIMANO-XTR-M9100-CRANK' Assert-Throws { Test-ProductSearchDataset $invalid } 'Judgments violating hard query constraints must be rejected.' @@ -52,9 +56,14 @@ $validMock = { return [pscustomobject]@{ StatusCode = 200; Content = $content; Headers = @{} } } if ($path -ne '/api/products') { throw "Unexpected mocked URI: $Uri" } + if (![uri]::UnescapeDataString($Uri.Query).Contains("sku ne 'LABOR'")) { throw 'Every catalog/search request must use the reviewed MTB eligibility filter.' } if (!$Uri.Query.Contains('$search=')) { - $items = @($dataset.catalog | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) - return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $items -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = "$($items.Count)" } } + $skip = 0 + $take = 100 + if ($Uri.Query -match '\$skip=(\d+)') { $skip = [int]$Matches[1] } + if ($Uri.Query -match '\$take=(\d+)') { $take = [int]$Matches[1] } + $items = @($dataset.catalog | Select-Object -Skip $skip -First $take | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) + return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $items -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = "$($dataset.catalog.Count)" } } } $queryString = $Uri.Query.TrimStart([char]'?') $searchPart = $queryString -split '&' | Where-Object { $_.StartsWith('$search=') } | Select-Object -First 1 @@ -79,8 +88,18 @@ $validMock = { }.GetNewClosure() $mockReport = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $validMock -Assert-Equal $mockReport.queries.Count 24 'The mocked end-to-end evaluation must exercise all curated queries.' -Assert-Equal $mockReport.splits.'held-out'.queryCount 6 'Tuning and held-out aggregates must remain split.' +Assert-Equal $mockReport.queries.Count 60 'The mocked end-to-end evaluation must exercise all curated queries.' +Assert-Equal $mockReport.splits.'held-out'.queryCount 12 'Tuning and held-out aggregates must remain split.' +Assert-Equal $mockReport.splits.tuning.queryCount 48 'Tuning and held-out aggregates must remain split.' +Assert-Equal @($dataset.catalog | Where-Object { 'e-bike' -in $_.tags -and 'emtb' -in $_.tags }).Count 3 'E-bike aliases must be limited to the electric mountain bikes.' +Assert-Equal @($dataset.catalog | Where-Object { 'dentist-bike' -in $_.tags -and $_.brandCode -ne 'YETI' }).Count 0 'The informal dentist-bike alias must be limited to Yeti catalog entries.' +$invalid = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" +$yetiAlias = $invalid.catalog | Where-Object sku -EQ 'YETI-ASR-C2-2025' +$yetiAlias.tagProvenance[0].origin = 'manufacturer' +Assert-Throws { Test-ProductSearchDataset $invalid } 'Informal search aliases must not be represented as manufacturer product facts.' +$invalid = Get-ProductSearchDataset "$PSScriptRoot\product-search-relevance.dataset.json" +$invalid.catalog | Where-Object sku -EQ 'SPECIALIZED-TURBO-LEVO' | ForEach-Object { $_.subCategoryCode = 'TR' } +Assert-Throws { Test-ProductSearchDataset $invalid } 'E-bike aliases must not be assigned to non-electric MTB categories.' $shortPageMock = { param([uri] $Uri) @@ -95,9 +114,13 @@ Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseA $driftMock = { param([uri] $Uri) if ($Uri.AbsolutePath -eq '/api/products' -and !$Uri.Query.Contains('$search=')) { - $items = @($dataset.catalog | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) + $skip = 0 + $take = 100 + if ($Uri.Query -match '\$skip=(\d+)') { $skip = [int]$Matches[1] } + if ($Uri.Query -match '\$take=(\d+)') { $take = [int]$Matches[1] } + $items = @($dataset.catalog | Select-Object -Skip $skip -First $take | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) $items[0].text = 'Changed catalog item' - return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $items -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = "$($items.Count)" } } + return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $items -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = "$($dataset.catalog.Count)" } } } & $validMock $Uri }.GetNewClosure() @@ -118,17 +141,19 @@ $violationMock = { $queryString = $Uri.Query.TrimStart([char]'?') $searchPart = $queryString -split '&' | Where-Object { $_.StartsWith('$search=') } | Select-Object -First 1 $queryText = [uri]::UnescapeDataString($searchPart.Substring(8)) - if ($queryText -eq 'a rear derailleur for a 12-speed mountain bike under $150') { + if ($queryText -eq 'mountain bike rear derailleur under $150') { $crank = $dataset.catalog | Where-Object sku -EQ 'SHIMANO-XTR-M9100-CRANK' - $eligible = @($dataset.catalog | Where-Object { [decimal]$_.price -lt 150 } | Select-Object -First 4) + $query = $dataset.queries | Where-Object id -EQ 'tune-derailleur-under-150' + $eligibleCatalog = @($dataset.catalog | Where-Object { [decimal]$_.price -lt [decimal]$query.constraints.price.value }) + $eligible = @($eligibleCatalog | Select-Object -First 4) $results = @(@($crank) + $eligible | ForEach-Object { @{ sku = $_.sku; text = $_.text; price = $_.price; description = $_.description } }) - return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $results -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = '8' } } + return [pscustomobject]@{ StatusCode = 200; Content = (ConvertTo-Json -InputObject $results -Depth 8 -Compress); Headers = @{ 'X-Paging-Total-Count' = "$($eligibleCatalog.Count)" } } } } & $validMock $Uri }.GetNewClosure() $violationReport = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $violationMock -$violationQuery = $violationReport.queries | Where-Object id -EQ 'tune-price-derailleur-under-150' +$violationQuery = $violationReport.queries | Where-Object id -EQ 'tune-derailleur-under-150' Assert-Equal $violationQuery.constraintViolationCount 1 'Hard price-filter violations must be measured rather than hidden.' Assert-Equal $violationReport.splits.tuning.constraintViolationCount 1 'Constraint violations must be included in split aggregates.' @@ -152,14 +177,34 @@ $tuningOnlyMock = { } & $validMock $Uri }.GetNewClosure() -$lexical = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $tuningOnlyMock -Approach fulltext -Split tuning -BaselinePath "$PSScriptRoot\..\samples\docs\product-search-relevance-baseline.json" -Assert-Equal $lexical.queries.Count 18 'Tuning-only must execute exactly 18 queries.' +$lexical = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $tuningOnlyMock -Approach fulltext -Split tuning +Assert-Equal $lexical.queries.Count 48 'Tuning-only must execute exactly 48 queries.' Assert-Equal $lexical.splits.ContainsKey('held-out') $false 'Held-out metrics must not be generated while tuning.' Assert-Equal $lexical.model $null 'Full-text must not claim model inference.' +Assert-Equal $lexical.countSemantics 'Eligible lexical matches; not the full eligible catalog.' 'Full-text count semantics must not imply complete catalog coverage.' Assert-Equal $lexical.splits.tuning.meanNdcgAt5 0 'Zero matching results are legitimate lexical results and must score zero.' -Assert-Equal $lexical.tuningComparison.split 'tuning' 'Only tuning scores may be compared with the baseline.' +Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $tuningOnlyMock -Approach fulltext -Split tuning -BaselinePath "$PSScriptRoot\..\samples\docs\product-search-relevance-baseline.json" } 'The historical v1 baseline must not be compared directly with the v2 catalog and query set.' Assert-Equal $mockReport.selectedSplit 'all' 'The original semantic full evaluation remains supported.' Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $tuningOnlyMock -Approach fulltext } 'Full-text must require an explicit tuning-only split.' Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $tuningOnlyMock -Approach fulltext -Split held-out } 'Held-out lexical execution must be rejected during this tuning milestone.' -Write-Host 'Product-search relevance tests passed: metric math, dataset validation, catalog drift, backfill completeness, constraint violations, and mocked HTTP failures.' +$hybridMock = { + param([uri] $Uri) + if ($Uri.AbsolutePath -eq '/api/products/$query') { + return [pscustomobject]@{ StatusCode = 200; Content = '{"search":{"mode":"hybrid","ranked":true}}'; Headers = @{} } + } + & $validMock $Uri +}.GetNewClosure() +$hybrid = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $hybridMock -Approach hybrid -Split tuning +Assert-Equal $hybrid.queries.Count 48 'Hybrid tuning must execute only the tuning split.' +Assert-Equal $hybrid.model $dataset.model 'Hybrid search must retain the semantic model identity.' +Assert-Equal $hybrid.candidatePool.candidatesPerBranch 100 'Hybrid reports must identify the per-branch candidate budget.' +Assert-Equal $hybrid.candidatePool.maximumUniqueCandidates 200 'Hybrid reports must identify the maximum unique candidate pool.' +Assert-Equal $hybrid.candidatePool.rrfK 60 'Hybrid reports must identify the fixed RRF constant.' +Assert-Equal $hybrid.countSemantics 'Deduplicated bounded union of eligible lexical and semantic candidates; not the full eligible catalog.' 'Hybrid count must describe the bounded union.' +$hybridHeldOut = Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $hybridMock -Approach hybrid -Split held-out +Assert-Equal $hybridHeldOut.queries.Count 12 'Hybrid held-out evaluation must remain available after tuning is finalized.' +Assert-Equal $hybridHeldOut.splits.ContainsKey('tuning') $false 'Hybrid held-out runs must not execute tuning queries.' +Assert-Throws { Invoke-ProductSearchRelevanceEvaluation -Dataset $dataset -BaseAddress 'http://mock/' -OllamaAddress 'http://mock/' -Request $hybridMock -Approach hybrid -Split held-out -BaselinePath "$PSScriptRoot\..\samples\docs\product-search-relevance-baseline.json" } 'The historical v1 baseline must not be compared directly with v2 held-out queries.' + +Write-Host 'Product-search relevance tests passed: metric math, dataset validation, catalog drift, count scope, split isolation, constraint violations, and mocked HTTP failures.' From 756c749ace5d1f47cc595038afc465f8f59f5479 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Thu, 15 Oct 2026 17:43:19 -0700 Subject: [PATCH 12/25] Improve interactive search and catalog setup experience Add search-session schema caching and catalog bootstrap progress, enable Hybrid in Aspire, and resolve local target frameworks early for Visual Studio. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .github/copilot-instructions.md | 2 +- CONTRIBUTING.md | 12 +- Directory.Build.props | 10 + Directory.Build.targets | 13 - samples/Directory.Build.props | 6 +- samples/aspire/Contoso.Aspire/AppHost.cs | 3 + .../DemoCatalogBootstrapTests.cs | 16 +- .../DemoCatalogProgressTests.cs | 103 +++++++ .../aspire/Contoso.Test.Aspire/GlobalUsing.cs | 4 + .../ProductSearchOperationTests.cs | 270 ++++++++++++++++++ samples/docs/aspire.md | 62 ++-- samples/docs/search.md | 177 +++--------- samples/docs/testing.md | 5 +- .../Repositories/ProductSearchIndex.cs | 28 +- .../tests/Contoso.E2E.Runner/GlobalUsing.cs | 1 + .../Infrastructure/ChoiceManager.cs | 4 +- .../Infrastructure/DemoCatalogBootstrap.cs | 80 ++++-- .../Infrastructure/DemoCatalogOperation.cs | 68 +++++ .../Infrastructure/ProductSearchOperation.cs | 136 +++++++++ samples/tests/Contoso.E2E.Runner/Program.cs | 10 +- .../Scenarios/DatabaseMigrationSetup.cs | 2 +- .../CoreEx.DomainDriven.csproj | 1 - src/Directory.Build.props | 6 +- src/Directory.Build.targets | 3 - .../CoreEx.AspNetCore.Test.Api.csproj | 1 - .../CoreEx.AspNetCore.Test.Unit.csproj | 3 +- ...zure.Messaging.ServiceBus.Test.Unit.csproj | 3 +- .../CoreEx.Caching.Redis.Test.Unit.csproj | 1 - .../CoreEx.Cosmos.Test.Unit.csproj | 1 - .../CoreEx.Data.GraphQL.Test.Unit.csproj | 1 - .../CoreEx.Data.Test.Unit.csproj | 3 +- ...reEx.Database.Postgres.Test.Console.csproj | 1 - .../CoreEx.Database.Postgres.Test.Unit.csproj | 1 - ...eEx.Database.SqlServer.Test.Console.csproj | 1 - ...CoreEx.Database.SqlServer.Test.Unit.csproj | 1 - .../CoreEx.Database.Test.Unit.csproj | 1 - .../CoreEx.DomainDriven.Test.Unit.csproj | 1 - .../CoreEx.Events.Test.Unit.csproj | 3 +- .../CoreEx.RefData.Test.Unit.csproj | 3 +- .../CoreEx.Test.Unit/CoreEx.Test.Unit.csproj | 1 - .../CoreEx.UnitTesting.Test.Unit.csproj | 1 - .../CoreEx.Validation.Test.Unit.csproj | 1 - tests/Directory.Build.props | 3 + tools/validate-template-pack.ps1 | 2 +- 44 files changed, 791 insertions(+), 264 deletions(-) create mode 100644 Directory.Build.props delete mode 100644 Directory.Build.targets create mode 100644 samples/aspire/Contoso.Test.Aspire/DemoCatalogProgressTests.cs create mode 100644 samples/aspire/Contoso.Test.Aspire/ProductSearchOperationTests.cs create mode 100644 samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogOperation.cs create mode 100644 samples/tests/Contoso.E2E.Runner/Infrastructure/ProductSearchOperation.cs delete mode 100644 src/Directory.Build.targets diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 52c90c185..96ce5d65b 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -37,7 +37,7 @@ solution; the rule applies to consumer solutions where these assets are installe - **Build**: `dotnet build CoreEx.sln` - **Test**: `dotnet test CoreEx.sln` or target specific projects. - **Single test**: `dotnet test --filter "FullyQualifiedName~"` -- **Local target framework (inner loop)**: projects multi-target `net8.0;net9.0;net10.0`. Locally, develop, build, and test on `net10.0` Debug only — set the user environment variable `CoreExLocalTargetFramework=net10.0` (honoured by the root `Directory.Build.targets`; ignored when `CI=true` and when packing). Do **not** use `dotnet build -f net10.0` on the solution: it fails with `NETSDK1005` on non-net10 projects such as `CoreEx.Generator` (netstandard2.0). Avoid local Release builds unless specifically required. CI builds and tests all target frameworks in Release; run a full multi-target build locally (`-p:CoreExLocalTargetFramework=`) only when changing `#if NET…` code, framework-conditional package versions in `Directory.Packages.props`, or framework-sensitive APIs. +- **Local target framework (inner loop)**: projects multi-target `net8.0;net9.0;net10.0`. Locally, develop, build, and test on `net10.0` Debug only — set the user environment variable `CoreExLocalTargetFramework=net10.0` (honoured by the root `Directory.Build.props` policy imported by `src`, `samples`, and `tests`; ignored when `CI=true` and by `dotnet pack`). The policy does not force Debug or change single-target projects. Keep framework defaults in props, not late targets or duplicate project declarations, so SDK initialization and Visual Studio framework discovery agree. Restart Visual Studio after changing the environment variable. Do **not** use `dotnet build -f net10.0` on the solution: it fails with `NETSDK1005` on non-net10 projects such as `CoreEx.Generator` (netstandard2.0). Avoid local Release builds unless specifically required. CI builds and tests all target frameworks in Release; run a full multi-target build locally (`-p:CoreExLocalTargetFramework=`) only when changing `#if NET…` code, framework-conditional package versions in `Directory.Packages.props`, or framework-sensitive APIs. For direct `msbuild -t:Pack` or Visual Studio Pack, explicitly clear `CoreExLocalTargetFramework`; unlike `dotnet pack`, those entry points do not supply the CLI's `_IsPacking` global property. `nuget-publish.ps1` already clears the switch. - **Reclaiming disk**: run [`tools/clean-build-outputs.ps1`](../tools/clean-build-outputs.ps1) (`-WhatIf` to preview) to remove all `bin`/`obj` `Debug`/`Release` outputs. - **Samples**: docker-compose infrastructure + dotnet run for Database projects + Aspire AppHost. - **`CoreEx.Template` changes**: `dotnet build`/`dotnet test` do **not** exercise `src/CoreEx.Template/content/**` — that's raw `dotnet new` template content, not compiled C#. Any change under `src/CoreEx.Template/content/` (a host's `Program.cs`/`GlobalUsing.cs`/`.csproj`, a `template.json` symbol, etc.) must be validated by actually scaffolding it: run [`tools/validate-template-pack.ps1`](../tools/validate-template-pack.ps1), which packs the template, installs it, scaffolds every parameter combination it knows about into temp directories, and `dotnet build`s the ones flagged `Build = $true`. It also runs in CI (`.github/workflows/CI.yml`). If you add a new template, host, or parameter combination, add a matching scenario to the script's `$testScenarios` array — parameter-conditional bugs (an unconditional `global using`/`ProjectReference` that should have been gated behind a symbol like `has-data-provider` or `implement-servicebus`) only surface when the generated code is actually compiled, which most existing host-template scenarios don't yet do since they scaffold in isolation without their `coreex` solution siblings. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 31c3fa314..6852874ba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,6 +45,16 @@ Additionally, all source code files (`/src/**/*.cs`) require this exact header a ``` +
+ +## Local builds and Visual Studio + +Set the user environment variable `CoreExLocalTargetFramework=net10.0` to opt into net10-only local builds, then launch Visual Studio from a process that has that environment value. The shared `Directory.Build.props` policy is imported by the framework, sample, and test props before SDK targets initialize; projects must inherit that framework list rather than redeclare it. Single-target projects, including the netstandard2.0 source generator, remain unchanged. The switch does not select a build configuration; Debug is recommended locally. + +Use `dotnet build CoreEx.sln -p:CoreExLocalTargetFramework=` for the full framework matrix. CI (`CI=true`) and `dotnet pack` ignore narrowing; `nuget-publish.ps1` and the template validator also clear it explicitly. Direct `msbuild -t:Pack` and Visual Studio Pack do not set the CLI's `_IsPacking` global property, so clear `CoreExLocalTargetFramework` explicitly for those entry points. + +After framework-policy changes, reload the solution and verify Visual Studio's framework selection before retrying **Contoso.Aspire → Start Without Debugging**. A successful CLI build or static-graph evaluation verifies MSBuild consistency, not the running IDE's configured-project state; no `.vs` deletion should be necessary. +
## Tests @@ -66,4 +76,4 @@ To help ensure that only the highest quality code makes its way into the project For example, sending a PR for just an update to a unit test might seem like a waste of time but the unit tests are just as important as the product code and as such, reviewing changes to them is also just as important. This also helps create visibility for your changes so that others can observe what is going on. -The advantages are numerous: improving code quality, more visibility on changes and their potential impact, avoiding duplication of effort, and creating general awareness of progress being made in various areas. \ No newline at end of file +The advantages are numerous: improving code quality, more visibility on changes and their potential impact, avoiding duplication of effort, and creating general awareness of progress being made in various areas. diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 000000000..a799e1a63 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,10 @@ + + + + + net8.0;net9.0;net10.0 + + $(CoreExLocalTargetFramework) + + + diff --git a/Directory.Build.targets b/Directory.Build.targets deleted file mode 100644 index 31d0e0d75..000000000 --- a/Directory.Build.targets +++ /dev/null @@ -1,13 +0,0 @@ - - - - - $(CoreExLocalTargetFramework) - - - diff --git a/samples/Directory.Build.props b/samples/Directory.Build.props index 1ff9bb6b2..8f20e55a1 100644 --- a/samples/Directory.Build.props +++ b/samples/Directory.Build.props @@ -1,6 +1,8 @@ + + - net8.0;net9.0;net10.0 + $(CoreExTargetFrameworks) enable enable preview @@ -13,4 +15,4 @@ - \ No newline at end of file + diff --git a/samples/aspire/Contoso.Aspire/AppHost.cs b/samples/aspire/Contoso.Aspire/AppHost.cs index 3588f8ce0..6a0495c4b 100644 --- a/samples/aspire/Contoso.Aspire/AppHost.cs +++ b/samples/aspire/Contoso.Aspire/AppHost.cs @@ -23,10 +23,13 @@ builder.AddProject("customers-relay").WithReference(cosmos).WithReference(serviceBus).AddEndpoints("/health/ready/detailed").AddHostedServiceSupport(); // Products domain. +var productSearchMode = builder.Configuration["Products:Search:Mode"] ?? "Hybrid"; var productsApi = builder.AddProject("products-api").WithReference(postgres).WithReference(redis).AddEndpoints("/health/ready/detailed"); +productsApi.WithEnvironment("Products__Search__Mode", productSearchMode); productsApi.WithReference(ollama).WithEnvironment("Ollama__BaseAddress", "http://ollama").WaitFor(ollama); builder.AddProject("products-relay").WithReference(postgres).WithReference(serviceBus).AddEndpoints("/health/ready/detailed").AddHostedServiceSupport(); var productsSubscribe = builder.AddProject("products-subscribe").WithReference(postgres).WithReference(redis).WithReference(serviceBus).AddEndpoints("/health/ready/detailed").AddHostedServiceSupport(); +productsSubscribe.WithEnvironment("Products__Search__Mode", productSearchMode); productsSubscribe.WithReference(ollama).WithEnvironment("Ollama__BaseAddress", "http://ollama"); foreach (var setting in builder.Configuration.GetSection("Ollama:Resilience").AsEnumerable().Where(x => x.Value is not null)) { diff --git a/samples/aspire/Contoso.Test.Aspire/DemoCatalogBootstrapTests.cs b/samples/aspire/Contoso.Test.Aspire/DemoCatalogBootstrapTests.cs index 03fbbdf8b..314fd011f 100644 --- a/samples/aspire/Contoso.Test.Aspire/DemoCatalogBootstrapTests.cs +++ b/samples/aspire/Contoso.Test.Aspire/DemoCatalogBootstrapTests.cs @@ -41,8 +41,16 @@ await cache.GetOrCreateAsync(typeof(BrandCollection), (_, _) => Task.FromResult(new BrandCollection { new Brand { Id = "OLD-SNAPSHOT", Code = "OLD-SNAPSHOT", Text = "Stale brand" } })).ConfigureAwait(false); } - var bootstrap = await DemoCatalogBootstrap.RunAsync(_configuration).ConfigureAwait(false); + var progress = new List(); + var bootstrap = await DemoCatalogBootstrap.RunAsync(_configuration, progress.Add).ConfigureAwait(false); bootstrap.Should().Be(173); + progress.Where(p => p.IsComplete).Select(p => p.Stage).Should().Equal( + "Migrating/resetting Products and loading catalog", "Migrating/resetting Shopping", "Synchronizing current products to Shopping", + "Loading Shopping baskets", "Invalidating Products and Shopping reference-data caches", "Rebuilding lexical index", "Loading search documents", + "Embedding products", "Publishing semantic index", "Verifying lexical/semantic search and query model", "Verifying Products, Shopping and basket integrity"); + var embeddings = progress.Where(p => p.Stage == "Embedding products" && !p.IsComplete).ToArray(); + embeddings.Select(p => p.Completed).Should().Equal(Enumerable.Range(0, 174).Select(i => (int?)i)); + embeddings.Should().OnlyContain(p => p.Total == 173); using (var cacheHost = DemoCatalogBootstrap.CreateServingCacheHost(_configuration, "Products")) { @@ -193,10 +201,14 @@ await ef.Products.CreateAsync(new Contoso.Shopping.Infrastructure.Persistence.Pr .AddInMemoryCollection(new Dictionary { ["Ollama:BaseAddress"] = "http://127.0.0.1:1/" }).Build(); using (var broken = DemoCatalogBootstrap.CreateProductsHost(brokenConfiguration, _productsConnection)) { - var rebuild = () => DemoCatalogBootstrap.PrepareSearchAsync(broken); + progress.Clear(); + var rebuild = () => DemoCatalogBootstrap.PrepareSearchAsync(broken, progress.Add); var failure = await rebuild.Should().ThrowAsync().WithMessage("*search preparation failed*").ConfigureAwait(false); failure.Which.InnerException.Should().NotBeNull(); failure.Which.ToString().Should().Contain("127.0.0.1:1"); + progress.Should().Contain(p => p.Stage == "Embedding products" && p.Completed == 0); + progress.Should().NotContain(p => p.Stage == "Embedding products" && (p.Completed > 0 || p.IsComplete)); + progress.Should().NotContain(p => p.Stage == "Publishing semantic index" || p.Stage.StartsWith("Verifying", StringComparison.Ordinal)); } // A full rerun must clear stale projections and restore deterministic product/basket identities. diff --git a/samples/aspire/Contoso.Test.Aspire/DemoCatalogProgressTests.cs b/samples/aspire/Contoso.Test.Aspire/DemoCatalogProgressTests.cs new file mode 100644 index 000000000..7ab512b7b --- /dev/null +++ b/samples/aspire/Contoso.Test.Aspire/DemoCatalogProgressTests.cs @@ -0,0 +1,103 @@ +namespace Contoso.Test.Aspire; + +[TestFixture] +public class DemoCatalogProgressTests +{ + [Test] + public void Progress_RendersMeasuredCountsAndStageTimingsWithoutClaimingBootstrapSuccess() + { + using var console = new ProductSearchOperationTests.ScriptConsole(); + var operation = new DemoCatalogOperation(console); + operation.Report(new("Lexical [index]")); + operation.Report(new("Lexical [index]", IsComplete: true)); + for (var i = 0; i <= 23; i++) + operation.Report(new("Embedding products", i, 23)); + operation.Report(new("Embedding products", 23, 23, IsComplete: true)); + operation.Report(new("Publishing semantic index")); + + console.Output.Should().Contain("Lexical [index]").And.Contain("Embedding products 0/23") + .And.Contain("Embedding products 1/23").And.Contain("Embedding products 10/23") + .And.Contain("Embedding products 20/23").And.Contain("Embedding products 23/23") + .And.NotContain("Embedding products 2/23").And.NotContain("%").And.NotContain("Demo bootstrap complete:"); + console.Output.Split('✓').Should().HaveCount(3, "only lexical work and embeddings have completed, not publication"); + console.Output.IndexOf("23/23", StringComparison.Ordinal).Should().BeLessThan(console.Output.IndexOf("Publishing semantic index", StringComparison.Ordinal)); + } + + [Test] + public async Task ConfigurationFailure_RemainsVisibleAndNeverPrintsSuccess() + { + using var console = new ProductSearchOperationTests.ScriptConsole(); + var operation = new DemoCatalogOperation(console); + var run = () => operation.RunAsync(new ConfigurationBuilder().Build()); + + await run.Should().ThrowAsync(); + + console.Output.Should().Contain("Demo bootstrap failed during configuration").And.NotContain("Demo bootstrap complete:").And.NotContain("✓"); + } + + [Test] + public async Task CancelledBootstrap_ReportsCancellationWithoutStartingStagesOrPrintingSuccess() + { + using var console = new ProductSearchOperationTests.ScriptConsole(); + var operation = new DemoCatalogOperation(console); + var run = () => operation.RunAsync(Configuration(), new CancellationToken(true)); + + await run.Should().ThrowAsync(); + + console.Output.Should().Contain("Demo bootstrap cancelled").And.NotContain("Demo bootstrap complete:").And.NotContain("✓"); + } + + [Test] + public async Task StageFailure_StopsNotificationsWithoutReportingCompletionOrStartingLaterWork() + { + var progress = new List(); + var failure = new InvalidOperationException("Stop before any database work."); + var run = () => DemoCatalogBootstrap.RunAsync(Configuration(), update => + { + progress.Add(update); + throw failure; + }); + + (await run.Should().ThrowAsync()).Which.Should().BeSameAs(failure); + + progress.Should().ContainSingle().Which.Should().Be(new ProductSearchIndex.RebuildProgress("Migrating/resetting Products and loading catalog")); + } + + [Test] + public async Task CancelledIndex_DoesNotStartLexicalWorkOrReportAnyCompletedEmbeddings() + { + using var host = DemoCatalogBootstrap.CreateProductsHost(Configuration(), "Host=localhost;Database=unused;Username=unused;Password=unused"); + await using var scope = host.Services.CreateAsyncScope(); + var index = scope.ServiceProvider.GetRequiredService(); + var progress = new List(); + var run = () => index.RebuildAsync(progress.Add, new CancellationToken(true)); + + await run.Should().ThrowAsync(); + + progress.Should().BeEmpty(); + } + + [Test] + public void BootstrapHosts_FilterRoutineNoiseButKeepWarningsAndUnrelatedInformation() + { + using var host = DemoCatalogBootstrap.CreateProductsHost(Configuration(), "Host=localhost;Database=unused;Username=unused;Password=unused"); + var logging = host.Services.GetRequiredService(); + foreach (var category in new[] { "CoreEx.RefData.ReferenceDataOrchestrator", "Polly", "System.Net.Http.HttpClient.Ollama.LogicalHandler" }) + { + var logger = logging.CreateLogger(category); + logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Information).Should().BeFalse(); + logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Warning).Should().BeTrue(); + logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Error).Should().BeTrue(); + } + + logging.CreateLogger("ZiggyCreatures.Caching.Fusion").IsEnabled(Microsoft.Extensions.Logging.LogLevel.Warning).Should().BeTrue(); + logging.CreateLogger("Unrelated").IsEnabled(Microsoft.Extensions.Logging.LogLevel.Information).Should().BeTrue(); + } + + private static IConfiguration Configuration() => new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["E2E:Products:ConnectionString"] = "Host=localhost;Database=unused;Username=unused;Password=unused", + ["E2E:Shopping:ConnectionString"] = "Server=localhost;Initial Catalog=unused;User Id=unused;Password=unused", + ["Ollama:BaseAddress"] = "http://localhost:11434" + }).Build(); +} diff --git a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs index 8d5d31dde..207d123f5 100644 --- a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs +++ b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs @@ -18,7 +18,11 @@ global using ModelContextProtocol.Client; global using ModelContextProtocol.Protocol; global using NUnit.Framework; +global using Spectre.Console; +global using Spectre.Console.Rendering; +global using System.IO; global using System.Net; +global using System.Net.Http; global using System.Text.Json; global using UnitTestEx; global using UnitTestEx.Aspire; diff --git a/samples/aspire/Contoso.Test.Aspire/ProductSearchOperationTests.cs b/samples/aspire/Contoso.Test.Aspire/ProductSearchOperationTests.cs new file mode 100644 index 000000000..71b814b3e --- /dev/null +++ b/samples/aspire/Contoso.Test.Aspire/ProductSearchOperationTests.cs @@ -0,0 +1,270 @@ +namespace Contoso.Test.Aspire; + +[TestFixture] +public class ProductSearchOperationTests +{ + [TestCase("semantic", "Semantic")] + [TestCase("fulltext", "FullText")] + [TestCase("hybrid", "Hybrid")] + public async Task Search_UsesEncodedRestRequestAndServerMode_RendersRankedProductLite(string mode, string expectedMode) + { + const string text = "mud & oil + #?/$= café [red]"; + using var handler = new SearchHandler(_ => Json("""[{"sku":"Z-FIRST","text":"Trail [red]bike[/]","price":1234.5},{"sku":"A-SECOND","text":"Second","price":9.95}]"""), mode); + using var client = CreateClient(handler); + using var console = new ScriptConsole(text, "/back"); + + await new ProductSearchOperation(client, console).RunAsync(); + + handler.Requests.Should().Equal("/api/products/$query", $"/api/products?$search={Uri.EscapeDataString(text)}&$take=10"); + console.Output.Should().Contain($"Searching Products ({expectedMode})").And.Contain($"{expectedMode} · Returned 2 products in ") + .And.Contain("Rank").And.Contain("SKU").And.Contain("Product (Text)") + .And.Contain("Price (USD)").And.Contain("1,234.50").And.Contain("9.95").And.Contain("Trail [red]bike[/]").And.Contain(text); + console.Output.IndexOf("Z-FIRST", StringComparison.Ordinal).Should().BeLessThan(console.Output.IndexOf("A-SECOND", StringComparison.Ordinal)); + console.Output.Should().NotContain("total matches").And.NotContain("candidate pool").And.NotContain("HTTP request:") + .And.NotContain("excludes schema lookup").And.NotContain("Search: "); + console.Output.Split(text).Should().HaveCount(2, "the prompt echoes the input once; results must not repeat it"); + } + + [Test] + public async Task Search_NoResultsAndRepeat_ReusesSessionSchemaAndDoesNotRequestCount() + { + using var handler = new SearchHandler(_ => Json("[]")); + using var client = CreateClient(handler); + using var console = new ScriptConsole("nothing", "try again", ""); + + await new ProductSearchOperation(client, console).RunAsync(); + + handler.Requests.Should().Equal("/api/products/$query", "/api/products?$search=nothing&$take=10", + "/api/products?$search=try%20again&$take=10"); + console.Output.Split("No products found.").Should().HaveCount(3); + console.Output.Should().NotContain("Price (USD)"); + } + + [Test] + public async Task ReenteringOperation_RefreshesSessionSchema() + { + using var handler = new SearchHandler(_ => Json("[]")); + using var client = CreateClient(handler); + using var console = new ScriptConsole("first", "/back", "second", "/back"); + var operation = new ProductSearchOperation(client, console); + + await operation.RunAsync(); + await operation.RunAsync(); + + handler.Requests.Should().Equal("/api/products/$query", "/api/products?$search=first&$take=10", + "/api/products/$query", "/api/products?$search=second&$take=10"); + } + + [Test] + public async Task ElapsedTime_IncludesSchemaLookupAndResponseDecoding() + { + using var handler = new SearchHandler(_ => Json("[]"), beforeSchema: cancellationToken => Task.Delay(150, cancellationToken)); + using var client = CreateClient(handler); + using var console = new ScriptConsole("mud", "/back"); + + await new ProductSearchOperation(client, console).RunAsync(); + + var timing = System.Text.RegularExpressions.Regex.Match(console.Output, @"Returned 0 products in ([0-9.,]+)s"); + timing.Success.Should().BeTrue(); + double.Parse(timing.Groups[1].Value, System.Globalization.CultureInfo.CurrentCulture).Should().BeGreaterThanOrEqualTo(0.1); + console.Output.Should().Contain("Reading Products search mode...").And.Contain("Searching Products (Semantic)..."); + } + + [Test] + public async Task Cancellation_DuringSchemaLookupReturnsToMenuWithoutSearching() + { + using var cancellation = new CancellationTokenSource(); + using var handler = new SearchHandler(_ => throw new InvalidOperationException("Unexpected search."), beforeSchema: token => + { + cancellation.Cancel(); + token.ThrowIfCancellationRequested(); + return Task.CompletedTask; + }); + using var client = CreateClient(handler); + using var console = new ScriptConsole("mud"); + + await new ProductSearchOperation(client, console).RunAsync(cancellation.Token); + + handler.Requests.Should().Equal("/api/products/$query"); + console.Output.Should().Contain("Product Search cancelled; returning to the menu.").And.NotContain("Returned ").And.NotContain("Product Search failed:"); + } + + [TestCase("")] + [TestCase(" ")] + [TestCase("/back")] + [TestCase(" /BACK ")] + public async Task BackOrBlank_DoesNotCallApi(string text) + { + using var handler = new SearchHandler(_ => throw new InvalidOperationException("Unexpected search.")); + using var client = CreateClient(handler); + using var console = new ScriptConsole(text); + + await new ProductSearchOperation(client, console).RunAsync(); + + handler.Requests.Should().BeEmpty(); + } + + [Test] + public async Task FailedModelRequest_DisplaysEscapedProblemAndAllowsRetryWithoutFallback() + { + var attempt = 0; + using var handler = new SearchHandler(_ => ++attempt == 1 + ? new HttpResponseMessage(HttpStatusCode.ServiceUnavailable) + { + Content = new StringContent("""{"title":"Model [red]unready[/]","detail":"Install [model] before searching.","status":503}""", null, "application/problem+json") + } + : Json("[]")); + using var client = CreateClient(handler); + using var console = new ScriptConsole("mud", "retry", "/back"); + + await new ProductSearchOperation(client, console).RunAsync(); + + console.Output.Should().Contain("Product Search failed: Model [red]unready[/]").And.Contain("Install [model] before searching.") + .And.Contain("No fallback or rebuild was attempted.").And.Contain("No products found."); + handler.Requests.Should().HaveCount(3); + handler.Requests.Should().OnlyContain(r => r == "/api/products/$query" || r.EndsWith("&$take=10", StringComparison.Ordinal)); + } + + [TestCase("http")] + [TestCase("connection")] + [TestCase("timeout")] + [TestCase("json")] + [TestCase("null")] + public async Task RequestFailure_IsVisibleAndDoesNotBecomeNoResults(string failure) + { + using var handler = new SearchHandler(_ => failure switch + { + "http" => new HttpResponseMessage(HttpStatusCode.BadGateway), + "connection" => throw new HttpRequestException("API [offline]"), + "timeout" => throw new TaskCanceledException("Timed out."), + "json" => Json("{not-json}"), + _ => Json("null") + }); + using var client = CreateClient(handler); + using var console = new ScriptConsole("mud", "/back"); + + await new ProductSearchOperation(client, console).RunAsync(); + + console.Output.Should().Contain("Product Search failed:").And.NotContain("No products found."); + if (failure == "timeout") + console.Output.Should().Contain("Products API request timed out."); + if (failure == "connection") + console.Output.Should().Contain("API [offline]"); + handler.Requests.Should().HaveCount(2); + } + + [TestCase("missing")] + [TestCase("keyword")] + public async Task UnsupportedSchema_FailsWithoutSearchingOrInventingMode(string mode) + { + using var handler = new SearchHandler(_ => throw new InvalidOperationException("Unexpected search."), mode); + using var client = CreateClient(handler); + using var console = new ScriptConsole("mud", "/back"); + + await new ProductSearchOperation(client, console).RunAsync(); + + handler.Requests.Should().Equal("/api/products/$query"); + console.Output.Should().Contain("does not advertise a supported").And.NotContain("No products found.").And.NotContain("Server search mode:"); + } + + [Test] + public async Task Cancellation_FlowsToHttpAndReturnsToMenu() + { + using var cancellation = new CancellationTokenSource(); + using var handler = new SearchHandler(cancellationToken => + { + cancellation.Cancel(); + cancellationToken.ThrowIfCancellationRequested(); + return Json("[]"); + }); + using var client = CreateClient(handler); + using var console = new ScriptConsole("mud"); + + await new ProductSearchOperation(client, console).RunAsync(cancellation.Token); + + console.Output.Should().Contain("Product Search cancelled; returning to the menu.").And.NotContain("Product Search failed:").And.NotContain("No products found."); + handler.Requests.Should().HaveCount(2); + } + + [Test] + public async Task AlreadyCancelled_DoesNotPromptOrRequest() + { + using var handler = new SearchHandler(_ => throw new InvalidOperationException("Unexpected search.")); + using var client = CreateClient(handler); + using var console = new ScriptConsole(); + + await new ProductSearchOperation(client, console).RunAsync(new CancellationToken(true)); + + handler.Requests.Should().BeEmpty(); + console.Output.Should().Contain("Product Search cancelled"); + } + + [Test] + public void InteractiveOperation_IsNotDiscoveredAsScenarioOrSetup() + { + var assembly = typeof(ProductSearchOperation).Assembly; + Contoso.E2E.Runner.Scenarios.ScenarioManager.Create(assembly).Scenarios.Values + .Should().NotContain(s => s.Factory().GetType() == typeof(ProductSearchOperation)); + Contoso.E2E.Runner.Scenarios.SetUpManager.Create(assembly).SetUps.Values + .Should().NotContain(s => s.Factory().GetType() == typeof(ProductSearchOperation)); + } + + private static HttpClient CreateClient(HttpMessageHandler handler) => new(handler) { BaseAddress = new Uri("https://products.example/") }; + + private static HttpResponseMessage Json(string value) => new(HttpStatusCode.OK) { Content = new StringContent(value, null, "application/json") }; + + private sealed class SearchHandler(Func search, string mode = "semantic", Func? beforeSchema = null) : HttpMessageHandler + { + public List Requests { get; } = []; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + request.Method.Should().Be(HttpMethod.Get); + var path = request.RequestUri!.PathAndQuery; + Requests.Add(path); + if (path == "/api/products/$query" && beforeSchema is not null) + await beforeSchema(cancellationToken).ConfigureAwait(false); + return path == "/api/products/$query" + ? Json(mode == "missing" ? "{}" : JsonSerializer.Serialize(new { search = new { mode, ranked = true } })) + : search(cancellationToken); + } + } + + internal sealed class ScriptConsole : IAnsiConsole, IAnsiConsoleInput, IDisposable + { + private readonly StringWriter _output = new(); + private readonly IAnsiConsole _console; + private readonly Queue _keys = new(); + + public ScriptConsole(params string[] lines) + { + _console = AnsiConsole.Create(new AnsiConsoleSettings { Ansi = AnsiSupport.No, Interactive = InteractionSupport.Yes, Out = new AnsiConsoleOutput(_output) }); + _console.Profile.Width = 180; + foreach (var line in lines) + { + foreach (var character in line) + _keys.Enqueue(new ConsoleKeyInfo(character, (ConsoleKey)0, false, false, false)); + _keys.Enqueue(new ConsoleKeyInfo('\r', ConsoleKey.Enter, false, false, false)); + } + } + + public string Output => _output.ToString(); + public Profile Profile => _console.Profile; + public IAnsiConsoleCursor Cursor => _console.Cursor; + public IAnsiConsoleInput Input => this; + public IExclusivityMode ExclusivityMode => _console.ExclusivityMode; + public RenderPipeline Pipeline => _console.Pipeline; + public void Clear(bool home) => _console.Clear(home); + public void Write(IRenderable renderable) => _console.Write(renderable); + public void WriteAnsi(Action action) => _console.WriteAnsi(action); + public bool IsKeyAvailable() => _keys.Count > 0; + public ConsoleKeyInfo? ReadKey(bool intercept) => _keys.Dequeue(); + public Task ReadKeyAsync(bool intercept, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(ReadKey(intercept)); + } + + public void Dispose() => _output.Dispose(); + } +} diff --git a/samples/docs/aspire.md b/samples/docs/aspire.md index 6f3aec118..b19b55af1 100644 --- a/samples/docs/aspire.md +++ b/samples/docs/aspire.md @@ -191,6 +191,14 @@ Per-scenario simulation parallelism and delay bounds are also configurable: } ``` +### Interactive Product Search + +The Aspire demo selects **Hybrid** search for both Products API and Products Subscribe. Set `Products:Search:Mode` on the AppHost to override this with `Semantic` or `FullText`; the same setting is forwarded to both hosts. Standalone hosts retain their existing Semantic default. After restarting with a different mode, exit and re-enter the runner's Product Search operation to refresh its session-cached mode. + +Select **Interactive operations → Product Search** to search the configured Products REST API. Enter search text to request `/api/products?$search=&$take=10`. Results appear in server rank order with **Rank**, **SKU**, **Product (Text)** and **Price (USD)** columns, without repeating the entered text. The runner reads the server's **Semantic**, **FullText** or **Hybrid** mode from `/api/products/$query` on the first search and caches it until returning to the menu; re-enter Product Search to refresh the displayed mode after changing server configuration. Failed schema lookups are retried on the next search. This lookup discovers the configured mode, not products, and never overrides it. Loading messages identify mode discovery and search; the result summary shows mode, returned rows and total elapsed time including any schema lookup and response decoding. No total-match count is requested or inferred (Hybrid uses bounded candidate pools). + +Enter another query to repeat, or submit blank input or `/back` to return to the main menu. **Ctrl+C** cancels the operation, including an in-flight HTTP request, and returns to the menu. Only Products needs to be available: this operation is not gated on Shopping or the other APIs. API, schema, timeout and model/index readiness failures are displayed explicitly; there is no fallback, implicit reset, seeding or index rebuild. Prepare the demo catalog separately if needed, observing the reset cautions below. Product Search is not a scenario and is never included in **Run all scenarios as simulation** or automated scenario discovery. + ### Scenarios #### Set-Up (run once) @@ -213,35 +221,15 @@ The Shopping Basket Lifecycle is the richest scenario — it exercises every int ### Shared demo catalog bootstrap -[`samples/data/demo-catalog.seed.yaml`](../data/demo-catalog.seed.yaml) is the single catalog source for -interactive demo/MCP, `DatabaseMigrationSetup`, and search relevance evaluation. It is linked/embedded in -`Contoso.Products.Database`, so the runner needs neither a repository checkout nor cwd-relative seed paths. -Research provenance is in [`demo-catalog.sources.json`](../data/demo-catalog.sources.json). Ordinary -Products/Shopping API, subscriber and automated Aspire test catalogs are deliberately unchanged. - -The snapshot has 174 source rows: 170 active MTB products, two inactive MTB products, one logically deleted -product, and the original non-stocked `LABOR` service. Original deterministic `^1`–`^32` IDs are preserved. -The **173 non-deleted** rows are read through Products' CoreEx accessor/mapping (inactive rows are retained), -serialized across the same JSON contract boundary used by events, and passed to Shopping's normal -`IProductSyncAdapter.ModifyAsync`/`DeleteAsync`. Existing Shopping mapping/upsert owns persistence. -There is no second Shopping catalog and no cross-database SQL between PostgreSQL and SQL Server. -Only this setup composition directly resolves both domains; business services still use adapters/HTTP/events. -The domains have separate DI/execution scopes and fresh local reference-data caches. Added Products -brands/subcategories live in Products' normal reference seed; Shopping only loads its own reference data. - -Shopping's product-free [`demo-baskets.seed.yaml`](../tests/Contoso.E2E.Runner/Data/demo-baskets.seed.yaml) -is loaded **after** product synchronization, retaining the original basket/item IDs and scenario state. -Setup verifies all replica IDs/SKUs/text/units/prices/activity/non-stocked flags, absence of stale/deleted -replicas, domain reference-data availability and basket/item references. It builds both lexical and -semantic projections using `AddProductEmbeddings` and the configured real provider/profile, verifies readiness -and query inference, and fails visibly if migrations, mappings, inference or integrity checks fail. -It evicts only the affected domains' reference-data collection keys through the normal Redis/FusionCache -backplane, waiting for distributed operations and surfacing failures; it never flushes shared Redis. - -**Snapshot bootstrap is not replication evidence.** It does not publish historical events or exercise -outbox/relay/broker delivery. Subsequent E2E product-change and basket scenarios continue exercising those paths. -Run setup with writers/relays/subscribers stopped, then restart the hosts; never use it to reconcile a live -catalog while scenarios are writing. Restart/clear serving hosts' reference-data caches after resetting data. +Setup reports each real stage: Products migration/reset and catalog load, Shopping migration/reset, current-product synchronization, basket load, targeted cache invalidation, lexical rebuild, document loading, semantic embedding, index publication and final verification. Embedding counts advance only after a valid model response (reported at 0, 1, every 10 products and the final count); reaching N/N is not overall completion, which requires successful publication and verification. Each completed stage and the successful bootstrap show elapsed seconds. Durable progress lines also work when output is redirected and do not overwrite warnings. Only bootstrap-created hosts filter routine reference-data, HTTP-client and Polly information logs; warnings/errors, including FusionCache serializer warnings, remain visible. Ctrl+C cancels setup; failures and cancellation never print bootstrap success, and incomplete setup must be rerun before starting hosts. + +[`samples/data/demo-catalog.seed.yaml`](../data/demo-catalog.seed.yaml) is the single catalog source for interactive demo/MCP, `DatabaseMigrationSetup`, and search relevance evaluation. It is linked/embedded in `Contoso.Products.Database`, so the runner needs neither a repository checkout nor cwd-relative seed paths. Research provenance is in [`demo-catalog.sources.json`](../data/demo-catalog.sources.json). Ordinary Products/Shopping API, subscriber and automated Aspire test catalogs are deliberately unchanged. + +The snapshot has 174 source rows: 170 active MTB products, two inactive MTB products, one logically deleted product, and the original non-stocked `LABOR` service. Original deterministic `^1`–`^32` IDs are preserved. The **173 non-deleted** rows are read through Products' CoreEx accessor/mapping (inactive rows are retained), serialized across the same JSON contract boundary used by events, and passed to Shopping's normal `IProductSyncAdapter.ModifyAsync`/`DeleteAsync`. Existing Shopping mapping/upsert owns persistence. There is no second Shopping catalog and no cross-database SQL between PostgreSQL and SQL Server. Only this setup composition directly resolves both domains; business services still use adapters/HTTP/events. The domains have separate DI/execution scopes and fresh local reference-data caches. Added Products brands/subcategories live in Products' normal reference seed; Shopping only loads its own reference data. + +Shopping's product-free [`demo-baskets.seed.yaml`](../tests/Contoso.E2E.Runner/Data/demo-baskets.seed.yaml) is loaded **after** product synchronization, retaining the original basket/item IDs and scenario state. Setup verifies all replica IDs/SKUs/text/units/prices/activity/non-stocked flags, absence of stale/deleted replicas, domain reference-data availability and basket/item references. It builds both lexical and semantic projections using `AddProductEmbeddings` and the configured real provider/profile, verifies readiness and query inference, and fails visibly if migrations, mappings, inference or integrity checks fail. It evicts only the affected domains' reference-data collection keys through the normal Redis/FusionCache backplane, waiting for distributed operations and surfacing failures; it never flushes shared Redis. + +**Snapshot bootstrap is not replication evidence.** It does not publish historical events or exercise outbox/relay/broker delivery. Subsequent E2E product-change and basket scenarios continue exercising those paths. Run setup with writers/relays/subscribers stopped, then restart the hosts; never use it to reconcile a live catalog while scenarios are writing. Restart/clear serving hosts' reference-data caches after resetting data. For Products/Shopping-only maintenance (no API health checks, Customers/Cosmos or Orders reset): @@ -252,18 +240,9 @@ $env:CoreExLocalTargetFramework = 'net10.0' dotnet run --project samples\tests\Contoso.E2E.Runner -- --demo-catalog-only --accept-demo-reset ``` -**Destructive:** the command resets demo data and runs migrations in the configured Products/Shopping databases. -The explicit flag acknowledges that reset; never target another person's shared databases without agreement. -The full interactive migration option additionally resets Customers and Orders. A model failure may leave a -partially prepared database; fix the reported cause and rerun the complete bootstrap before starting hosts. -`Data Seeding for E2E Testing` still performs stock adjustments through the API after hosts are running; -the base snapshot's inventory is intentionally small. +**Destructive:** the command resets demo data and runs migrations in the configured Products/Shopping databases. The explicit flag acknowledges that reset; never target another person's shared databases without agreement. The full interactive migration option additionally resets Customers and Orders. A model failure may leave a partially prepared database; fix the reported cause and rerun the complete bootstrap before starting hosts. `Data Seeding for E2E Testing` still performs stock adjustments through the API after hosts are running; the base snapshot's inventory is intentionally small. -The opt-in `DemoCatalogBootstrapTests` fixture lives in the single-target `Contoso.Test.Aspire` project, -but directly exercises setup composition without starting AppHost or the ordinary smoke fixtures. -It refuses to run unless E2E connections target exactly -`contoso_search_eval` and `Contoso_demo_eval`, and its Redis connection must use `defaultDatabase=14`. -With existing PostgreSQL/SQL Server/Redis/Ollama infrastructure, set the two database overrides and run: +The opt-in `DemoCatalogBootstrapTests` fixture lives in the single-target `Contoso.Test.Aspire` project, but directly exercises setup composition without starting AppHost or the ordinary smoke fixtures. It refuses to run unless E2E connections target exactly `contoso_search_eval` and `Contoso_demo_eval`, and its Redis connection must use `defaultDatabase=14`. With existing PostgreSQL/SQL Server/Redis/Ollama infrastructure, set the two database overrides and run: ```powershell $env:CoreExLocalTargetFramework = 'net10.0' @@ -274,8 +253,7 @@ $env:Aspire__StackExchange__Redis__ConnectionString = 'localhost:6379,defaultDat dotnet test samples\aspire\Contoso.Test.Aspire --framework net10.0 --filter 'FullyQualifiedName~DemoCatalogBootstrapTests' ``` -It exercises actual source/replica agreement, flags/soft deletion, stale removal, basket integrity, -repeatable reconciliation/reset, cancellation and explicit real-provider failure. It does not touch Cosmos. +It exercises actual source/replica agreement, flags/soft deletion, stale removal, basket integrity, repeatable reconciliation/reset, cancellation and explicit real-provider failure. It does not touch Cosmos. #### Load simulation diff --git a/samples/docs/search.md b/samples/docs/search.md index 04184aaed..4afb11bf0 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -299,14 +299,7 @@ Keep these sources and descriptions aligned when changing the fixtures. A produc The expanded shared demo catalog has a [product/source manifest](../data/demo-catalog.sources.json): each added entry carries a source URL, source type, original short description, internal demo SKU, and illustrative USD price. FOX in the catalog means FOX Factory suspension; FOXRACING is the separate Fox Racing apparel/protection brand. Product facts and model-family descriptions are supported by their linked manufacturer pages (with retailer sources explicitly identified where necessary). Tags `e-bike` and `emtb` are user-requested colloquial search aliases on the three verified electric mountain bikes, alongside the `electric` category term. `dentist-bike` is a user-provided informal search alias on Yeti bikes for an offhand demo query. It is neither manufacturer provenance nor a factual claim about intended riders or affordability. -The shared `LABOR` listing retains its hourly, non-stocked workshop-service identity and explicitly describes -professional MTB maintenance/repair labour without promising particular operations or included parts. -Its curated service-intent tags include `bike-service`, `bicycle-repair`, `workshop-labour`/`workshop-labor`, -`bike-mechanic` and `maintenance`. User-requested `wrench-monkey`, `grease-monkey`, `bike-whisperer` and -`spanner-man` are informal demo search aliases, not manufacturer facts or staff credentials; their provenance -is recorded separately under the manifest's sample service definitions. `hourly bike mechanic` and -`bike-whisperer` are interactive service-intent probes, not new MTB benchmark judgments: the benchmark's -`sku ne 'LABOR'` filter intentionally excludes this workshop service. +The shared `LABOR` listing retains its hourly, non-stocked workshop-service identity and explicitly describes professional MTB maintenance/repair labour without promising particular operations or included parts. Its curated service-intent tags include `bike-service`, `bicycle-repair`, `workshop-labour`/`workshop-labor`, `bike-mechanic` and `maintenance`. User-requested `wrench-monkey`, `grease-monkey`, `bike-whisperer` and `spanner-man` are informal demo search aliases, not manufacturer facts or staff credentials; their provenance is recorded separately under the manifest's sample service definitions. `hourly bike mechanic` and `bike-whisperer` are interactive service-intent probes, not new MTB benchmark judgments: the benchmark's `sku ne 'LABOR'` filter intentionally excludes this workshop service. ### Keep rapidly changing facts relational @@ -377,43 +370,17 @@ Search does not resolve authentication or authorization. The sample's existing a ### Initial readiness: an explicit, atomic backfill -Rebuilding is reusable Infrastructure logic, **not an API startup responsibility**. There is no HTTP rebuild -endpoint. The E2E Runner's [shared demo bootstrap](aspire.md#shared-demo-catalog-bootstrap) invokes the same -Infrastructure rebuild explicitly, including its `--demo-catalog-only --accept-demo-reset` maintenance path. - -Products read-test `OneTimeSetUp` seeds the catalog and invokes `ProductSearchIndex.RebuildAsync` with controlled -embeddings. Aspire E2E setup seeds the catalog, clears Redis, then creates a small test-owned service scope using -the actual PostgreSQL resource connection string and AppHost Ollama configuration. It invokes the same rebuild -logic with the real model before starting the hosts. The scope uses fresh in-memory reference-data caching over -the real database; the serving API uses its normal Redis-backed cache, cleared by setup. The AppHost exposes -Ollama as a Compose-owned external resource. CI explicitly starts the search profile and pulls the selected model. -Neither API nor AppHost startup seeds or rebuilds anything during ordinary manual execution. - -The API, Subscribe host and real-model test fixture register `CommunityToolkit.Aspire.OllamaSharp` through the -shared `AddProductEmbeddings` composition helper. Aspire registers `IOllamaApiClient`; CoreEx's `AddNamedHttpClient` -augments the SDK's named client using the separate `Ollama` configuration section, adding idempotency handling, -service discovery and standard HTTP resilience from `Ollama:Resilience`, -with `HttpClient.Timeout` disabled so the resilience timeouts control request duration. The container remains -Compose-owned; the client integration does not launch it. - -The SDK handles request serialization, transport and response disposal. `OllamaHttpClient` is now a validated SDK -wrapper, not a hand-written HTTP transport. It preserves Nomic's prefixes, `truncate=false`, the pinned model and -768-dimensional finite, non-zero vector checks. OllamaSharp's response type omits the returned model name, so a -small read-only response converter retains that field for validation. Provider/protocol errors return failed -CoreEx results; cancellation still propagates. No provider types leak through `IProductEmbeddingAdapter`. -The SDK's direct embedding API is used instead of `IEmbeddingGenerator` to retain the provider-specific truncation -policy and response model metadata; portability remains at the application-owned adapter boundary. +Rebuilding is reusable Infrastructure logic, **not an API startup responsibility**. There is no HTTP rebuild endpoint. The E2E Runner's [shared demo bootstrap](aspire.md#shared-demo-catalog-bootstrap) invokes the same Infrastructure rebuild explicitly, including its `--demo-catalog-only --accept-demo-reset` maintenance path. + +Products read-test `OneTimeSetUp` seeds the catalog and invokes `ProductSearchIndex.RebuildAsync` with controlled embeddings. Aspire E2E setup seeds the catalog, clears Redis, then creates a small test-owned service scope using the actual PostgreSQL resource connection string and AppHost Ollama configuration. It invokes the same rebuild logic with the real model before starting the hosts. The scope uses fresh in-memory reference-data caching over the real database; the serving API uses its normal Redis-backed cache, cleared by setup. The AppHost exposes Ollama as a Compose-owned external resource. CI explicitly starts the search profile and pulls the selected model. Neither API nor AppHost startup seeds or rebuilds anything during ordinary manual execution. + +The API, Subscribe host and real-model test fixture register `CommunityToolkit.Aspire.OllamaSharp` through the shared `AddProductEmbeddings` composition helper. Aspire registers `IOllamaApiClient`; CoreEx's `AddNamedHttpClient` augments the SDK's named client using the separate `Ollama` configuration section, adding idempotency handling, service discovery and standard HTTP resilience from `Ollama:Resilience`, with `HttpClient.Timeout` disabled so the resilience timeouts control request duration. The container remains Compose-owned; the client integration does not launch it. + +The SDK handles request serialization, transport and response disposal. `OllamaHttpClient` is now a validated SDK wrapper, not a hand-written HTTP transport. It preserves Nomic's prefixes, `truncate=false`, the pinned model and 768-dimensional finite, non-zero vector checks. OllamaSharp's response type omits the returned model name, so a small read-only response converter retains that field for validation. Provider/protocol errors return failed CoreEx results; cancellation still propagates. No provider types leak through `IProductEmbeddingAdapter`. The SDK's direct embedding API is used instead of `IEmbeddingGenerator` to retain the provider-specific truncation policy and response model metadata; portability remains at the application-owned adapter boundary. The Products Subscribe host registers the Ollama client for asynchronous product indexing. Ordinary reservation handlers do not call the model and consume a separate command queue. -The rebuild indexes the current non-deleted catalog, including inactive products so `$inactive` works. It does -not seed products or download the model. It prepares documents and calls Ollama **before** starting a database -transaction when invoked outside an existing transaction. Only publication takes a short framework-managed unit -of work: lock the product table against writes, verify that its searchable documents still match the captured -catalog, replace the selected profile/recipe, and commit. A model failure leaves the previous vector index untouched; -a concurrent searchable catalog edit rejects publication. Deleted-product entries disappear on rebuild, and -physical deletion also cascades through the foreign key. Other profiles remain separate. Rebuild and synchronization -reject a parent transaction: the lexical publication must commit independently before any model work. +The rebuild indexes the current non-deleted catalog, including inactive products so `$inactive` works. It does not seed products or download the model. It prepares documents and calls Ollama **before** starting a database transaction when invoked outside an existing transaction. Only publication takes a short framework-managed unit of work: lock the product table against writes, verify that its searchable documents still match the captured catalog, replace the selected profile/recipe, and commit. A model failure leaves the previous vector index untouched; a concurrent searchable catalog edit rejects publication. Deleted-product entries disappear on rebuild, and physical deletion also cascades through the foreign key. Other profiles remain separate. Rebuild and synchronization reject a parent transaction: the lexical publication must commit independently before any model work. `products.product_search_readiness` records successful initial preparation for each profile/recipe. The readiness marker commits atomically with backfill publication, even for an empty catalog. Incremental events never mark a partially indexed catalog ready. A failed initial backfill leaves it unready; a failed later backfill preserves the previous published projection and readiness. @@ -440,57 +407,23 @@ There is no relevance cutoff: semantic mode returns every eligible compatible ve ### Independent lexical projection -`ProductLexicalSearchIndex` owns `products.product_lexical_search` and its separate -`product_lexical_search_readiness` table. Both are SQL-only derived state managed by the new DbEx migration, -not generated CRUD models. There is exactly one row per product. Recipe `product-lexical-english-v1` uses -explicit `pg_catalog.english` processing on both documents and queries: +`ProductLexicalSearchIndex` owns `products.product_lexical_search` and its separate `product_lexical_search_readiness` table. Both are SQL-only derived state managed by the new DbEx migration, not generated CRUD models. There is exactly one row per product. Recipe `product-lexical-english-v1` uses explicit `pg_catalog.english` processing on both documents and queries: - Weight A: canonical SKU and product name. - Weight B: brand/category/subcategory labels (code fallback) and existing JSON tags. - Weight C: description. -The document is a PostgreSQL `tsvector`; its GIN index supports lexical matching. A separate SKU B-tree -supports the exact-match alternative. `websearch_to_tsquery` receives bound text, not interpolated SQL or -raw `to_tsquery` syntax. English stemming and stopwords apply; punctuation is parsed by PostgreSQL. -Ordinary words are ANDed; quoted phrases, `OR`, and exclusions follow PostgreSQL websearch semantics. -This is not BM25, substring matching, fuzzy spelling correction, synonym expansion, or semantic retrieval. - -The entire normalized input is also compared with the stored SKU after trimming and the contract's existing -uppercase transformation. An exact eligible SKU is included even if tokenization removes it (e.g. a stopword -SKU). Ranking is exact-SKU first, then descending `ts_rank_cd` using PostgreSQL's default weight values and -no length normalization, then product ID. The OR predicate does not union duplicate rows. An exact but -inactive, deleted, or filtered-out SKU cannot bypass live eligibility. - -The repository composes these matches with the **same** product/subcategory/inventory joins and all existing -filters. It counts and pages the complete matching set in a `RepeatableRead` snapshot; compatible parent -snapshots enlist, weaker ones fail explicitly. An explicit `$orderby` is rejected. No-match, stopword-only, -and punctuation-only inputs return an empty set, not semantic neighbors or the whole catalog. An unprepared -lexical recipe returns HTTP 503. Cancellation propagates. Existing `QueryArgs` behavior is unchanged: -an empty/whitespace `$search` is treated as absent and performs an ordinary catalog query, not lexical matching. - -Lexical synchronization reads current authoritative state and publishes it in one short transaction. -It shares the product source-write lock, also locks reference-data sources during publication, and serializes -lexical publishers. Delete/reinsert is atomic to readers; duplicate or late events reconcile current state. -The complete lexical rebuild replaces all rows and publishes readiness in that **same** transaction, including -an empty catalog. Individual events cannot mark a partly indexed initial catalog ready. Cancellation/failure -rolls back publication and preserves any prior complete version. - -In default semantic mode, `ProductSearchIndex.SynchronizeAsync` and `RebuildAsync` first **commit** lexical -work and only then call the embedding provider. A semantic failure still retries the event, but cannot roll back -already-published lexical content/readiness. In explicitly configured FullText Subscribe mode, the application -adapter calls **only** the lexical index: an offline model does not cause event retries or prevent later lexical -updates. Configure API and Subscribe consistently. No background poller, automatic startup backfill, new service, -or new package is introduced. - -For a lexical-only initial backfill call `ProductLexicalSearchIndex.RebuildAsync` from a maintenance/test-owned -scope. The guarded evaluation fixture below selects that path when `Products__Search__Mode=FullText`. -Switching an installation that has run lexical-only maintenance back to Semantic requires a semantic rebuild: -FullText mode does not maintain vector rows. Changing the lexical document/configuration recipe also requires a -version bump and full rebuild. Readiness means a completed initial build, **not** a proof of continuous freshness. -Events can lag or be missed; live filters still exclude ineligible rows but term matching can temporarily reflect -older content. Reference-data-only edits have no automatic reindex trigger; rebuild explicitly. Lexical labels -come directly from database text, not localized request/cache values; English is the sole supported language. -The table locks/full rebuild are deliberately small-catalog sample choices, not a large-catalog online rebuild design. +The document is a PostgreSQL `tsvector`; its GIN index supports lexical matching. A separate SKU B-tree supports the exact-match alternative. `websearch_to_tsquery` receives bound text, not interpolated SQL or raw `to_tsquery` syntax. English stemming and stopwords apply; punctuation is parsed by PostgreSQL. Ordinary words are ANDed; quoted phrases, `OR`, and exclusions follow PostgreSQL websearch semantics. This is not BM25, substring matching, fuzzy spelling correction, synonym expansion, or semantic retrieval. + +The entire normalized input is also compared with the stored SKU after trimming and the contract's existing uppercase transformation. An exact eligible SKU is included even if tokenization removes it (e.g. a stopword SKU). Ranking is exact-SKU first, then descending `ts_rank_cd` using PostgreSQL's default weight values and no length normalization, then product ID. The OR predicate does not union duplicate rows. An exact but inactive, deleted, or filtered-out SKU cannot bypass live eligibility. + +The repository composes these matches with the **same** product/subcategory/inventory joins and all existing filters. It counts and pages the complete matching set in a `RepeatableRead` snapshot; compatible parent snapshots enlist, weaker ones fail explicitly. An explicit `$orderby` is rejected. No-match, stopword-only, and punctuation-only inputs return an empty set, not semantic neighbors or the whole catalog. An unprepared lexical recipe returns HTTP 503. Cancellation propagates. Existing `QueryArgs` behavior is unchanged: an empty/whitespace `$search` is treated as absent and performs an ordinary catalog query, not lexical matching. + +Lexical synchronization reads current authoritative state and publishes it in one short transaction. It shares the product source-write lock, also locks reference-data sources during publication, and serializes lexical publishers. Delete/reinsert is atomic to readers; duplicate or late events reconcile current state. The complete lexical rebuild replaces all rows and publishes readiness in that **same** transaction, including an empty catalog. Individual events cannot mark a partly indexed initial catalog ready. Cancellation/failure rolls back publication and preserves any prior complete version. + +In default semantic mode, `ProductSearchIndex.SynchronizeAsync` and `RebuildAsync` first **commit** lexical work and only then call the embedding provider. A semantic failure still retries the event, but cannot roll back already-published lexical content/readiness. In explicitly configured FullText Subscribe mode, the application adapter calls **only** the lexical index: an offline model does not cause event retries or prevent later lexical updates. Configure API and Subscribe consistently. No background poller, automatic startup backfill, new service, or new package is introduced. + +For a lexical-only initial backfill call `ProductLexicalSearchIndex.RebuildAsync` from a maintenance/test-owned scope. The guarded evaluation fixture below selects that path when `Products__Search__Mode=FullText`. Switching an installation that has run lexical-only maintenance back to Semantic requires a semantic rebuild: FullText mode does not maintain vector rows. Changing the lexical document/configuration recipe also requires a version bump and full rebuild. Readiness means a completed initial build, **not** a proof of continuous freshness. Events can lag or be missed; live filters still exclude ineligible rows but term matching can temporarily reflect older content. Reference-data-only edits have no automatic reindex trigger; rebuild explicitly. Lexical labels come directly from database text, not localized request/cache values; English is the sole supported language. The table locks/full rebuild are deliberately small-catalog sample choices, not a large-catalog online rebuild design. ### Asynchronous maintenance @@ -794,8 +727,7 @@ Rerun and review the benchmark when the embedding model/profile, document recipe ### Lexical tuning-only comparison -After the guarded `contoso_search_eval` Reset/Data/Execute sequence above, set the isolated connection and -FullText mode for the indexing fixture and the separately started Products API: +After the guarded `contoso_search_eval` Reset/Data/Execute sequence above, set the isolated connection and FullText mode for the indexing fixture and the separately started Products API: ```powershell $env:Aspire__Npgsql__ConnectionString = $connectionString @@ -806,21 +738,11 @@ dotnet test samples\tests\Contoso.Products.Test.Api --no-build --framework net10 -OutputPath artifacts\product-search-lexical-tuning.json ``` -This evaluator makes **no Ollama requests** in FullText mode, identifies the lexical recipe and embedded SQL -hashes, and selects tuning queries **before issuing any search requests**. Full-text without `-Split tuning` -is rejected. Semantic evaluation retains its original full-split default; `-Split tuning` also supports a -semantic tuning-only run. A baseline comparison is valid only for a report with the identical dataset, -catalog, seed, and query IDs; the historical v1 baseline therefore cannot be used for these v2 commands. -No judgments change. +This evaluator makes **no Ollama requests** in FullText mode, identifies the lexical recipe and embedded SQL hashes, and selects tuning queries **before issuing any search requests**. Full-text without `-Split tuning` is rejected. Semantic evaluation retains its original full-split default; `-Split tuning` also supports a semantic tuning-only run. A baseline comparison is valid only for a report with the identical dataset, catalog, seed, and query IDs; the historical v1 baseline therefore cannot be used for these v2 commands. No judgments change. -Lexical counts are matching-set counts, not semantic full-catalog candidate counts. The runner checks that -they are bounded by the live eligible catalog and that page size agrees with count. It cannot infer lexical -index completeness from a naturally sparse/no-match result; the guarded host fixture performs the actual full -backfill and verifies the database-derived non-deleted source-row count (172 in the current v2 seed; 170 active). No fake “every product must match” completeness -probe is sent to the lexical branch. +Lexical counts are matching-set counts, not semantic full-catalog candidate counts. The runner checks that they are bounded by the live eligible catalog and that page size agrees with count. It cannot infer lexical index completeness from a naturally sparse/no-match result; the guarded host fixture performs the actual full backfill and verifies the database-derived non-deleted source-row count (172 in the current v2 seed; 170 active). No fake “every product must match” completeness probe is sent to the lexical branch. -Historical v1 measurement, recorded locally on **2026-10-15**, through the real FullText REST API over the -then-current isolated seed. These results are not measurements of v2: +Historical v1 measurement, recorded locally on **2026-10-15**, through the real FullText REST API over the then-current isolated seed. These results are not measurements of v2: | Tuning metric | Original semantic baseline | Independent lexical v1 | |---|---:|---:| @@ -829,23 +751,13 @@ then-current isolated seed. These results are not measurements of v2: | Exact-SKU top-1 | 2/3 | **3/3** | | Returned constrained results (3 price queries) | 15 | **0** | -The lexical branch returned no matches for 14 of 18 tuning queries, including all three price-constrained -requests. Zero filter violations across zero constrained results is **not** evidence of useful constrained -retrieval; its violation rate is undefined. The exact-SKU improvement is real, but long natural-language -requests (often containing price words/numbers) are too restrictive under websearch's AND semantics, and -stemming does not supply absent synonyms or paraphrases. Only the three exact-SKU requests and the cross-country -race request returned results. No parsing policy, catalog content, or judgments were adjusted to improve these -scores. This is a useful independent lexical baseline, **not** an improvement to replace semantic defaults. +The lexical branch returned no matches for 14 of 18 tuning queries, including all three price-constrained requests. Zero filter violations across zero constrained results is **not** evidence of useful constrained retrieval; its violation rate is undefined. The exact-SKU improvement is real, but long natural-language requests (often containing price words/numbers) are too restrictive under websearch's AND semantics, and stemming does not supply absent synonyms or paraphrases. Only the three exact-SKU requests and the cross-country race request returned results. No parsing policy, catalog content, or judgments were adjusted to improve these scores. This is a useful independent lexical baseline, **not** an improvement to replace semantic defaults. -The ignored historical report `artifacts/product-search-lexical-tuning.json` contains the 18 v1 per-query -results, matched counts, identities and baseline deltas; test/setup/API diagnostics are also under `artifacts/`. -The checked-in semantic v1 baseline remains unchanged. No held-out lexical requests or scores were run. +The ignored historical report `artifacts/product-search-lexical-tuning.json` contains the 18 v1 per-query results, matched counts, identities and baseline deltas; test/setup/API diagnostics are also under `artifacts/`. The checked-in semantic v1 baseline remains unchanged. No held-out lexical requests or scores were run. ### Hybrid tuning and final held-out comparison -Use the same isolated setup sequence above. Hybrid setup invokes the real semantic rebuild, which also rebuilds -the lexical projection, and checks both projections against database-derived row counts (172 non-deleted, 170 active). Set Hybrid mode both for the -explicit setup fixture and for the separately started Products API: +Use the same isolated setup sequence above. Hybrid setup invokes the real semantic rebuild, which also rebuilds the lexical projection, and checks both projections against database-derived row counts (172 non-deleted, 170 active). Set Hybrid mode both for the explicit setup fixture and for the separately started Products API: ```powershell $env:Aspire__Npgsql__ConnectionString = $connectionString @@ -856,29 +768,18 @@ dotnet test samples\tests\Contoso.Products.Test.Api --no-build --framework net10 -OutputPath artifacts\product-search-hybrid-tuning.json ``` -Hybrid evaluation verifies the same exact eligible catalog identity, asks Ollama for the pinned model identity, -and validates every returned count against the constrained eligible catalog and 200-candidate ceiling. The -report retains the separate full eligible catalog count and the bounded result count, plus the lexical recipe, -`k=60`, and per-branch budget of 100. Candidate counts are not interpreted as proof -that either index is complete; the guarded setup fixture checks each stored index row count directly. +Hybrid evaluation verifies the same exact eligible catalog identity, asks Ollama for the pinned model identity, and validates every returned count against the constrained eligible catalog and 200-candidate ceiling. The report retains the separate full eligible catalog count and the bounded result count, plus the lexical recipe, `k=60`, and per-branch budget of 100. Candidate counts are not interpreted as proof that either index is complete; the guarded setup fixture checks each stored index row count directly. -For a new catalog or ranking revision, review all tuning rankings and settle the policy before running the -held-out split once. Do not adjust `k`, candidate budget, labels, document content, or any other ranking choice -after that final observation: +For a new catalog or ranking revision, review all tuning rankings and settle the policy before running the held-out split once. Do not adjust `k`, candidate budget, labels, document content, or any other ranking choice after that final observation: ```powershell .\tools\product-search-relevance.ps1 -BaseAddress http://localhost:5217/ -Approach hybrid -Split held-out ` -OutputPath artifacts\product-search-hybrid-held-out.json ``` -This was the final pre-rename v2 held-out observation, not a comparison to the historical v1 report. It was -run before SKU prefix removal; do not relabel it as a current-catalog measurement or change the policy based on its output. The evaluator's -held-out command is available for Semantic or Hybrid; FullText remains tuning-only. The checked-in v1 semantic -baseline is not replaced, and reports remain in ignored `artifacts/`. -The separate monthly/manual workflow remains semantic-only and has not been trialed remotely from this branch. +This was the final pre-rename v2 held-out observation, not a comparison to the historical v1 report. It was run before SKU prefix removal; do not relabel it as a current-catalog measurement or change the policy based on its output. The evaluator's held-out command is available for Semantic or Hybrid; FullText remains tuning-only. The checked-in v1 semantic baseline is not replaced, and reports remain in ignored `artifacts/`. The separate monthly/manual workflow remains semantic-only and has not been trialed remotely from this branch. -Historical v1 comparison, measured locally on **2026-10-15** against the then-reviewed catalog, seed, -judgments and pinned Nomic model. It is not a v2 result: +Historical v1 comparison, measured locally on **2026-10-15** against the then-reviewed catalog, seed, judgments and pinned Nomic model. It is not a v2 result: | Tuning metric | Semantic baseline | Independent FullText | Hybrid (100 per branch, `k=60`) | |---|---:|---:|---:| @@ -887,13 +788,9 @@ judgments and pinned Nomic model. It is not a v2 result: | Exact-SKU top-1 | 2/3 | 3/3 | **3/3** | | Price-constrained result rows / violations | 15 / 0 | 0 / 0 | **15 / 0** | -In that v1 tuning run, Hybrid exceeded its semantic baseline by 0.0588 nDCG@5 and 0.0118 Precision@5, while preserving -all 15 returned price-constrained rows without a violation and fixing the missed exact SKU. FullText's zero -constrained rows are not useful filter-retrieval evidence; its violation rate is undefined. These observed -tuning results support retaining the fixed Hybrid policy as an opt-in, not making it the default. +In that v1 tuning run, Hybrid exceeded its semantic baseline by 0.0588 nDCG@5 and 0.0118 Precision@5, while preserving all 15 returned price-constrained rows without a violation and fixing the missed exact SKU. FullText's zero constrained rows are not useful filter-retrieval evidence; its violation rate is undefined. These observed tuning results support retaining the fixed Hybrid policy as an opt-in, not making it the default. -In the historical v1 run, the fixed policy was compared with its semantic baseline on held-out exactly once, -without changing the policy or judgments: +In the historical v1 run, the fixed policy was compared with its semantic baseline on held-out exactly once, without changing the policy or judgments: | Held-out metric | Semantic baseline | Hybrid | |---|---:|---:| @@ -902,11 +799,7 @@ without changing the policy or judgments: | Exact-SKU top-1 | 0/1 | **1/1** | | Price-constrained result rows / violations | 5 / 0 | **5 / 0** | -That v1 held-out set contained only three relevant queries; its quality figures are descriptive and highly uncertain. -It also contains three no-relevant/out-of-catalog queries, for which both approaches still return candidates. -Those measured v1 results do not justify changing the semantic default or a general quality claim. The complete -historical per-query reports are `artifacts/product-search-hybrid-tuning.json` and -`artifacts/product-search-hybrid-held-out.json`; the checked-in semantic baseline remains unchanged. +That v1 held-out set contained only three relevant queries; its quality figures are descriptive and highly uncertain. It also contains three no-relevant/out-of-catalog queries, for which both approaches still return candidates. Those measured v1 results do not justify changing the semantic default or a general quality claim. The complete historical per-query reports are `artifacts/product-search-hybrid-tuning.json` and `artifacts/product-search-hybrid-held-out.json`; the checked-in semantic baseline remains unchanged. Judgments are graded 0 (irrelevant), 1 (marginal), 2 (useful secondary result), or 3 (direct fit). The report uses gain `2^grade - 1` for nDCG@5; Precision@5 counts grade 2 or 3 and always divides by five. Exact-SKU top-1 accuracy is shown only for exact-SKU queries. Queries with no grade-above-zero product are listed separately and excluded from nDCG/precision means rather than counted as successful empty results. Price-filter violations are counted and reported; no similarity/confidence cutoff or quality pass threshold is invented. The checked-in query set is not itself a measured baseline: capture and review a real report before making claims about full-catalog relevance. diff --git a/samples/docs/testing.md b/samples/docs/testing.md index 1c085bf26..667990bbe 100644 --- a/samples/docs/testing.md +++ b/samples/docs/testing.md @@ -301,7 +301,4 @@ Each `*.Test.Api` and `*.Test.Relay` project has a `Resources/` folder containin See [aspire.md](aspire.md) for the full Aspire setup, `Contoso.Test.Aspire`, E2E Runner usage, scenario descriptions, load-simulation configuration, and the recommended first-run order. -The runner's [demo snapshot bootstrap](aspire.md#shared-demo-catalog-bootstrap) is distinct from those event-flow -scenarios: it directly composes the Products accessor and Shopping sync adapter before loading basket data, -then builds real-model search. Its explicit `DemoCatalogBootstrapTests` fixture uses only guarded isolated -PostgreSQL/SQL Server databases; it neither resets Cosmos nor replaces ordinary host-test catalogs. +The runner's [demo snapshot bootstrap](aspire.md#shared-demo-catalog-bootstrap) is distinct from those event-flow scenarios: it directly composes the Products accessor and Shopping sync adapter before loading basket data, then builds real-model search. Its explicit `DemoCatalogBootstrapTests` fixture uses only guarded isolated PostgreSQL/SQL Server databases; it neither resets Cosmos nor replaces ordinary host-test catalogs. diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs index 39da2d2d9..4951e233f 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs @@ -120,17 +120,35 @@ await _unitOfWork.TransactionAsync(async token => /// Builds embeddings outside transactions and atomically replaces this profile's projection. /// A short publish-time lock rejects concurrent catalog edits instead of installing stale documents. - public async Task RebuildAsync(CancellationToken cancellationToken = default) + public Task RebuildAsync(CancellationToken cancellationToken = default) => RebuildAsync(null, cancellationToken); + + /// Rebuilds with synchronous, ordered progress notifications; counts represent validated embeddings, not committed index rows. + public async Task RebuildAsync(Action? progress, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke(new("Rebuilding lexical index")); await _lexicalIndex.RebuildAsync(cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke(new("Rebuilding lexical index", IsComplete: true)); + progress?.Invoke(new("Loading search documents")); await LoadLabelsAsync(cancellationToken).ConfigureAwait(false); var products = await _ef.Products.Model.Query().AsNoTracking().OrderBy(p => p.Id).ToListAsync(cancellationToken).ConfigureAwait(false); var documents = products.Select(product => (Product: product, Document: Document(product))).ToArray(); + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke(new("Loading search documents", IsComplete: true)); var vectors = new List(documents.Length); + progress?.Invoke(new("Embedding products", 0, documents.Length)); foreach (var item in documents) + { vectors.Add(await EmbedDocumentAsync(item.Document.Text, cancellationToken).ConfigureAwait(false)); + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke(new("Embedding products", vectors.Count, documents.Length)); + } - return await _unitOfWork.TransactionAsync(async token => + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke(new("Embedding products", vectors.Count, documents.Length, IsComplete: true)); + progress?.Invoke(new("Publishing semantic index")); + var count = await _unitOfWork.TransactionAsync(async token => { await _ef.Database.Statement(_lockSource).NonQueryAsync(token).ConfigureAwait(false); var current = await _ef.Products.Model.Query().AsNoTracking().OrderBy(p => p.Id).ToListAsync(token).ConfigureAwait(false); @@ -156,8 +174,14 @@ await _ef.Database.Statement(_markReady) return documents.Length; }, cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke(new("Publishing semantic index", IsComplete: true)); + return count; } + /// Describes actual rebuild work; completion applies only to the named stage. + public sealed record RebuildProgress(string Stage, int? Completed = null, int? Total = null, bool IsComplete = false); + /// Reads the stored fingerprint, or null when this product has no compatible projection. private Task ReadSourceHashAsync(string id, CancellationToken cancellationToken) => _ef.Database.Statement(_sourceHash) diff --git a/samples/tests/Contoso.E2E.Runner/GlobalUsing.cs b/samples/tests/Contoso.E2E.Runner/GlobalUsing.cs index af0082c3a..539dfdee2 100644 --- a/samples/tests/Contoso.E2E.Runner/GlobalUsing.cs +++ b/samples/tests/Contoso.E2E.Runner/GlobalUsing.cs @@ -21,6 +21,7 @@ global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using Spectre.Console; global using StackExchange.Redis; diff --git a/samples/tests/Contoso.E2E.Runner/Infrastructure/ChoiceManager.cs b/samples/tests/Contoso.E2E.Runner/Infrastructure/ChoiceManager.cs index c775b8614..cf299c188 100644 --- a/samples/tests/Contoso.E2E.Runner/Infrastructure/ChoiceManager.cs +++ b/samples/tests/Contoso.E2E.Runner/Infrastructure/ChoiceManager.cs @@ -25,6 +25,7 @@ public void CacheChoices() _choicesCache.Add($"{i++}. {scenario.Value.Text}{(scenario.Value.Attribute.RequiresApi ? _requiresApi : string.Empty)}", (scenario.Value, 1, () => RunScenarioAsync(scenario.Value), ChoiceResult.ContinueWithPrompt)); _choicesCache.Add($"{i++}. Run all scenarios as simulation{_requiresApi}", (null, 1, () => new LoadSimulationRunner(_context).RunAsync(), ChoiceResult.Continue)); + _choicesCache.Add($"{i++}. Product Search [grey](requires Products API)[/]", (null, 2, () => new ProductSearchOperation(_context.ProductsHttpClient).RunAsync(), ChoiceResult.Continue)); _choicesCache.Add($"{i++}. Retry APIs", (null, -2, () => Task.CompletedTask, ChoiceResult.RetryApi)); _choicesCache.Add($"{i++}. Exit", (null, -1, () => Task.CompletedTask, ChoiceResult.Stop)); } @@ -39,6 +40,7 @@ public SelectionPrompt AddRunnerChoices(SelectionPrompt prompt) prompt.AddChoiceGroup("Set-up:", _choicesCache.Where(c => c.Value.Group == 0).Select(c => c.Key)); prompt.AddChoiceGroup("Scenarios:", _choicesCache.Where(c => c.Value.Group == 1).Select(c => c.Key)); + prompt.AddChoiceGroup("Interactive operations:", _choicesCache.Where(c => c.Value.Group == 2).Select(c => c.Key)); prompt.AddChoiceGroup("Other:", _choicesCache.Where(c => c.Value.Group < 0).Select(c => c.Key)); return prompt; } @@ -57,4 +59,4 @@ public async Task RunChoiceAsync(string choice, bool apisHealthy) await item.RunAsync(); return item.Result; } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogBootstrap.cs b/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogBootstrap.cs index ba62042c7..4e6b1fb3a 100644 --- a/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogBootstrap.cs +++ b/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogBootstrap.cs @@ -5,7 +5,10 @@ namespace Contoso.E2E.Runner.Infrastructure; public static class DemoCatalogBootstrap { /// Resets only Products and Shopping, seeds replicas before baskets, and verifies search readiness. - public static async Task RunAsync(IConfiguration configuration, CancellationToken cancellationToken = default) + public static Task RunAsync(IConfiguration configuration, CancellationToken cancellationToken = default) => RunAsync(configuration, null, cancellationToken); + + /// Runs setup with ordered synchronous notifications, stopping immediately on failure or cancellation. + public static async Task RunAsync(IConfiguration configuration, Action? progress, CancellationToken cancellationToken = default) { var productsConnection = RequiredConnection(configuration, "Products"); var shoppingConnection = RequiredConnection(configuration, "Shopping"); @@ -16,31 +19,38 @@ public static async Task RunAsync(IConfiguration configuration, Cancellatio productsArgs.DataParserArgs.AddNamed("ref-data.seed.yaml", "demo-catalog.seed.yaml"); using (var migration = new PostgresMigration(productsArgs)) - await MigrateAsync(migration, cancellationToken).ConfigureAwait(false); + await StageAsync("Migrating/resetting Products and loading catalog", () => MigrateAsync(migration, cancellationToken), progress, cancellationToken).ConfigureAwait(false); var shoppingArgs = Shopping.Database.Program.ConfigureMigrationArgs(new MigrationArgs(MigrationCommand.ResetAndDatabase, shoppingConnection)); using (var migration = new SqlServerMigration(shoppingArgs)) - await MigrateAsync(migration, cancellationToken).ConfigureAwait(false); + await StageAsync("Migrating/resetting Shopping", () => MigrateAsync(migration, cancellationToken), progress, cancellationToken).ConfigureAwait(false); using var products = CreateProductsHost(configuration, productsConnection); using var shopping = CreateShoppingHost(configuration, shoppingConnection); - var count = await SynchronizeAsync(products, shopping, cancellationToken).ConfigureAwait(false); + var count = 0; + await StageAsync("Synchronizing current products to Shopping", async () => + { + count = await SynchronizeAsync(products, shopping, cancellationToken).ConfigureAwait(false); + }, progress, cancellationToken).ConfigureAwait(false); var basketArgs = Shopping.Database.Program.ConfigureMigrationArgs(new MigrationArgs(MigrationCommand.Data, shoppingConnection)); basketArgs.AddAssembly().DataParserArgs.AddNamed("demo-baskets.seed.yaml"); using (var migration = new SqlServerMigration(basketArgs)) - await MigrateAsync(migration, cancellationToken).ConfigureAwait(false); + await StageAsync("Loading Shopping baskets", () => MigrateAsync(migration, cancellationToken), progress, cancellationToken).ConfigureAwait(false); - using (var cache = CreateServingCacheHost(configuration, "Products")) - await InvalidateReferenceDataAsync(cache, [typeof(BrandCollection), typeof(CategoryCollection), typeof(SubCategoryCollection), - typeof(Products.Contracts.UnitOfMeasureCollection), typeof(MovementKindCollection), typeof(MovementStatusCollection)], cancellationToken).ConfigureAwait(false); + await StageAsync("Invalidating Products and Shopping reference-data caches", async () => + { + using (var cache = CreateServingCacheHost(configuration, "Products")) + await InvalidateReferenceDataAsync(cache, [typeof(BrandCollection), typeof(CategoryCollection), typeof(SubCategoryCollection), + typeof(Products.Contracts.UnitOfMeasureCollection), typeof(MovementKindCollection), typeof(MovementStatusCollection)], cancellationToken).ConfigureAwait(false); - using (var cache = CreateServingCacheHost(configuration, "Shopping")) - await InvalidateReferenceDataAsync(cache, [typeof(Shopping.Contracts.UnitOfMeasureCollection), - typeof(Shopping.Contracts.BasketStatusCollection), typeof(Shopping.Contracts.DiscountCouponCollection)], cancellationToken).ConfigureAwait(false); + using (var cache = CreateServingCacheHost(configuration, "Shopping")) + await InvalidateReferenceDataAsync(cache, [typeof(Shopping.Contracts.UnitOfMeasureCollection), + typeof(Shopping.Contracts.BasketStatusCollection), typeof(Shopping.Contracts.DiscountCouponCollection)], cancellationToken).ConfigureAwait(false); + }, progress, cancellationToken).ConfigureAwait(false); - await VerifyAsync(products, shopping, cancellationToken).ConfigureAwait(false); - await PrepareSearchAsync(products, cancellationToken).ConfigureAwait(false); + await PrepareSearchAsync(products, progress, cancellationToken).ConfigureAwait(false); + await StageAsync("Verifying Products, Shopping and basket integrity", () => VerifyAsync(products, shopping, cancellationToken), progress, cancellationToken).ConfigureAwait(false); return count; } @@ -50,6 +60,7 @@ public static IHost CreateProductsHost(IConfiguration configuration, string conn var builder = Host.CreateApplicationBuilder(); builder.Configuration.Sources.Clear(); builder.Configuration.AddConfiguration(configuration); + ConfigureLogging(builder); builder.AddProductEmbeddings(); builder.Services.AddSingleton(_ => Npgsql.NpgsqlDataSource.Create(connectionString)); @@ -70,6 +81,7 @@ public static IHost CreateShoppingHost(IConfiguration configuration, string conn var builder = Host.CreateApplicationBuilder(); builder.Configuration.Sources.Clear(); builder.Configuration.AddConfiguration(configuration); + ConfigureLogging(builder); builder.Services.AddExecutionContext().AddMemoryCache().AddMemoryOnlyHybridCache().AddDefaultCacheKeyProvider() .AddReferenceDataOrchestrator() @@ -88,6 +100,7 @@ public static IHost CreateServingCacheHost(IConfiguration configuration, string var builder = Host.CreateApplicationBuilder(); builder.Configuration.Sources.Clear(); builder.Configuration.AddConfiguration(configuration); + ConfigureLogging(builder); builder.AddHostSettings(solutionName: "Contoso", domainName: domainName); builder.AddRedisDistributedCache("redis"); builder.Services.AddExecutionContext().AddMemoryCache().AddDefaultCacheKeyProvider().AddFusionHybridCache(); @@ -168,20 +181,27 @@ await InScopeAsync(shopping, async services => /// Builds lexical and semantic projections with the configured real provider; failures are not swallowed or replaced by test vectors. public static Task PrepareSearchAsync(IHost products, CancellationToken cancellationToken = default) + => PrepareSearchAsync(products, null, cancellationToken); + + /// Builds and verifies projections while reporting actual rebuild work. + public static Task PrepareSearchAsync(IHost products, Action? progress, CancellationToken cancellationToken = default) => InScopeAsync(products, async services => { var profile = services.GetRequiredService().Profile; try { var semantic = services.GetRequiredService(); - var count = await semantic.RebuildAsync(cancellationToken).ConfigureAwait(false); - await semantic.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); - await services.GetRequiredService().EnsureReadyAsync(cancellationToken).ConfigureAwait(false); - await semantic.EmbedQueryAsync("mountain bike", cancellationToken).ConfigureAwait(false); - - var expected = await services.GetRequiredService().Products.Model.Query().CountAsync(cancellationToken).ConfigureAwait(false); - if (count != expected) - throw new InvalidOperationException($"Demo search projection is incomplete: expected {expected}, indexed {count}."); + var count = await semantic.RebuildAsync(progress, cancellationToken).ConfigureAwait(false); + await StageAsync("Verifying lexical/semantic search and query model", async () => + { + await semantic.EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + await services.GetRequiredService().EnsureReadyAsync(cancellationToken).ConfigureAwait(false); + await semantic.EmbedQueryAsync("mountain bike", cancellationToken).ConfigureAwait(false); + + var expected = await services.GetRequiredService().Products.Model.Query().CountAsync(cancellationToken).ConfigureAwait(false); + if (count != expected) + throw new InvalidOperationException($"Demo search projection is incomplete: expected {expected}, indexed {count}."); + }, progress, cancellationToken).ConfigureAwait(false); return count; } @@ -191,6 +211,24 @@ public static Task PrepareSearchAsync(IHost products, CancellationToken can } }); + /// Suppresses routine success chatter only in bootstrap-created hosts; warnings and errors remain visible. + private static void ConfigureLogging(HostApplicationBuilder builder) + { + builder.Logging.AddFilter("CoreEx.RefData", LogLevel.Warning); + builder.Logging.AddFilter("Polly", LogLevel.Warning); + builder.Logging.AddFilter("System.Net.Http.HttpClient", LogLevel.Warning); + } + + /// Reports completion only after the stage succeeds and cancellation has been checked. + private static async Task StageAsync(string stage, Func action, Action? progress, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke(new(stage)); + await action().ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke(new(stage, IsComplete: true)); + } + private static Task> ReadSnapshotAsync(IHost products, CancellationToken cancellationToken) => InScopeAsync(products, async services => { diff --git a/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogOperation.cs b/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogOperation.cs new file mode 100644 index 000000000..5f95aeec5 --- /dev/null +++ b/samples/tests/Contoso.E2E.Runner/Infrastructure/DemoCatalogOperation.cs @@ -0,0 +1,68 @@ +namespace Contoso.E2E.Runner.Infrastructure; + +/// Shows durable setup stages and real work counts without a live display that could overwrite warnings. +public sealed class DemoCatalogOperation(IAnsiConsole? console = null) +{ + private readonly IAnsiConsole _console = console ?? AnsiConsole.Console; + private readonly object _gate = new(); + private readonly Stopwatch _stageTime = new(); + private string? _stage; + + /// Runs the acknowledged bootstrap, preserving failures and cancelling on Ctrl+C. + public async Task RunAsync(IConfiguration configuration, CancellationToken cancellationToken = default) + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ConsoleCancelEventHandler cancel = (_, e) => + { + e.Cancel = true; + cancellation.Cancel(); + }; + Console.CancelKeyPress += cancel; + var elapsed = Stopwatch.StartNew(); + try + { + _console.MarkupLine("[grey]Preparing demo catalog: reset Products/Shopping, synchronize products/baskets, invalidate caches, rebuild search, then verify. Ctrl+C cancels.[/]"); + var count = await DemoCatalogBootstrap.RunAsync(configuration, Report, cancellation.Token).ConfigureAwait(false); + cancellation.Token.ThrowIfCancellationRequested(); + _console.MarkupLine($"[green]Demo bootstrap complete: {count} products replicated; baskets and lexical/semantic search verified in {elapsed.Elapsed.TotalSeconds:F1}s.[/]"); + return count; + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + _console.MarkupLine("[yellow]Demo bootstrap cancelled. Setup may be incomplete; rerun before starting hosts.[/]"); + throw; + } + catch (Exception ex) + { + _console.MarkupLine($"[red]Demo bootstrap failed during {(_stage ?? "configuration").EscapeMarkup()} after {elapsed.Elapsed.TotalSeconds:F1}s: {ex.Message.EscapeMarkup()}[/]"); + throw; + } + finally + { + Console.CancelKeyPress -= cancel; + } + } + + /// Synchronously renders ordered stage events; redirected output uses the same truthful, bounded lines. + public void Report(ProductSearchIndex.RebuildProgress progress) + { + lock (_gate) + { + if (_stage != progress.Stage) + { + _stage = progress.Stage; + _stageTime.Restart(); + _console.MarkupLine($"[grey]▶[/] {progress.Stage.EscapeMarkup()}..."); + } + + if (progress.IsComplete) + _console.MarkupLine($"[green] ✓[/] {progress.Stage.EscapeMarkup()}{Counts(progress)} [grey]({_stageTime.Elapsed.TotalSeconds:F1}s)[/]"); + else if (progress.Completed is { } completed && (completed <= 1 || completed % 10 == 0 || completed == progress.Total)) + _console.MarkupLine($"[grey] {progress.Stage.EscapeMarkup()}{Counts(progress)}[/]"); + } + } + + /// Formats only measured work, never estimated percentages. + private static string Counts(ProductSearchIndex.RebuildProgress progress) + => progress.Completed is { } completed && progress.Total is { } total ? $" {completed}/{total}" : string.Empty; +} diff --git a/samples/tests/Contoso.E2E.Runner/Infrastructure/ProductSearchOperation.cs b/samples/tests/Contoso.E2E.Runner/Infrastructure/ProductSearchOperation.cs new file mode 100644 index 000000000..836589134 --- /dev/null +++ b/samples/tests/Contoso.E2E.Runner/Infrastructure/ProductSearchOperation.cs @@ -0,0 +1,136 @@ +namespace Contoso.E2E.Runner.Infrastructure; + +/// Interactively searches the live Products API without running scenario setup or load simulation. +public sealed class ProductSearchOperation(HttpClient client, IAnsiConsole? console = null) +{ + private readonly HttpClient _client = client.ThrowIfNull(); + private readonly IAnsiConsole _console = console ?? AnsiConsole.Console; + private string? _mode; + + /// Prompts for successive searches until the user returns to the menu or cancels. + public async Task RunAsync(CancellationToken cancellationToken = default) + { + _mode = null; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ConsoleCancelEventHandler cancel = (_, e) => + { + e.Cancel = true; + cancellation.Cancel(); + }; + + Console.CancelKeyPress += cancel; + try + { + _console.Write(new Rule("[bold blue]Product Search[/]").RuleStyle("blue").LeftJustified()); + _console.MarkupLine("[grey]Search again after each result. Blank input or /back returns to the menu; Ctrl+C cancels.[/]"); + while (true) + { + cancellation.Token.ThrowIfCancellationRequested(); + var text = await _console.PromptAsync(new TextPrompt("Search text:").AllowEmpty(), cancellation.Token).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(text) || text.Trim().Equals("/back", StringComparison.OrdinalIgnoreCase)) + return; + + try + { + await SearchAsync(text, cancellation.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + var detail = ex is CoreEx.Http.ProblemDetailsException problem ? problem.ProblemDetails.Detail : null; + var message = ex is OperationCanceledException ? "Products API request timed out." : ex.Message; + _console.MarkupLine($"[red]Product Search failed: {message.EscapeMarkup()}[/]"); + + if (!string.IsNullOrWhiteSpace(detail)) + _console.MarkupLine($"[red]{detail.EscapeMarkup()}[/]"); + + _console.MarkupLine("[grey]Check the Products API and its configured search/index/model readiness. No fallback or rebuild was attempted.[/]"); + } + } + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + _console.MarkupLine("[yellow]Product Search cancelled; returning to the menu.[/]"); + } + finally + { + _mode = null; + Console.CancelKeyPress -= cancel; + } + } + + /// Reads the configured server mode, then requests and displays the first ten results in server rank order. + private async Task SearchAsync(string text, CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + (string Mode, ProductLite[] Products) result; + if (_console.Profile.Capabilities.Interactive && _console.Profile.Capabilities.Ansi) + { + result = await _console.Status().Spinner(Spinner.Known.Dots).StartAsync(_mode is null ? "Reading Products search mode..." : $"Searching Products ({_mode})...", + status => RequestAsync(text, message => status.Status(message), cancellationToken)).ConfigureAwait(false); + } + else + result = await RequestAsync(text, message => _console.MarkupLine($"[grey]{message.EscapeMarkup()}[/]"), cancellationToken).ConfigureAwait(false); + + stopwatch.Stop(); + var products = result.Products; + _console.MarkupLine($"[grey]{result.Mode} · Returned {products.Length} {(products.Length == 1 ? "product" : "products")} in {stopwatch.Elapsed.TotalSeconds:F2}s.[/]"); + if (products.Length == 0) + { + _console.MarkupLine("[yellow]No products found.[/]"); + return; + } + + var table = new Table().Border(TableBorder.Rounded).BorderColor(Color.Grey) + .AddColumn(new TableColumn("Rank").RightAligned()) + .AddColumn("SKU") + .AddColumn("Product (Text)") + .AddColumn(new TableColumn("Price (USD)").RightAligned()); + + for (var i = 0; i < products.Length; i++) + { + var product = products[i]; + table.AddRow((i + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), (product.Sku ?? string.Empty).EscapeMarkup(), + (product.Text ?? string.Empty).EscapeMarkup(), product.Price.ToString("N2", System.Globalization.CultureInfo.GetCultureInfo("en-US"))); + } + + _console.Write(table); + } + + /// Discovers the server mode and loads results without overriding ranking or rebuilding anything. + private async Task<(string Mode, ProductLite[] Products)> RequestAsync(string text, Action status, CancellationToken cancellationToken) + { + if (_mode is null) + _mode = await ReadModeAsync(status, cancellationToken).ConfigureAwait(false); + + status($"Searching Products ({_mode})..."); + + using var response = await _client.GetAsync($"/api/products?$search={Uri.EscapeDataString(text)}&$take=10", cancellationToken).ConfigureAwait(false); + var products = await response.GetValueAsync(cancellationToken).ConfigureAwait(false); + return (_mode, products); + } + + private async Task ReadModeAsync(Action status, CancellationToken cancellationToken) + { + status("Reading Products search mode..."); + using var schemaResponse = await _client.GetAsync("/api/products/$query", cancellationToken).ConfigureAwait(false); + var schema = await schemaResponse.GetValueAsync(cancellationToken).ConfigureAwait(false); + var mode = schema.TryGetProperty("search", out var search) && search.TryGetProperty("mode", out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString()?.ToLowerInvariant() switch + { + "semantic" => "Semantic", + "fulltext" => "FullText", + "hybrid" => "Hybrid", + _ => null + } + : null; + + if (mode is null) + throw new InvalidOperationException("Products query schema does not advertise a supported Semantic, FullText or Hybrid search mode."); + + return mode; + } +} diff --git a/samples/tests/Contoso.E2E.Runner/Program.cs b/samples/tests/Contoso.E2E.Runner/Program.cs index c23e1db00..a30b5fe73 100644 --- a/samples/tests/Contoso.E2E.Runner/Program.cs +++ b/samples/tests/Contoso.E2E.Runner/Program.cs @@ -13,8 +13,14 @@ { if (!args.Contains("--accept-demo-reset", StringComparer.Ordinal)) throw new InvalidOperationException("Demo setup resets the configured Products and Shopping schemas. Stop their writers and explicitly pass --accept-demo-reset after reviewing E2E connection strings."); - var count = await DemoCatalogBootstrap.RunAsync(config).ConfigureAwait(false); - Console.WriteLine($"Demo bootstrap complete: {count} products replicated; basket integrity and lexical/semantic search readiness verified."); + try + { + await new DemoCatalogOperation().RunAsync(config).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + Environment.ExitCode = 130; + } return; } diff --git a/samples/tests/Contoso.E2E.Runner/Scenarios/DatabaseMigrationSetup.cs b/samples/tests/Contoso.E2E.Runner/Scenarios/DatabaseMigrationSetup.cs index a9c0e5fd6..196bdd9bc 100644 --- a/samples/tests/Contoso.E2E.Runner/Scenarios/DatabaseMigrationSetup.cs +++ b/samples/tests/Contoso.E2E.Runner/Scenarios/DatabaseMigrationSetup.cs @@ -9,7 +9,7 @@ public sealed class DatabaseMigrationSetup : IScenario /// public async Task RunAsync(ScenarioContext context) { - await context.StepAsync("Products and Shopping demo catalog bootstrap.", () => DemoCatalogBootstrap.RunAsync(context.TestContext.Config), + await context.StepAsync("Products and Shopping demo catalog bootstrap.", () => new DemoCatalogOperation().RunAsync(context.TestContext.Config), count => $"{count} non-deleted products replicated; baskets and search projections verified.").ConfigureAwait(false); // Step 3: Customers database migration. diff --git a/src/CoreEx.DomainDriven/CoreEx.DomainDriven.csproj b/src/CoreEx.DomainDriven/CoreEx.DomainDriven.csproj index de4cb7b2d..7f86ae05f 100644 --- a/src/CoreEx.DomainDriven/CoreEx.DomainDriven.csproj +++ b/src/CoreEx.DomainDriven/CoreEx.DomainDriven.csproj @@ -6,7 +6,6 @@ - net8.0;net9.0;net10.0 true diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 84c46282f..95f7f5e38 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -1,8 +1,10 @@ + + - net8.0;net9.0;net10.0 + $(CoreExTargetFrameworks) enable enable preview @@ -81,4 +83,4 @@ $(AzdPublishTargetFramework) - \ No newline at end of file + diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets deleted file mode 100644 index 144371187..000000000 --- a/src/Directory.Build.targets +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/tests/CoreEx.AspNetCore.Test.Api/CoreEx.AspNetCore.Test.Api.csproj b/tests/CoreEx.AspNetCore.Test.Api/CoreEx.AspNetCore.Test.Api.csproj index c0b05beeb..9f907fe3e 100644 --- a/tests/CoreEx.AspNetCore.Test.Api/CoreEx.AspNetCore.Test.Api.csproj +++ b/tests/CoreEx.AspNetCore.Test.Api/CoreEx.AspNetCore.Test.Api.csproj @@ -1,7 +1,6 @@ - net8.0;net9.0;net10.0 enable enable false diff --git a/tests/CoreEx.AspNetCore.Test.Unit/CoreEx.AspNetCore.Test.Unit.csproj b/tests/CoreEx.AspNetCore.Test.Unit/CoreEx.AspNetCore.Test.Unit.csproj index 455716d24..59f4230cd 100644 --- a/tests/CoreEx.AspNetCore.Test.Unit/CoreEx.AspNetCore.Test.Unit.csproj +++ b/tests/CoreEx.AspNetCore.Test.Unit/CoreEx.AspNetCore.Test.Unit.csproj @@ -1,7 +1,6 @@ - net8.0;net9.0;net10.0 enable enable preview @@ -53,4 +52,4 @@ - \ No newline at end of file + diff --git a/tests/CoreEx.Azure.Messaging.ServiceBus.Test.Unit/CoreEx.Azure.Messaging.ServiceBus.Test.Unit.csproj b/tests/CoreEx.Azure.Messaging.ServiceBus.Test.Unit/CoreEx.Azure.Messaging.ServiceBus.Test.Unit.csproj index 7a81b8143..579258a86 100644 --- a/tests/CoreEx.Azure.Messaging.ServiceBus.Test.Unit/CoreEx.Azure.Messaging.ServiceBus.Test.Unit.csproj +++ b/tests/CoreEx.Azure.Messaging.ServiceBus.Test.Unit/CoreEx.Azure.Messaging.ServiceBus.Test.Unit.csproj @@ -1,6 +1,5 @@ - net8.0;net9.0;net10.0 enable enable preview @@ -37,4 +36,4 @@ - \ No newline at end of file + diff --git a/tests/CoreEx.Caching.Redis.Test.Unit/CoreEx.Caching.Redis.Test.Unit.csproj b/tests/CoreEx.Caching.Redis.Test.Unit/CoreEx.Caching.Redis.Test.Unit.csproj index f89e40f89..c2feca6e5 100644 --- a/tests/CoreEx.Caching.Redis.Test.Unit/CoreEx.Caching.Redis.Test.Unit.csproj +++ b/tests/CoreEx.Caching.Redis.Test.Unit/CoreEx.Caching.Redis.Test.Unit.csproj @@ -1,7 +1,6 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.Cosmos.Test.Unit/CoreEx.Cosmos.Test.Unit.csproj b/tests/CoreEx.Cosmos.Test.Unit/CoreEx.Cosmos.Test.Unit.csproj index 73768d3ae..71d929b2d 100644 --- a/tests/CoreEx.Cosmos.Test.Unit/CoreEx.Cosmos.Test.Unit.csproj +++ b/tests/CoreEx.Cosmos.Test.Unit/CoreEx.Cosmos.Test.Unit.csproj @@ -1,6 +1,5 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.Data.GraphQL.Test.Unit/CoreEx.Data.GraphQL.Test.Unit.csproj b/tests/CoreEx.Data.GraphQL.Test.Unit/CoreEx.Data.GraphQL.Test.Unit.csproj index b5532eeed..05dd53b89 100644 --- a/tests/CoreEx.Data.GraphQL.Test.Unit/CoreEx.Data.GraphQL.Test.Unit.csproj +++ b/tests/CoreEx.Data.GraphQL.Test.Unit/CoreEx.Data.GraphQL.Test.Unit.csproj @@ -1,6 +1,5 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.Data.Test.Unit/CoreEx.Data.Test.Unit.csproj b/tests/CoreEx.Data.Test.Unit/CoreEx.Data.Test.Unit.csproj index ed582a149..b9542fbb3 100644 --- a/tests/CoreEx.Data.Test.Unit/CoreEx.Data.Test.Unit.csproj +++ b/tests/CoreEx.Data.Test.Unit/CoreEx.Data.Test.Unit.csproj @@ -1,6 +1,5 @@ - net8.0;net9.0;net10.0 enable enable preview @@ -42,4 +41,4 @@ - \ No newline at end of file + diff --git a/tests/CoreEx.Database.Postgres.Test.Console/CoreEx.Database.Postgres.Test.Console.csproj b/tests/CoreEx.Database.Postgres.Test.Console/CoreEx.Database.Postgres.Test.Console.csproj index 9aeb57db1..ae04ad8a3 100644 --- a/tests/CoreEx.Database.Postgres.Test.Console/CoreEx.Database.Postgres.Test.Console.csproj +++ b/tests/CoreEx.Database.Postgres.Test.Console/CoreEx.Database.Postgres.Test.Console.csproj @@ -2,7 +2,6 @@ Exe - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.Database.Postgres.Test.Unit/CoreEx.Database.Postgres.Test.Unit.csproj b/tests/CoreEx.Database.Postgres.Test.Unit/CoreEx.Database.Postgres.Test.Unit.csproj index 5b3912a29..6dce3492a 100644 --- a/tests/CoreEx.Database.Postgres.Test.Unit/CoreEx.Database.Postgres.Test.Unit.csproj +++ b/tests/CoreEx.Database.Postgres.Test.Unit/CoreEx.Database.Postgres.Test.Unit.csproj @@ -1,6 +1,5 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.Database.SqlServer.Test.Console/CoreEx.Database.SqlServer.Test.Console.csproj b/tests/CoreEx.Database.SqlServer.Test.Console/CoreEx.Database.SqlServer.Test.Console.csproj index ad636e81a..7f7e48eca 100644 --- a/tests/CoreEx.Database.SqlServer.Test.Console/CoreEx.Database.SqlServer.Test.Console.csproj +++ b/tests/CoreEx.Database.SqlServer.Test.Console/CoreEx.Database.SqlServer.Test.Console.csproj @@ -2,7 +2,6 @@ Exe - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.Database.SqlServer.Test.Unit/CoreEx.Database.SqlServer.Test.Unit.csproj b/tests/CoreEx.Database.SqlServer.Test.Unit/CoreEx.Database.SqlServer.Test.Unit.csproj index 0bb8441e9..ba8a27914 100644 --- a/tests/CoreEx.Database.SqlServer.Test.Unit/CoreEx.Database.SqlServer.Test.Unit.csproj +++ b/tests/CoreEx.Database.SqlServer.Test.Unit/CoreEx.Database.SqlServer.Test.Unit.csproj @@ -1,6 +1,5 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.Database.Test.Unit/CoreEx.Database.Test.Unit.csproj b/tests/CoreEx.Database.Test.Unit/CoreEx.Database.Test.Unit.csproj index 9d6011e34..405c5e254 100644 --- a/tests/CoreEx.Database.Test.Unit/CoreEx.Database.Test.Unit.csproj +++ b/tests/CoreEx.Database.Test.Unit/CoreEx.Database.Test.Unit.csproj @@ -1,6 +1,5 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.DomainDriven.Test.Unit/CoreEx.DomainDriven.Test.Unit.csproj b/tests/CoreEx.DomainDriven.Test.Unit/CoreEx.DomainDriven.Test.Unit.csproj index f919166b4..01803c292 100644 --- a/tests/CoreEx.DomainDriven.Test.Unit/CoreEx.DomainDriven.Test.Unit.csproj +++ b/tests/CoreEx.DomainDriven.Test.Unit/CoreEx.DomainDriven.Test.Unit.csproj @@ -1,7 +1,6 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.Events.Test.Unit/CoreEx.Events.Test.Unit.csproj b/tests/CoreEx.Events.Test.Unit/CoreEx.Events.Test.Unit.csproj index 0d167a040..28a581ea8 100644 --- a/tests/CoreEx.Events.Test.Unit/CoreEx.Events.Test.Unit.csproj +++ b/tests/CoreEx.Events.Test.Unit/CoreEx.Events.Test.Unit.csproj @@ -1,7 +1,6 @@ - net8.0;net9.0;net10.0 enable enable preview @@ -32,4 +31,4 @@ - \ No newline at end of file + diff --git a/tests/CoreEx.RefData.Test.Unit/CoreEx.RefData.Test.Unit.csproj b/tests/CoreEx.RefData.Test.Unit/CoreEx.RefData.Test.Unit.csproj index b6309c55b..4ac3c8e60 100644 --- a/tests/CoreEx.RefData.Test.Unit/CoreEx.RefData.Test.Unit.csproj +++ b/tests/CoreEx.RefData.Test.Unit/CoreEx.RefData.Test.Unit.csproj @@ -1,7 +1,6 @@ - net8.0;net9.0;net10.0 enable enable preview @@ -33,4 +32,4 @@ - \ No newline at end of file + diff --git a/tests/CoreEx.Test.Unit/CoreEx.Test.Unit.csproj b/tests/CoreEx.Test.Unit/CoreEx.Test.Unit.csproj index 11ae0c736..f1e2591ef 100644 --- a/tests/CoreEx.Test.Unit/CoreEx.Test.Unit.csproj +++ b/tests/CoreEx.Test.Unit/CoreEx.Test.Unit.csproj @@ -1,7 +1,6 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.UnitTesting.Test.Unit/CoreEx.UnitTesting.Test.Unit.csproj b/tests/CoreEx.UnitTesting.Test.Unit/CoreEx.UnitTesting.Test.Unit.csproj index 897e05fcd..0bcd7c264 100644 --- a/tests/CoreEx.UnitTesting.Test.Unit/CoreEx.UnitTesting.Test.Unit.csproj +++ b/tests/CoreEx.UnitTesting.Test.Unit/CoreEx.UnitTesting.Test.Unit.csproj @@ -1,7 +1,6 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/CoreEx.Validation.Test.Unit/CoreEx.Validation.Test.Unit.csproj b/tests/CoreEx.Validation.Test.Unit/CoreEx.Validation.Test.Unit.csproj index 7a4327ef4..5a1ec3f47 100644 --- a/tests/CoreEx.Validation.Test.Unit/CoreEx.Validation.Test.Unit.csproj +++ b/tests/CoreEx.Validation.Test.Unit/CoreEx.Validation.Test.Unit.csproj @@ -1,7 +1,6 @@ - net8.0;net9.0;net10.0 enable enable preview diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 66a391da7..961d26186 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -1,5 +1,8 @@ + + + $(CoreExTargetFrameworks) IDE1006 diff --git a/tools/validate-template-pack.ps1 b/tools/validate-template-pack.ps1 index 395598b29..1270af144 100644 --- a/tools/validate-template-pack.ps1 +++ b/tools/validate-template-pack.ps1 @@ -857,7 +857,7 @@ function Invoke-Assertion { } try { - # Packages must contain every target framework; ignore any local inner-loop narrowing (see Directory.Build.targets). + # Packages must contain every target framework; ignore any local inner-loop narrowing (see Directory.Build.props). $env:CoreExLocalTargetFramework = $null # Step 1: Build and pack CoreEx.Template From 9f0c46d52634797366a0d3930e861951a6ca6546 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Mon, 19 Oct 2026 10:18:46 -0700 Subject: [PATCH 13/25] Add MCP product search filters Expose structured category, subcategory, brand and price filters on the MCP product search tool, add a live filter-options discovery tool, and cover the new contract with focused Aspire tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .../aspire/Contoso.Test.Aspire/GlobalUsing.cs | 1 + .../ProductSearchToolTests.cs | 170 ++++++++++++++++++ samples/src/Contoso.Mcp/GlobalUsing.cs | 2 + .../src/Contoso.Mcp/Tools/CommerceTools.cs | 41 ++++- .../Contoso.Mcp/Tools/ProductSearchOptions.cs | 15 ++ 5 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 samples/aspire/Contoso.Test.Aspire/ProductSearchToolTests.cs create mode 100644 samples/src/Contoso.Mcp/Tools/ProductSearchOptions.cs diff --git a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs index 207d123f5..5dbfa2edc 100644 --- a/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs +++ b/samples/aspire/Contoso.Test.Aspire/GlobalUsing.cs @@ -5,6 +5,7 @@ global using Azure.Messaging.ServiceBus; global using Azure.Messaging.ServiceBus.Administration; global using Contoso.E2E.Runner.Infrastructure; +global using Contoso.Mcp.Tools; global using Contoso.Products.Application.Adapters.Embeddings; global using Contoso.Products.Contracts; global using Contoso.Products.Infrastructure.Clients.Ollama; diff --git a/samples/aspire/Contoso.Test.Aspire/ProductSearchToolTests.cs b/samples/aspire/Contoso.Test.Aspire/ProductSearchToolTests.cs new file mode 100644 index 000000000..607cae984 --- /dev/null +++ b/samples/aspire/Contoso.Test.Aspire/ProductSearchToolTests.cs @@ -0,0 +1,170 @@ +namespace Contoso.Test.Aspire; + +[TestFixture] +public class ProductSearchToolTests +{ + [Test] + public async Task Search_WithoutFilters_PreservesSearchPagingAndRankOrder() + { + using var handler = new ProductsHandler("""[{"sku":"Z-FIRST","price":123.45},{"sku":"A-SECOND","price":20}]"""); + using var client = new HttpClient(handler) { BaseAddress = new Uri("http://products/") }; + var factory = new ProductsFactory(client); + + var result = await CommerceTools.SearchProductsAsync(factory, "mud & oil + #?/$='", take: 2); + + result.Select(x => x.Sku).Should().Equal("Z-FIRST", "A-SECOND"); + var query = Query(handler); + query["$search"].ToString().Should().Be("mud & oil + #?/$='"); + query["$take"].ToString().Should().Be("2"); + query.Should().NotContainKey("$filter").And.NotContainKey("$orderby").And.NotContainKey("$inactive"); + } + + [Test] + [SetCulture("fr-FR")] + public async Task Search_AllFilters_AreConjoinedAndPricesAreCultureInvariant() + { + using var handler = new ProductsHandler("[]"); + using var client = new HttpClient(handler) { BaseAddress = new Uri("http://products/") }; + + var result = await CommerceTools.SearchProductsAsync(new ProductsFactory(client), "descending with purpose", categoryCode: " b ", + subCategoryCode: "em", brandCode: "yeti", minPrice: 1000.25m, maxPrice: 10000.50m); + + result.Should().BeEmpty(); + var filter = Query(handler)["$filter"].ToString(); + filter.Should().Be("category eq 'B' and subcategory eq 'EM' and brand eq 'YETI' and price ge 1000.25 and price le 10000.50"); + } + + [TestCase("category", "B", null, null, null, null, "category eq 'B'")] + [TestCase("subcategory", null, "EM", null, null, null, "subcategory eq 'EM'")] + [TestCase("brand", null, null, "FOX", null, null, "brand eq 'FOX'")] + [TestCase("minimum", null, null, null, 0, null, "price ge 0")] + [TestCase("maximum", null, null, null, null, 0, "price le 0")] + [TestCase("equal bounds", null, null, null, 100, 100, "price ge 100 and price le 100")] + public async Task Search_OptionalFilters(string scenario, string? categoryCode, string? subCategoryCode, string? brandCode, int? minPrice, int? maxPrice, string expected) + { + using var handler = new ProductsHandler("[]"); + using var client = new HttpClient(handler) { BaseAddress = new Uri("http://products/") }; + + await CommerceTools.SearchProductsAsync(new ProductsFactory(client), scenario, categoryCode: categoryCode, subCategoryCode: subCategoryCode, + brandCode: brandCode, minPrice: minPrice, maxPrice: maxPrice); + + Query(handler)["$filter"].ToString().Should().Be(expected); + } + + [Test] + public async Task Search_CodeLiteral_CannotIntroduceFilterOrQuerySyntax() + { + using var handler = new ProductsHandler("[]"); + using var client = new HttpClient(handler) { BaseAddress = new Uri("http://products/") }; + + await CommerceTools.SearchProductsAsync(new ProductsFactory(client), "bike", brandCode: "YETI' or price ge 0 or brand eq 'FOX&$inactive=true"); + + var query = Query(handler); + query["$filter"].ToString().Should().Be("brand eq 'YETI'' OR PRICE GE 0 OR BRAND EQ ''FOX&$INACTIVE=TRUE'"); + query.Should().HaveCount(3).And.NotContainKey("$inactive"); + } + + [TestCase("", null, null)] + [TestCase(null, " ", null)] + [TestCase(null, null, "")] + public void Search_BlankCode_IsRejectedBeforeHttp(string? categoryCode, string? subCategoryCode, string? brandCode) + { + using var handler = new ProductsHandler("[]"); + using var client = new HttpClient(handler) { BaseAddress = new Uri("http://products/") }; + + Action act = () => CommerceTools.SearchProductsAsync(new ProductsFactory(client), "bike", categoryCode: categoryCode, subCategoryCode: subCategoryCode, brandCode: brandCode); + + act.Should().Throw(); + handler.RequestUri.Should().BeNull(); + } + + [TestCase(-1, null)] + [TestCase(null, -1)] + [TestCase(100, 99)] + public void Search_InvalidPrices_AreRejectedBeforeHttp(int? minimum, int? maximum) + { + using var handler = new ProductsHandler("[]"); + using var client = new HttpClient(handler) { BaseAddress = new Uri("http://products/") }; + + Action act = () => CommerceTools.SearchProductsAsync(new ProductsFactory(client), "bike", minPrice: minimum, maxPrice: maximum); + + act.Should().Throw(); + handler.RequestUri.Should().BeNull(); + } + + [TestCase(0)] + [TestCase(26)] + public void Search_InvalidTake_IsRejectedBeforeHttp(int take) + { + using var handler = new ProductsHandler("[]"); + using var client = new HttpClient(handler) { BaseAddress = new Uri("http://products/") }; + Action act = () => CommerceTools.SearchProductsAsync(new ProductsFactory(client), "bike", take); + act.Should().Throw(); + handler.RequestUri.Should().BeNull(); + } + + [TestCase(HttpStatusCode.BadRequest)] + [TestCase(HttpStatusCode.ServiceUnavailable)] + public async Task Search_ApiFailure_IsNotAnEmptySuccess(HttpStatusCode status) + { + using var handler = new ProductsHandler("""{"title":"Search failed","status":400}""", status); + using var client = new HttpClient(handler) { BaseAddress = new Uri("http://products/") }; + Func act = () => CommerceTools.SearchProductsAsync(new ProductsFactory(client), "bike", categoryCode: "UNKNOWN"); + await act.Should().ThrowAsync(); + } + + [Test] + public async Task Options_ReadsLiveCodesAndParentCategories() + { + using var handler = new ProductsHandler("""{"categories":[{"code":"B","text":"Bikes"}],"sub-categories":[{"code":"EM","text":"E-MTB","categoryCode":"B"}],"brands":[{"code":"YETI","text":"Yeti Cycles"}]}"""); + using var client = new HttpClient(handler) { BaseAddress = new Uri("http://products/") }; + + var options = await CommerceTools.GetProductSearchOptionsAsync(new ProductsFactory(client)); + + handler.RequestUri.Should().Be(new Uri("http://products/api/refdata?name=categories&name=sub-categories&name=brands")); + options.Categories.Should().ContainSingle().Which.Code.Should().Be("B"); + options.SubCategories.Should().ContainSingle().Which.Code.Should().Be("EM"); + options.Brands.Should().ContainSingle().Which.Code.Should().Be("YETI"); + } + + [Test] + public void Search_ProtocolSchema_ExposesOptionalFiltersButNotServiceOrCancellation() + { + var tool = ModelContextProtocol.Server.McpServerTool.Create(CommerceTools.SearchProductsAsync); + var schema = tool.ProtocolTool.InputSchema; + var properties = schema.GetProperty("properties"); + foreach (var name in new[] { "categoryCode", "subCategoryCode", "brandCode", "minPrice", "maxPrice", "take" }) + properties.TryGetProperty(name, out _).Should().BeTrue(); + + schema.GetProperty("required").EnumerateArray().Select(x => x.GetString()).Should().Contain("searchTerm"); + properties.TryGetProperty("cancellationToken", out _).Should().BeFalse(); + } + + private static Dictionary Query(ProductsHandler handler) + { + handler.RequestUri.Should().NotBeNull(); + handler.RequestUri!.AbsolutePath.Should().Be("/api/products"); + return Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(handler.RequestUri.Query); + } + + private sealed class ProductsFactory(HttpClient client) : IHttpClientFactory + { + public HttpClient CreateClient(string name) + { + name.Should().Be("ProductsApi"); + return client; + } + } + + private sealed class ProductsHandler(string body, HttpStatusCode status = HttpStatusCode.OK) : HttpMessageHandler + { + public Uri? RequestUri { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + RequestUri = request.RequestUri; + return Task.FromResult(new HttpResponseMessage(status) { Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") }); + } + } +} diff --git a/samples/src/Contoso.Mcp/GlobalUsing.cs b/samples/src/Contoso.Mcp/GlobalUsing.cs index 1e744a05e..1ae76a2b7 100644 --- a/samples/src/Contoso.Mcp/GlobalUsing.cs +++ b/samples/src/Contoso.Mcp/GlobalUsing.cs @@ -12,6 +12,8 @@ global using ModelContextProtocol.Server; global using System; global using System.ComponentModel; +global using System.Globalization; global using System.Net; global using System.Net.Http.Json; global using System.Text.Json; +global using System.Text.Json.Serialization; diff --git a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs index 0c954e36f..7f41bcf77 100644 --- a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs +++ b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs @@ -4,24 +4,63 @@ namespace Contoso.Mcp.Tools; public static class CommerceTools { /// Searches products using the Products server's configured ranked retrieval mode. - [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products using the Products server's ranked search mode (semantic by default; optionally English full-text or hybrid). Hybrid fuses up to 100 eligible candidates per branch with Reciprocal Rank Fusion and prioritizes an exact eligible SKU. Returns up to the requested number of summaries; candidates are not guaranteed matches and may come from a bounded pool. Requires every configured index and embedding provider to be ready.")] + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Find active products using the Products server's ranked search mode (semantic, full-text or hybrid). Optional category, subcategory, brand and inclusive USD price filters restrict eligible products before ranking; all supplied filters must match. Use get_product_search_options to discover current codes rather than guessing. Use filters for explicit requirements, not inferred preferences. Hybrid fuses up to 100 eligible candidates per branch and prioritizes an exact eligible SKU. Returns up to the requested number of summaries, not guaranteed matches. Requires every configured index and embedding provider to be ready.")] public static Task SearchProductsAsync( [FromServices] IHttpClientFactory clientFactory, [Description("Product text, intended use, or a SKU. Semantic mode does not guarantee exact SKU matching; FullText and Hybrid prioritize an exact eligible SKU, while Hybrid otherwise combines lexical and semantic ranks.")] string searchTerm, [Description("Maximum number of products to return (1-25).")] int take = 10, + [Description("Optional category code from get_product_search_options, for example B for Bikes. Omit for any category.")] string? categoryCode = null, + [Description("Optional subcategory code from get_product_search_options, for example EM for E-MTB or XC for Cross country. Omit for any subcategory.")] string? subCategoryCode = null, + [Description("Optional brand code from get_product_search_options, for example YETI. Omit for any brand.")] string? brandCode = null, + [Description("Optional inclusive minimum price in USD; must be nonnegative.")] decimal? minPrice = null, + [Description("Optional inclusive maximum price in USD; must be nonnegative and at least minPrice.")] decimal? maxPrice = null, CancellationToken cancellationToken = default) { searchTerm = searchTerm.Required(); if (take is < 1 or > 25) throw new ArgumentOutOfRangeException(nameof(take), "The result limit must be between 1 and 25."); + if (minPrice < 0) + throw new ArgumentOutOfRangeException(nameof(minPrice), "The minimum price must be nonnegative."); + + if (maxPrice < 0) + throw new ArgumentOutOfRangeException(nameof(maxPrice), "The maximum price must be nonnegative."); + + if (minPrice > maxPrice) + throw new ArgumentException("The maximum price must be at least the minimum price.", nameof(maxPrice)); + + var filters = new List(); + AddCodeFilter("category", categoryCode, nameof(categoryCode)); + AddCodeFilter("subcategory", subCategoryCode, nameof(subCategoryCode)); + AddCodeFilter("brand", brandCode, nameof(brandCode)); + if (minPrice.HasValue) + filters.Add($"price ge {minPrice.Value.ToString(CultureInfo.InvariantCulture)}"); + + if (maxPrice.HasValue) + filters.Add($"price le {maxPrice.Value.ToString(CultureInfo.InvariantCulture)}"); + return ProductsApi(clientFactory) .Send(HttpMethod.Get, "/api/products") .WithSearch(searchTerm) + .WithQuery(filter: filters.Count == 0 ? null : string.Join(" and ", filters)) .WithPaging(take: take) .GetValueAsync(cancellationToken); + + /// Adds an equality predicate with a normalized, escaped code literal. + void AddCodeFilter(string field, string? code, string name) + { + if (code is not null) + filters.Add($"{field} eq '{code.Required(name).Trim().ToUpperInvariant().Replace("'", "''")}'"); + } } + /// Gets the live reference-data codes supported by structured product search filters. + [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("List current product-search categories, subcategories (including their parent category codes), and brands. Use the returned codes in search_products filters; do not invent codes or treat tags as filterable fields. These are classification options, not product availability or result counts.")] + public static Task GetProductSearchOptionsAsync([FromServices] IHttpClientFactory clientFactory, CancellationToken cancellationToken = default) + => ProductsApi(clientFactory) + .Send(HttpMethod.Get, "/api/refdata?name=categories&name=sub-categories&name=brands") + .GetValueAsync(cancellationToken); + /// Gets the full details for a product. [McpServerTool(UseStructuredContent = true, ReadOnly = true), Description("Get product details by product ID.")] public static Task GetProductAsync( diff --git a/samples/src/Contoso.Mcp/Tools/ProductSearchOptions.cs b/samples/src/Contoso.Mcp/Tools/ProductSearchOptions.cs new file mode 100644 index 000000000..cc529479e --- /dev/null +++ b/samples/src/Contoso.Mcp/Tools/ProductSearchOptions.cs @@ -0,0 +1,15 @@ +namespace Contoso.Mcp.Tools; + +/// Live Products reference data available for structured search filters. +public sealed class ProductSearchOptions +{ + /// Categories and their filter codes. + public required Category[] Categories { get; init; } + + /// Subcategories, their filter codes and parent category codes. + [JsonPropertyName("sub-categories")] + public required SubCategory[] SubCategories { get; init; } + + /// Brands and their filter codes. + public required Brand[] Brands { get; init; } +} From e048be47b8e741e2471c58748b20b64a1e48032e Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 20 Oct 2026 10:30:35 -0700 Subject: [PATCH 14/25] Finalize v4.2.0 HTTP builders and release guidance Add single-use fluent HTTP request execution, explicit JSON/content bodies and verb shortcuts; update callers, coverage, SQL resource guidance and MCP testing documentation. Complete the v4.2.0 changelog. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .github/copilot-instructions.md | 2 +- .../coreex-repositories.instructions.md | 29 +- .github/skills/coreex-adapter/SKILL.md | 2 +- .../coreex-adapter/references/workflow.md | 5 +- .github/skills/coreex-repository/SKILL.md | 3 + .../coreex-repository/references/workflow.md | 36 +- CHANGELOG.md | 12 +- samples/docs/infrastructure-layer.md | 5 +- samples/docs/search.md | 23 +- .../src/Contoso.Mcp/Tools/CommerceTools.cs | 15 +- .../Clients/Products/ProductsHttpClient.cs | 2 +- .../Clients/SendGrid/SendGridHttpClient.cs | 2 +- src/CoreEx.Database/AGENTS.md | 24 +- src/CoreEx.Database/README.md | 32 +- src/CoreEx/AGENTS.md | 7 +- src/CoreEx/Extensions.HttpClient.cs | 175 +++------- src/CoreEx/Extensions.HttpRequestMessage.cs | 31 ++ src/CoreEx/Http/HttpRequestBuilder.cs | 93 ++++- src/CoreEx/Http/README.md | 41 ++- src/CoreEx/README.md | 2 +- tests/CoreEx.Test.Unit/GlobalUsing.cs | 6 + .../Http/HttpClientExtensionsTests.cs | 4 +- .../Http/HttpRequestBodyTests.cs | 327 ++++++++++++++++++ .../Http/HttpRequestBuilderTests.cs | 36 +- 24 files changed, 717 insertions(+), 197 deletions(-) create mode 100644 tests/CoreEx.Test.Unit/Http/HttpRequestBodyTests.cs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 96ce5d65b..daa58fd46 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -115,7 +115,7 @@ Connection strings for each service in development are in each host's `appsettin - When a domain needs to interact with another domain or external service, define an **adapter interface** in `Application\Adapters\`. The Application layer depends on this domain-idiomatic abstraction — never on the remote API's schema or transport directly. - Infrastructure implements the adapter using a **typed HTTP client** (`Infrastructure\Clients\`) for the transport concern, keeping client and orchestration in separate focused classes. - Register typed HTTP clients in `Program.cs` via `builder.AddTypedHttpClient("ServiceName")` (or `builder.AddNamedHttpClient("ServiceName")`), which automatically wires base address configuration, `IdempotencyKeyHandler`, service discovery, and `Microsoft.Extensions.Http.Resilience` standard resilience handlers. -- Inside typed clients, use `client.Send(...)` or `client.SendValue(...)` returning `HttpRequestBuilder` for fluent request building (`WithQuery`, `WithPaging`, headers, `WithStatusResult`, `WithValidator`) and single-await execution with automatic response disposal. +- Inside typed clients, use `client.Send(method, uri)` or `Get`/`Post`/`Put`/`Patch`/`Delete`, then `WithJson(value)` for JSON or `WithContent(content)` for explicit content. `HttpRequestBuilder` is single-use; exactly one body assignment is allowed. Fluent configuration (`WithQuery`, `WithPaging`, headers, `WithStatusResult`, `WithValidator`) precedes terminal execution with automatic response disposal (except raw-response terminals). - Two adapter roles appear in Shopping: - **Synchronous adapter** (`IProductAdapter`) — real-time calls (e.g. inventory reservation at checkout); the HTTP client is called live inside the unit of work. - **Sync/replication adapter** (`IProductSyncAdapter`) — event-driven data replication; receives published domain events and maintains a local eventually-consistent copy in the domain's own store. diff --git a/.github/instructions/coreex-repositories.instructions.md b/.github/instructions/coreex-repositories.instructions.md index 09c042dc7..53ecc3d68 100644 --- a/.github/instructions/coreex-repositories.instructions.md +++ b/.github/instructions/coreex-repositories.instructions.md @@ -23,10 +23,11 @@ tags: ["repositories", "infrastructure", "data-access", "efcore", "cosmos", "map | Package | Key types provided | |---|---| -| `CoreEx` | `[ScopedService]`, `.ThrowIfNull()`, `ItemsResult`, `Result`, `.GoAsync()`, `.ThenAs()`, `.ThenAsAsync()`, `HttpRequestBuilder`, `client.Send(...)`, `client.SendValue(...)`, `builder.AddNamedHttpClient(...)`, `builder.AddTypedHttpClient(...)` | +| `CoreEx` | `[ScopedService]`, `.ThrowIfNull()`, `ItemsResult`, `Result`, `.GoAsync()`, `.ThenAs()`, `.ThenAsAsync()`, `HttpRequestBuilder`, `client.Send(...)`, `client.Post(...).WithJson(...)`, `builder.AddNamedHttpClient(...)`, `builder.AddTypedHttpClient(...)` | | `CoreEx.Validation` | `Validator`, `Validator.Create()`, `.HasProperty(...)`, `.Mandatory()`, `.Email()` for inline response-contract rules | | `CoreEx.Events` | `EventData` | | `CoreEx.Data` | `IUnitOfWork`, `DataResult`, `QueryArgsConfig`, `QueryFilterOperator`, `.Where(parsed)`, `.OrderBy(parsed)` | +| `CoreEx.Database` | `IDatabase`, `SqlStatement`, `DatabaseCommand` for custom relational SQL | | `CoreEx.Cosmos` | `CosmosDb`, `CosmosDbContainer`, `CosmosDbMappedContainer`, `CosmosDbArgs`, `CosmosDbQuery`, `.GetAsync()`, `.CreateAsync()`, `.UpdateAsync()`, `.DeleteAsync()`, `.Query()`, `.ToMappedItemsResultAsync()` | | `CoreEx.EntityFrameworkCore` | `EfDb`, `EfDbModel`, `EfDbMappedModel`, `EfDbOptions`, `.GetAsync()`, `.CreateAsync()`, `.UpdateAsync()`, `.DeleteAsync()`, `.GetWithResultAsync()`, `.CreateWithResultAsync()`, `.UpdateWithResultAsync()`, `.Query()`, `.ToMappedItemsResultAsync()` | | `CoreEx.Database.SqlServer` | SQL Server outbox publisher, ADO.NET helpers | @@ -214,6 +215,26 @@ Querying (relational): `Query(...)` returns a filtered `IQueryable` (log Per-model behaviour is configured on `EfDbOptions` / `EfDbModelOptions`: `WithModel(...)`, `WithLogicalDeleteFilter()`, `WithTenantFilter()`, `WithFilter(...)`, `WithGetKey(...)`, `WithArgs(...)`, `WithOnBeforeCreateOrUpdate(...)`, `WithUpdateModelMapper(...)`. +## Custom Relational SQL + +Use `EfDb` for normal CRUD and LINQ queries. For custom relational operations, put application-owned static SQL +(short or long) in a `.sql` / `.pgsql` file under `Infrastructure/Repositories/Sql/`, embed it in the project, then +load it with `SqlStatement.FromResource(...)` and execute via `_ef.Database.Statement(...)`. Keep +resource filenames unique within the assembly because lookup matches by suffix. Bind all values with `.Param(...)`. +Reserve `FromText(...)` for SQL that genuinely must be generated at runtime; validate or safely quote dynamic +identifiers, and continue binding data values. Schema changes stay in DbEx migrations rather than runtime SQL +resources. + +```csharp +private static readonly SqlStatement _isReady = + SqlStatement.FromResource("ProductSearchIsReady.pgsql"); + +var ready = await _ef.Database.Statement(_isReady) + .Param("Profile", profile, dbType: DbType.String) + .ScalarAsync(cancellationToken) + .ConfigureAwait(false); +``` + ## Dynamic Query Configuration > **GlobalUsing requirement:** `QueryArgsConfig` and the related query types live in the `CoreEx.Data.Querying` namespace. When introducing query configuration, ensure `global using CoreEx.Data.Querying;` is present in the Infrastructure project's `GlobalUsing.cs` — add it if missing. @@ -446,7 +467,8 @@ public class ProductsHttpClient(HttpClient httpClient) public Task CreateReservationAsync(MovementRequest request, CancellationToken cancellationToken = default) => _httpClient - .SendValue(HttpMethod.Post, "api/inventory/reserve", request) + .Post("api/inventory/reserve") + .WithJson(request) .ToResultAsync(cancellationToken); } @@ -469,7 +491,7 @@ Keep the typed HTTP client and the adapter orchestration in separate, independen Register typed HTTP clients in the host `Program.cs` via `builder.AddTypedHttpClient("ProductsApi")` (or `builder.AddNamedHttpClient("ProductsApi")`). This automatically binds `BaseAddress` from configuration, adds `IdempotencyKeyHandler`, wires service discovery, and adds `Microsoft.Extensions.Http.Resilience` standard resilience handlers. -Inside the client, use `_httpClient.Send(...)` and `_httpClient.SendValue(...)` to fluently construct requests (adding headers, queries, or paging) and execute them. Terminal methods on `HttpRequestBuilder` (`ToResultAsync`, `GetValueAsync`, `GetStringAsync`, etc.) execute the request and **automatically dispose** the underlying `HttpResponseMessage`. +Inside the client, use `_httpClient.Send(method, uri)` or `Get`/`Post`/`Put`/`Patch`/`Delete` to construct requests. Add JSON via `WithJson(value)` or explicit content via `WithContent(content)`; exactly one body assignment is allowed, and `Patch` does not imply JSON Merge Patch. Builders are single-use, even after failure/cancellation; configure headers, queries, and paging before execution. Terminal methods (`ToResultAsync`, `GetValueAsync`, `GetStringAsync`, etc.) execute the request and **automatically dispose** the response (except raw-response terminals). ### Client response validation @@ -546,6 +568,7 @@ Always call `.ConfigureAwait(false)` on every `await` inside repository and adap ## Further Reading - [Infrastructure Layer Guide](/.github/docs/coreex/infrastructure-layer.md) — full walkthrough of persistence models, repositories, mapping, and external client/adapter patterns (docs-sync cache; after `/coreex-docs-sync`). Source: [samples/docs/infrastructure-layer.md](https://github.com/Avanade/CoreEx/blob/main/samples/docs/infrastructure-layer.md). +- [CoreEx.Database guide](/.github/docs/coreex/agents/CoreEx.Database.md) — `IDatabase`, `DatabaseCommand`, and embedded SQL statements (docs-sync cache; after `/coreex-docs-sync`). - [Pattern Catalog](/.github/docs/coreex/patterns.md) — Adapter, Repository, Mapper, Persistence, and HTTP Client pattern entries (docs-sync cache; after `/coreex-docs-sync`). Source: [samples/docs/patterns.md](https://github.com/Avanade/CoreEx/blob/main/samples/docs/patterns.md). - [Layer Dependencies](/.github/docs/coreex/layers.md) — layer dependency rules and the role of the Infrastructure layer (docs-sync cache; after `/coreex-docs-sync`). Source: [samples/docs/layers.md](https://github.com/Avanade/CoreEx/blob/main/samples/docs/layers.md). - [Tooling Guide](/.github/docs/coreex/tooling.md) — `*.Database` project: schema, persistence-model generation, and outbox provisioning (docs-sync cache; after `/coreex-docs-sync`). Source: [samples/docs/tooling.md](https://github.com/Avanade/CoreEx/blob/main/samples/docs/tooling.md). diff --git a/.github/skills/coreex-adapter/SKILL.md b/.github/skills/coreex-adapter/SKILL.md index fca62513c..0d33cf4b5 100644 --- a/.github/skills/coreex-adapter/SKILL.md +++ b/.github/skills/coreex-adapter/SKILL.md @@ -45,7 +45,7 @@ Guides you through adding or modifying an adapter — the boundary that isolates - `[ScopedService]` on the implementation — auto-discovered via `AddDynamicServicesUsing()` - Typed HTTP client registered via `builder.AddTypedHttpClient("ServiceName")` in `Program.cs` — auto-configures `BaseAddress`, `IdempotencyKeyHandler`, service discovery, and standard resilience handlers - Never call `HttpClient` directly in adapter methods — always delegate to the typed client class -- Inside the client, use `client.Send(...)` and `client.SendValue(...)` returning `HttpRequestBuilder` for fluent request building (headers, queries, paging) and terminal execution with automatic response disposal +- Inside the client, use `client.Send(method, uri)` or `Get`/`Post`/`Put`/`Patch`/`Delete`, then `WithJson(value)` for JSON or `WithContent(content)` for explicit content. The builder is single-use, exactly one body assignment is allowed, and `Patch` does not imply JSON Merge Patch. Configure headers, queries, and paging before terminal execution. - **Response validation belongs to the client**, not `Application/Validators/`. For small, client-only rules use a private static readonly `Validator` configured once with `Validator.Create().HasProperty(...)`, then chain `.WithValidator(_validator).ToResultAsync(ct)`. Validate fields the consumer requires; do not copy create/update rules or add identifier matching by default. Extract a colocated validator only when complexity or genuine reuse warrants it; see the workflow. - Terminal methods on `HttpRequestBuilder` (`ToResultAsync`, `GetValueAsync`, etc.) **automatically dispose** the response. When inspecting raw responses (e.g. `builder.GetResponseAsync()`), dispose the response with `using var`. Reported validation errors become `HttpRequestException` failures with inner diagnostics, not caller validation errors. Test inline rules through the client, including invalid 2xx responses and success; cached adapters must not cache invalid responses. - `response.ToResultAsync()` maps HTTP status to `Result` (2xx → `Success`, error + ProblemDetails → `BusinessException`/`ProblemDetailsException`, plain error → `HttpRequestException`) diff --git a/.github/skills/coreex-adapter/references/workflow.md b/.github/skills/coreex-adapter/references/workflow.md index f9dab1736..7590a5310 100644 --- a/.github/skills/coreex-adapter/references/workflow.md +++ b/.github/skills/coreex-adapter/references/workflow.md @@ -74,7 +74,7 @@ public interface IProductSyncAdapter Create in `Infrastructure/Clients/{ExternalDomain}/`. One class per external service; one method per remote endpoint. -Use `_httpClient.Send(...)` and `_httpClient.SendValue(...)` to fluently construct requests (adding headers, query parameters, or paging) and execute them. Terminal methods on `HttpRequestBuilder` (`ToResultAsync`, `GetValueAsync`, `GetStringAsync`, etc.) execute the request and **automatically dispose** the underlying `HttpResponseMessage`. +Use `_httpClient.Send(method, uri)` or the `Get`/`Post`/`Put`/`Patch`/`Delete` shortcuts to construct requests; add JSON with `WithJson(value)` or explicit content with `WithContent(content)`. The builder is single-use (including failures/cancellation), and exactly one body assignment is allowed. Configure headers, query parameters, and paging before execution. `Patch` does not imply JSON Merge Patch. Terminal methods (`ToResultAsync`, `GetValueAsync`, `GetStringAsync`, etc.) execute the request and **automatically dispose** the response (except raw-response terminals). ### Client class @@ -92,7 +92,8 @@ public class ProductsHttpClient(HttpClient httpClient) /// Creates a new inventory reservation. public Task CreateReservationAsync(MovementRequest request, CancellationToken ct = default) => _httpClient - .SendValue(HttpMethod.Post, "api/inventory/reserve", request) + .Post("api/inventory/reserve") + .WithJson(request) .ToResultAsync(ct); } ``` diff --git a/.github/skills/coreex-repository/SKILL.md b/.github/skills/coreex-repository/SKILL.md index 4297d692a..446422103 100644 --- a/.github/skills/coreex-repository/SKILL.md +++ b/.github/skills/coreex-repository/SKILL.md @@ -50,6 +50,7 @@ Guides you through creating or modifying a CoreEx Infrastructure-layer repositor - `*WithResultAsync` variants for `Result` ROP pipelines (per-project style choice) - `BiDirectionMapper`: override **both** `OnMap` overloads; map `Id` explicitly; **never** map `ETag` or `ChangeLog` — base mapper owns them - `QueryArgsConfig`: create a dedicated `{Name}QueryArgsConfig : QueryArgsConfig<{Name}QueryArgsConfig>` class per entity in `Infrastructure/Repositories/`; access via `.Default`; call `.Parse(query).ThrowOnError()` before use — never instantiate per-request. Relational queries materialize from `IQueryable` extensions; Cosmos queries materialize from `CosmosDbQuery` instance methods (`Container.Query(...).ToMappedItemsResultAsync(...)`). +- Custom relational SQL: prefer `EfDb` for ordinary CRUD/LINQ; put application-owned static SQL (short or long) in embedded `.sql` / `.pgsql` resources loaded by `SqlStatement.FromResource`, and bind values with `.Param(...)`. Reserve `FromText(...)` for SQL genuinely generated at runtime, with dynamic identifiers validated or safely quoted; schema changes belong in DbEx migrations. - `AddReferenceDataField(field, model, ...)`: `field` is always the contract's generated nav property (`{Name}`, never `{Name}Code`) — a fixed, documented source-generator convention, **not** something to verify by exploring generated code - `ToMappedItemsResultAsync(mapper, paging, cancellationToken: cancellationToken)`: always pass `cancellationToken` **by name** — `autoCount` (`bool`, defaults `true`) sits before it in the signature, and a bare positional token there fails to compile - Always `.ConfigureAwait(false)` on every `await` @@ -59,7 +60,9 @@ For full workflow and code examples see [`references/workflow.md`](references/wo ## Key References - [`/.github/instructions/coreex-repositories.instructions.md`](/.github/instructions/coreex-repositories.instructions.md) — full conventions: EfDb, DbContext, mappers, query config, adapters +- [CoreEx.Database guide](/.github/docs/coreex/agents/CoreEx.Database.md) — direct relational SQL with `IDatabase`, `SqlStatement`, and embedded resources - Related skills: [`coreex-db-migration`](../coreex-db-migration/SKILL.md) (generates the persistence models this maps), [`coreex-app-service`](../coreex-app-service/SKILL.md) + [`coreex-adapter`](../coreex-adapter/SKILL.md) (consumers), [`coreex-aggregate`](../coreex-aggregate/SKILL.md) (persisting aggregates) - Illustrative examples (CoreEx sample — not present in your project): - [contract-mapping repositories](https://github.com/Avanade/CoreEx/tree/main/samples/src/Contoso.Products.Infrastructure/Repositories) — `ProductRepository` (CRUD + query), `ProductsEfDb`, `ProductsDbContext` — and [BiDirectionMapper](https://github.com/Avanade/CoreEx/tree/main/samples/src/Contoso.Products.Infrastructure/Mapping) (`ProductMapper`) + - [Custom SQL with embedded resources](https://github.com/Avanade/CoreEx/blob/main/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs) — `SqlStatement.FromResource`, bound parameters, and `ProductsEfDb.Database` - [Domain-mapping repositories](https://github.com/Avanade/CoreEx/tree/main/samples/src/Contoso.Shopping.Infrastructure/Repositories) — `BasketRepository` (`Result` + Domain mapping) — and [one-way Domain ↔ Persistence mappers](https://github.com/Avanade/CoreEx/tree/main/samples/src/Contoso.Shopping.Infrastructure/Mapping) (`BasketMapper`) diff --git a/.github/skills/coreex-repository/references/workflow.md b/.github/skills/coreex-repository/references/workflow.md index 406cb5929..1555b14be 100644 --- a/.github/skills/coreex-repository/references/workflow.md +++ b/.github/skills/coreex-repository/references/workflow.md @@ -200,6 +200,38 @@ public Task DeleteAsync(string id) => _ef.{Name}s.DeleteAsync(id); --- +## Custom SQL Operations + +Use EF Core / `EfDb` for ordinary CRUD and LINQ queries. Choose direct SQL only when the operation needs database +features or composition that the model APIs do not express cleanly. Put application-owned static SQL in a named +`.sql` / `.pgsql` file under `Infrastructure/Repositories/Sql/`—regardless of length—explicitly embed those files +in the Infrastructure `.csproj`, and load each statement once with `SqlStatement.FromResource`. + +```xml + + + +``` + +```csharp +private static readonly SqlStatement _isReady = + SqlStatement.FromResource("ProductSearchIsReady.pgsql"); + +var ready = await _ef.Database.Statement(_isReady) + .Param("Profile", profile, dbType: DbType.String) + .ScalarAsync(cancellationToken) + .ConfigureAwait(false); +``` + +Use `.Param(...)` for values; never interpolate data values into SQL. Reserve `SqlStatement.FromText(...)` for SQL +that genuinely must be generated at runtime (for example, when a validated or safely quoted identifier is dynamic); +data values must still be bound as parameters. Keep schema changes in the `*.Database` project's DbEx migrations +rather than embedding DDL in runtime resources. See the Products sample's +[`ProductSearchIndex`](https://github.com/Avanade/CoreEx/blob/main/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs) +and [search guide](https://github.com/Avanade/CoreEx/blob/main/samples/docs/search.md#postgresql-and-pgvector). + +--- + ## Path C — Add a Query Method with Dynamic Filtering/Ordering ### C0 — Interview (required before writing any code) @@ -263,9 +295,9 @@ Collect the following from the developer. **AI cannot infer field selection, ope **Case sensitivity (string fields):** `.AsUpperCase()` wraps both the stored column value and the filter input in `ToUpperInvariant()`, making all string comparisons case-insensitive. `.AsLowerCase()` does the same with `ToLowerInvariant()`. Apply to any string field where case should be ignored (SKUs, codes, free-text search). Omit for fields where case is meaningful. -### C2 — Create the `{Name}QueryArgsConfig` class +### C2 — Create the query config class -Create `Infrastructure/Repositories/{Name}QueryArgsConfig.cs`. Extend `QueryArgsConfig<{Name}QueryArgsConfig>` (CRTP) — all configuration goes in the constructor; the base provides a thread-safe lazy `Default` singleton: +Create `Infrastructure/Repositories/{Name}QueryArgsConfig.cs`. The class extends `QueryArgsConfig<{Name}QueryArgsConfig>` (CRTP) — all configuration goes in the constructor; the base provides a thread-safe lazy `Default` singleton: ```csharp namespace {Solution}.Infrastructure.Repositories; diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c23e9b73..f284f62e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,17 @@ Represents the **NuGet** versions. ## v4.2.0 -- *Enhancement:* Added OData-style free-text `$search` support to the `QueryArgs` querying pipeline: `QueryArgs.Search`; `QueryArgsConfig.WithSearch(Action)` with `QuerySearchMode` (`Keyword` executed by `QueryExtensions.Where` as a provider-agnostic predicate where every term must match a configured keyword field; `FullText`/`Semantic`/`Hybrid` are relevance-ranked and executed by the repository via `QueryArgsParseResult.IsRankedSearch`/`SearchResult.Text`, with `$orderby` rejected); `QuerySearchParserException`; `WithSearch` on `HttpRequestMessage`/`HttpRequestBuilder`; `QueryAttribute.SupportsSearch` (NSwag `$search` parameter); and a GraphQL-lite `search` argument on list roots. `$search` is rejected where not configured. The Products sample (and its MCP `search_products` tool) now uses `$search`. +- *Enhancement:* Added single-use fluent HTTP request builders with `Get`/`Post`/`Put`/`Patch`/`Delete` shortcuts, `WithJson` for JSON bodies and `WithContent` for explicit HTTP content (also available on `HttpRequestMessage`). Exactly one body assignment is permitted; repeated execution, configuration after execution starts, and conflicting body assignments throw `InvalidOperationException`. PATCH does not imply JSON Merge Patch. +- *Enhancement:* Added `WithStatusResult` / `WithErrorStatus` to HTTP request builders, including validated builders, to map selected response statuses to CoreEx results before normal deserialization and validation. `ValidatedHttpRequestBuilder` integrates response validation with fluent request execution and automatic response disposal. +- *Enhancement:* Added `HttpResponseMessage.GetStringAsync()` and `ToStringResultAsync()` for raw response text, returning a null value for successful empty/204 responses; corresponding terminal methods are available on `HttpRequestBuilder`. +- *Enhancement:* Added host-builder `AddNamedHttpClient()` and its `AddHttpClient()` alias, sharing idempotency key handling, service discovery and standard resilience with `AddTypedHttpClient()`. A separate configuration-section overload supports augmenting SDK-owned named clients without replacing their registrations; base addresses may also be supplied by an existing registration or configuration callback. +- *Enhancement:* Added `IEfDb.SqlQuery(SqlStatement, params object[])` for deferred, parameterized, composable EF queries over scalar or unmapped result types. Queries use the existing context and transaction, not the CoreEx Database command pipeline; CoreEx tenant and logical-delete filters are not automatically applied. +- *Enhancement:* Added `ReferenceDataOrchestrator.PrefetchAsync(IEnumerable)` for typed reference-data prefetch, validating all requested types before loading, deduplicating them and honoring configured parallelism. Name-based prefetch now also checks cancellation immediately. +- *Enhancement:* Added OData-style free-text `$search` support to the `QueryArgs` querying pipeline: `QueryArgs.Search`; `QueryArgsConfig.WithSearch(Action)` with `QuerySearchMode` (`Keyword` executed by `QueryExtensions.Where` as a provider-agnostic predicate where every term must match a configured keyword field; `FullText`/`Semantic`/`Hybrid` are relevance-ranked and executed by the repository via `QueryArgsParseResult.IsRankedSearch`/`SearchResult.Text`, with `$orderby` rejected); `QuerySearchParserException`; `WithSearch` on `HttpRequestMessage`/`HttpRequestBuilder`; `QueryAttribute.SupportsSearch` (NSwag `$search` parameter); and a GraphQL-lite `search` argument on list roots. `$search` is rejected where not configured. The Products sample (and its MCP `search_products` tool) now uses `$search` to demonstrate. +- *Behavior change:* Added a string-specific `Required()` validation overload that rejects null, empty and whitespace values with a `ValidationException`. Existing string call sites now reject empty/whitespace strings rather than only null. +- *Maintenance:* Updated framework dependencies and corresponding `CoreEx.Template` dependencies, including Aspire 13.6.1, EF/.NET servicing packages and FusionCache 2.9.0. The SQL Server package now explicitly references `System.Security.Cryptography.Pkcs` 10.0.13; the Aspire template's MessagePack override is updated to 2.5.307. +- *Fixed:* Named/typed HTTP client registration now binds standard resilience settings from the nested `Resilience` configuration section rather than the parent client section. +- *Fixed:* SQL Server and PostgreSQL unit-of-work root transactions now honor `DatabaseArgs.IsolationLevel`; nested invocations use savepoints and inherit the existing transaction's isolation level. - *Fixed:* `HttpRequestMessage.WithQuery(QueryArgs)` (and therefore `HttpRequestBuilder.WithQuery(QueryArgs)`) previously ignored `IsIncludeText` and `IsIncludeInactive`; these are now sent as `$text=true` and `$inactive=true` respectively. ## v4.1.0 diff --git a/samples/docs/infrastructure-layer.md b/samples/docs/infrastructure-layer.md index 950e32ed6..8c85398cb 100644 --- a/samples/docs/infrastructure-layer.md +++ b/samples/docs/infrastructure-layer.md @@ -131,7 +131,8 @@ public class ProductsHttpClient(HttpClient httpClient) public Task CreateReservationAsync(MovementRequest request, CancellationToken ct = default) => _httpClient - .SendValue(HttpMethod.Post, "api/inventory/reserve", request) + .Post("api/inventory/reserve") + .WithJson(request) .ToResultAsync(ct); } @@ -143,7 +144,7 @@ public class ProductAdapter(ShoppingEfDb ef, IEventPublisher eventPublisher, Pro } ``` -This split — client for transport, adapter for orchestration — keeps each class focused on a single responsibility and makes the HTTP interaction independently testable. Using `_httpClient.Send(...)` or `_httpClient.SendValue(...)` returns an `HttpRequestBuilder`, allowing fluent setup of headers, query parameters, and paging, with automatic response disposal when calling terminal methods (`ToResultAsync`, `GetValueAsync`, etc.). +This split — client for transport, adapter for orchestration — keeps each class focused on a single responsibility and makes the HTTP interaction independently testable. Use `_httpClient.Send(method, uri)` or `Get`/`Post`/`Put`/`Patch`/`Delete`, with `WithJson(value)` for JSON or `WithContent(content)` for explicit content. Builders are single-use and permit exactly one body assignment. Configure headers, query parameters, and paging before terminal methods (`ToResultAsync`, `GetValueAsync`, etc.), which automatically dispose the response unless returning it raw. `Patch` selects only the verb, not JSON Merge Patch. `CustomersHttpClient` / `CustomerAdapter` are the purely real-time variant: no local replica, the adapter delegates to the client (`GET api/customers/{id}`), mapping a 404 to `Result.NotFoundError()`. The lookup is wrapped in a single `IHybridCache.GetOrCreateWithResultAsync(...)` call: only successful results are cached (short, configurable expiry via `CoreEx:Caching:Customer:*`), failures such as not-found are never cached. The lookup is single-flight per key (per node) under FusionCache, so concurrent misses make one remote call; coalescing across nodes sharing Redis would need an opt-in FusionCache distributed locker, which isn't worth it for this cheap read. The client is registered in the Api and Subscribe hosts with `AddTypedHttpClient("CustomersApi")`, the `CustomersApi:BaseAddress` setting being overridden with service discovery under Aspire. diff --git a/samples/docs/search.md b/samples/docs/search.md index 4afb11bf0..dd1632579 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -12,7 +12,7 @@ The goal is not merely to store some numbers and label the result "AI". It is to |---|---|---| | CoreEx `QueryArgs.Search` and search parser configuration | Implemented | REST, HTTP clients, OpenAPI, and GraphQL-lite can carry search text. | | Products semantic search | Implemented; default for `$search` | Exact cosine-distance ranking replaces the earlier keyword matching. Requests without `$search` retain ordinary filtering and ordering. | -| MCP `search_products` forwarding to Products | Implemented | The tool calls the Products API with `$search`; it does not query the database. | +| MCP `search_products` forwarding to Products | Implemented | The tool forwards `$search`, optional category/subcategory/brand/price filters, and paging to the Products API; it does not query the database. | | Ranked-search mode plumbing | Implemented | CoreEx validates and carries ranked search intent; the repository must execute it. | | Canonical product search document | Implemented | Versioned, whitespace-normalized documents and source hashes exclude price, status, inventory and business metadata. | | Product descriptions and price filtering | Implemented | Optional descriptive content supports CRUD; decimal price constraints use existing REST and GraphQL-lite query configuration. | @@ -21,7 +21,7 @@ The goal is not merely to store some numbers and label the result "AI". It is to | Independent PostgreSQL full-text retrieval | Implemented; opt-in server mode | Weighted English `tsvector`/GIN matching, safe websearch parsing, exact eligible SKU first, no embedding calls. | | Hybrid ranking / RRF | Implemented; opt-in server mode | Bounded lexical and semantic lists share live eligibility and structured filters, then fuse by RRF with exact eligible-SKU priority. | | Explicit backfill / asynchronous search indexing | Implemented | Test setup publishes an atomic initial backfill; one globbed product-event subscriber reconciles subsequent changes, including logical deletion. | -| Real-model relevance evaluation | v1 baseline historical; expanded MTB v2 catalog and judgments measured locally | A version-controlled 170-product mountain-bike catalog and 60-query reviewed set use an isolated seed and opt-in REST/Ollama runner. The checked-in v1 report remains historical and is not comparable to v2. Scores are observations, not quality gates. | +| Real-model relevance evaluation | Historical v1 baseline and pre-SKU-rename v2 measurements; current catalog not re-evaluated | A version-controlled 170-product mountain-bike catalog and 60-query reviewed set use an isolated seed and opt-in REST/Ollama runner. The checked-in v1 report remains historical and is not comparable to v2; the last v2 report predates the current SKU rename. Scores are observations, not quality gates. | | HNSW benchmark | Proposed | Exact full-catalog retrieval is still the comparison baseline; index performance has not been measured. | The default mode remains semantic. Hybrid is enabled only with `Products:Search:Mode=Hybrid`; it requires both independently ready projections and a successful query embedding. The fixed initial policy takes the top 100 eligible candidates from each branch, uses RRF with `k=60`, prioritizes an eligible exact canonical SKU, and deterministically breaks remaining ties. Hybrid count is the deduplicated bounded union (at most 200), not a full-catalog or relevance count; paging is stable within that pool. These are explicit initial policy choices, not proven optimal tuning. @@ -362,7 +362,20 @@ MCP (Model Context Protocol) lets an AI client discover and invoke tools such as The tool sends search text to Products and returns product data. It does not generate its own embeddings, query PostgreSQL directly, or maintain its own index. Non-MCP clients use the same capability. -The tool description defers to the Products server's configured ranked mode: semantic by default, or lexical/hybrid when explicitly selected. It does not promise exact-SKU matching in semantic mode; FullText and Hybrid prioritize an exact eligible SKU. Hybrid requires both indexes and embedding inference, and its results come from a bounded union. Price/brand filters remain available in REST/GraphQL-lite but are not yet arguments on the MCP tool. +The tool description defers to the Products server's configured ranked mode: semantic by default, or lexical/hybrid when explicitly selected. It does not promise exact-SKU matching in semantic mode; FullText and Hybrid prioritize an exact eligible SKU. Hybrid requires both indexes and embedding inference, and its results come from a bounded union. `search_products` accepts optional category, subcategory, brand, and inclusive USD price filters, plus a result limit; `get_product_search_options` returns the live codes and category/subcategory relationship so clients do not have to guess. The tool combines supplied filters with AND semantics and forwards them to the Products API, which applies them before ranking. + +#### Testing MCP search + +Keep deterministic tool behavior checks separate from protocol and model evaluation: + +| Test level | What it verifies | Run | +|---|---|---| +| Direct MCP tool tests | Calls `CommerceTools` directly with a mocked Products HTTP handler; checks query forwarding, special-character handling, combined filters, culture-invariant prices, validation, API failures, live filter options, and the MCP input schema. This does not start the MCP server, database, or embedding model. | `dotnet test samples/aspire/Contoso.Test.Aspire --filter "FullyQualifiedName~ProductSearchToolTests"` | +| Streamable HTTP protocol E2E | Starts the real Aspire topology, connects through the MCP client SDK, discovers tools and output schemas, then exercises semantic product search and the basket lifecycle through MCP. This verifies the MCP transport and real cross-domain calls, not just the static tool methods. | `dotnet test samples/aspire/Contoso.Test.Aspire --filter "FullyQualifiedName~Contoso_Mcp_FullBasketLifecycle"` | + +The E2E fixture migrates/seeds Products, Shopping, and Customers, clears Redis, rebuilds the real-model search projection, and resets Service Bus before starting the hosts. Run it only against disposable local infrastructure; it resets shared local test services. Locally, start the [Compose search profile and install the pinned model](local-dev.md#optional-local-embedding-model) first. CI includes this test in `CoreEx.Samples.Test.E2E.slnf`; the workflow starts the search profile and pulls the model before running Aspire E2E tests. + +These tests verify tool behavior when called and the real MCP protocol flow. They do not test whether an LLM chooses to call `search_products` for a natural-language prompt; that is a separate, nondeterministic agent evaluation, not a deterministic CI test. Search does not resolve authentication or authorization. The sample's existing authentication gaps remain separate concerns. @@ -874,7 +887,9 @@ These files own the implemented behavior and the foundations for later milestone | [Database configuration](../src/Contoso.Products.Database/dbex.yaml) | DbEx-owned persistence generation and schema configuration. | | [DbContext](../src/Contoso.Products.Infrastructure/Repositories/ProductsDbContext.cs) | Hand-authored EF/Npgsql composition; generated partials remain generator-owned. | | [Subscribe host](../src/Contoso.Products.Subscribe/Program.cs) | Existing message receiving/composition foundations. | -| [MCP tools](../src/Contoso.Mcp/Tools/CommerceTools.cs) | API-only search forwarding and tool description. | +| [MCP tools](../src/Contoso.Mcp/Tools/CommerceTools.cs) | API-only search forwarding, structured filters, and tool descriptions. | +| [MCP tool tests](../aspire/Contoso.Test.Aspire/ProductSearchToolTests.cs) | Direct tool behavior, filter forwarding, validation, failures, and tool schema with a mocked Products API. | +| [MCP protocol E2E](../aspire/Contoso.Test.Aspire/E2ETest.Mcp.cs) | Tool discovery and structured outputs through real Streamable HTTP MCP and the full Aspire topology. | | [Compose infrastructure](../../docker-compose.yml) | Pinned PostgreSQL/pgvector image and optional local inference resource. | | [Search projection](../src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs) | Exact distances, readiness, atomic backfill and current-state reconciliation. | | [Product-event subscriber](../src/Contoso.Products.Subscribe/Subscribers/ProductSearchSubscriber.cs) | One globbed handler for all product lifecycle notifications. | diff --git a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs index 7f41bcf77..02729d518 100644 --- a/samples/src/Contoso.Mcp/Tools/CommerceTools.cs +++ b/samples/src/Contoso.Mcp/Tools/CommerceTools.cs @@ -117,10 +117,8 @@ public static Task AddBasketItemAsync( [Description("The quantity to add.")] decimal quantity, CancellationToken cancellationToken = default) => ShoppingApi(clientFactory) - .SendValue( - HttpMethod.Post, - $"/api/baskets/{Uri.EscapeDataString(basketId.Required())}/items", - new BasketItemAddRequest { ProductId = productId.Required(), Quantity = quantity }) + .Post($"/api/baskets/{Uri.EscapeDataString(basketId.Required())}/items") + .WithJson(new BasketItemAddRequest { ProductId = productId.Required(), Quantity = quantity }) .GetValueAsync(cancellationToken); /// Changes the quantity of a basket item. @@ -141,10 +139,8 @@ public static async Task UpdateBasketItemAsync( ?? throw new InvalidOperationException($"Basket item '{basketItemId}' was not found in basket '{basketId}'."); return await client - .SendValue( - HttpMethod.Put, - $"/api/baskets/{Uri.EscapeDataString(basketId)}/items/{Uri.EscapeDataString(basketItemId)}", - new BasketItemUpdateRequest { Quantity = quantity, ETag = item.ETag }) + .Put($"/api/baskets/{Uri.EscapeDataString(basketId)}/items/{Uri.EscapeDataString(basketItemId)}") + .WithJson(new BasketItemUpdateRequest { Quantity = quantity, ETag = item.ETag }) .GetValueAsync(cancellationToken).ConfigureAwait(false); } @@ -178,7 +174,8 @@ public static Task UpdateBasketShippingAddressAsync( Address shippingAddress, CancellationToken cancellationToken = default) => ShoppingApi(clientFactory) - .SendValue(HttpMethod.Put, $"/api/baskets/{Uri.EscapeDataString(basketId.Required())}/shipping-address", shippingAddress) + .Put($"/api/baskets/{Uri.EscapeDataString(basketId.Required())}/shipping-address") + .WithJson(shippingAddress) .GetValueAsync(cancellationToken); /// Checks out a basket and starts inventory reservation. diff --git a/samples/src/Contoso.Shopping.Infrastructure/Clients/Products/ProductsHttpClient.cs b/samples/src/Contoso.Shopping.Infrastructure/Clients/Products/ProductsHttpClient.cs index f973948c9..2eb240516 100644 --- a/samples/src/Contoso.Shopping.Infrastructure/Clients/Products/ProductsHttpClient.cs +++ b/samples/src/Contoso.Shopping.Infrastructure/Clients/Products/ProductsHttpClient.cs @@ -13,5 +13,5 @@ public class ProductsHttpClient(HttpClient httpClient) /// /// The . public Task CreateReservationAsync(MovementRequest request, CancellationToken ct = default) - => _httpClient.SendValue(HttpMethod.Post, "api/inventory/reserve", request).ToResultAsync(ct); + => _httpClient.Post("api/inventory/reserve").WithJson(request).ToResultAsync(ct); } diff --git a/samples/src/Contoso.Shopping.Infrastructure/Clients/SendGrid/SendGridHttpClient.cs b/samples/src/Contoso.Shopping.Infrastructure/Clients/SendGrid/SendGridHttpClient.cs index 3bdf499ac..7a61d01f4 100644 --- a/samples/src/Contoso.Shopping.Infrastructure/Clients/SendGrid/SendGridHttpClient.cs +++ b/samples/src/Contoso.Shopping.Infrastructure/Clients/SendGrid/SendGridHttpClient.cs @@ -14,5 +14,5 @@ public class SendGridHttpClient(HttpClient httpClient) /// The . /// A successful send returns (202) with no response body. public Task SendMailAsync(SendGridMailRequest request, CancellationToken ct = default) - => _httpClient.SendValue(HttpMethod.Post, "v3/mail/send", request).ToResultAsync(ct); + => _httpClient.Post("v3/mail/send").WithJson(request).ToResultAsync(ct); } diff --git a/src/CoreEx.Database/AGENTS.md b/src/CoreEx.Database/AGENTS.md index 833aa633c..6512363e1 100644 --- a/src/CoreEx.Database/AGENTS.md +++ b/src/CoreEx.Database/AGENTS.md @@ -4,21 +4,32 @@ Provides the `IDatabase` ADO.NET abstraction, `DatabaseCommand` fluent builder, ## IDatabase and DatabaseCommand -Use `DatabaseCommand` for all ADO.NET queries. Never write raw `DbCommand` code. +Use `DatabaseCommand` for all ADO.NET queries. Never write raw `DbCommand` code. Prefer `EfDb` for ordinary CRUD +and LINQ queries; put application-owned static SQL in an embedded `.sql` / `.pgsql` resource, regardless of length, +and load it once with `SqlStatement.FromResource`. ```csharp -// Single row -var order = await _db.SqlStatement("SELECT * FROM orders WHERE id = @id") - .Param("@id", id) +private static readonly SqlStatement _getOrder = + SqlStatement.FromResource("GetOrderById.sql"); + +var order = await _db.Statement(_getOrder) + .Param("OrderId", id) .SelectSingleAsync(OrderMapper.Default) .ConfigureAwait(false); -// Collection -var orders = await _db.StoredProcedure("spGetOrders") +// Stored procedure +var orders = await _db.Statement(SqlStatement.StoredProcedure("spGetOrders")) .SelectQueryAsync(OrderMapper.Default) .ConfigureAwait(false); ``` +Explicitly embed SQL files in the Infrastructure project's `.csproj` (for example, +``). Bind values with `.Param(...)`; never interpolate data +values into SQL. Resource lookup matches the end of the fully qualified name, so keep resource filenames unique +within the assembly. Use `SqlStatement.FromText(...)` only when SQL must genuinely be generated at runtime (for +example, when an identifier is dynamic); bind data values as parameters and validate or safely quote dynamic +identifiers. Schema changes belong in the `*.Database` project's DbEx migrations, not in runtime SQL resources. + ## Explicit Mappers Extend `DatabaseMapper` for each entity. Hand-write column assignments — do not use reflection or AutoMapper. @@ -70,6 +81,7 @@ Service Bus session mapping is a separate boundary. Coordinate destination polic - Do not write raw `DbCommand` or `SqlCommand` code — use `DatabaseCommand`. - Do not use reflection-based mappers — extend `DatabaseMapper` with explicit column reads. +- Do not inline application-owned static SQL in C# — keep it in an embedded resource and load it with `SqlStatement.FromResource`. Reserve `SqlStatement.FromText(...)` for genuinely dynamic SQL. ## Further Reading diff --git a/src/CoreEx.Database/README.md b/src/CoreEx.Database/README.md index 2cb55ec42..6b9d306aa 100644 --- a/src/CoreEx.Database/README.md +++ b/src/CoreEx.Database/README.md @@ -21,7 +21,7 @@ The outbox sub-namespace implements the [Transactional Outbox pattern](https://m - 🦁 **Database wildcard**: `DatabaseWildcard` translates CoreEx `WildcardResult` selections into database `LIKE` patterns (`%`, `_`) with configurable escape character. - 📡 **DatabaseInvoker tracing**: Every command execution is wrapped by `DatabaseInvoker` (an `InvokerBase`) emitting OpenTelemetry spans tagged with command text, operation result, and error details. - 📤 **Transactional outbox relay**: `DatabaseOutboxRelayBase` polls an outbox table, deserializes events, publishes them via `IEventPublisher`, and marks them as sent — all within a configurable polling hosted service. -- 🔢 **SQL statement abstraction**: `SqlStatement` carries `CommandText`, `CommandType` (`Text` or `StoredProcedure`), and convenience factory methods for inline SQL and stored procedure calls. +- 🔢 **SQL statement abstraction**: `SqlStatement` carries `CommandText`, `CommandType` (`Text` or `StoredProcedure`), and factory methods for inline SQL, embedded SQL resources, and stored procedure calls. ## Key types @@ -32,7 +32,7 @@ The outbox sub-namespace implements the [Transactional Outbox pattern](https://m | **[`DatabaseCommand`](./DatabaseCommand.cs)** | Fluent command builder: `Parameters` (`DatabaseParameterCollection`), `NonQueryAsync`, `ScalarAsync`, `SelectSingleAsync`, `SelectFirstOrDefaultAsync`, `SelectQueryAsync`, `SelectMultiSetAsync`. | | **[`DatabaseRecord`](./DatabaseRecord.cs)** | Wraps `DbDataReader`: `GetValue(string/int)`, `GetRowVersion()`, ordinal caching, and null-safe typed reads with `DateTimeTransform` applied. | | **[`DatabaseParameterCollection`](./DatabaseParameterCollection.cs)** | `DbParameterCollection` wrapper with `AddParameter`, `AddRowVersionParameter`, `AddReturnValueParameter`, and batch helpers; fluent via `IDatabaseParameters`. | -| **[`SqlStatement`](./SqlStatement.cs)** | Carries `CommandText` and `CommandType`; factory methods `StoredProcedure(name)`, `FromText(text)` (in-line) and `FromResource(resourceName)` (embedded resource). | +| **[`SqlStatement`](./SqlStatement.cs)** | Carries `CommandText` and `CommandType`; factory methods `StoredProcedure(name)`, `FromResource(resourceName)` (embedded resource), and `FromText(text)` (runtime-generated SQL). | | **[`DatabaseArgs`](./Abstractions/DatabaseArgs.cs)** | Per-operation options: transaction `IsolationLevel`, `Refresh`, and `TransformException` behavior overrides. | | **[`DatabaseInvoker`](./Abstractions/DatabaseInvoker.cs)** | `InvokerBase` emitting OpenTelemetry spans and structured log entries for every command execution. | | **[`IDatabaseMapper`](./Mapping/IDatabaseMapperT.cs)** | Bidirectional mapper interface: `MapFromDb(DatabaseRecord, OperationType)` → `T`; `MapToDb(T, DatabaseParameterCollection, OperationType)`. | @@ -64,6 +64,34 @@ and is not a substitute for SQL Server `Snapshot`. Read-only callbacks need not enqueue events. At the root boundary, a configured outbox publishes any queued events before commit; nested calls leave publication to their parent. +## SQL statement source + +Prefer the EF Core / `EfDb` model APIs for ordinary CRUD and LINQ queries. When application code needs custom +relational SQL, put the static SQL in a named `.sql` or `.pgsql` file and embed it in the Infrastructure assembly, +regardless of its length. Load it once with `SqlStatement.FromResource` and execute it through +`IDatabase.Statement`; keeping SQL in a discoverable resource makes it easier to review and extend without mixing +it into orchestration code. + +```csharp +private static readonly SqlStatement _isReady = + SqlStatement.FromResource("ProductSearchIsReady.pgsql"); + +var ready = await _ef.Database.Statement(_isReady) + .Param("Profile", profile, dbType: DbType.String) + .ScalarAsync(cancellationToken) + .ConfigureAwait(false); +``` + +The Infrastructure project must explicitly embed the files (for example, +``). Resource lookup matches the end of the fully qualified +resource name, so keep filenames unique within the assembly. Bind values with `DatabaseCommand.Param`; do not +interpolate user or data values into SQL. Use `SqlStatement.FromText` only when SQL must genuinely be generated at +runtime, such as when a validated or safely quoted identifier is dynamic; continue to bind data values as +parameters. Keep schema changes in the `*.Database` project's DbEx migrations, not in runtime query resources. +The Products search implementation is a complete example +([`ProductSearchIndex`](https://github.com/Avanade/CoreEx/blob/main/samples/src/Contoso.Products.Infrastructure/Repositories/ProductSearchIndex.cs), +[search guide](https://github.com/Avanade/CoreEx/blob/main/samples/docs/search.md#postgresql-and-pgvector)). + ## Namespaces | Namespace | Description | diff --git a/src/CoreEx/AGENTS.md b/src/CoreEx/AGENTS.md index bf895f604..74b61329d 100644 --- a/src/CoreEx/AGENTS.md +++ b/src/CoreEx/AGENTS.md @@ -55,7 +55,9 @@ return await GetOrderAsync(id) ## HTTP Request Builder and Extensions -Use `client.Send(...)` or `client.SendValue(...)` to build, configure, and execute HTTP requests fluently. Terminal methods (`GetValueAsync`, `GetValueOrDefaultAsync`, `GetStringAsync`, `ToResultAsync`, `ToProblemDetailsAsync`, `ThrowOnErrorAsync`) execute the request and automatically dispose the underlying `HttpResponseMessage`. +Use `client.Send(method, uri)` or the `Get`, `Post`, `Put`, `Patch`, and `Delete` shortcuts to build HTTP requests. Add a JSON body with `WithJson(value)` (CoreEx serializer defaults; null sends JSON null), or explicit `HttpContent` with `WithContent(content)`. Exactly one body assignment is allowed; duplicates or mixing the two throw `InvalidOperationException`. `Patch` does not imply JSON Merge Patch; select it explicitly with `WithMergePatchJsonContentType()`. + +Builders are single-use: the first terminal call consumes the builder, even on failure or cancellation. Repeated/concurrent execution or subsequent configuration throws `InvalidOperationException`; configure before sending. `Configure` cannot replace/remove an assigned body. Accepted explicit content is disposed during execution; the caller owns rejected content or content on a builder that is never executed. Terminal methods (`GetValueAsync`, `GetValueOrDefaultAsync`, `GetStringAsync`, `ToResultAsync`, `ToProblemDetailsAsync`, `ThrowOnErrorAsync`) automatically dispose the response; callers own raw responses from `GetResponseAsync`/`SendAsync`. ```csharp // Fluent request with query filter, paging, and typed response deserialization + disposal @@ -67,7 +69,8 @@ var products = await _httpClient // POST JSON body and map response to CoreEx Result with automatic disposal return await _httpClient - .SendValue(HttpMethod.Post, "api/inventory/reserve", request) + .Post("api/inventory/reserve") + .WithJson(request) .ToResultAsync(cancellationToken).ConfigureAwait(false); // Intercept specific HTTP status codes (e.g. 404 -> Result.NotFoundError) diff --git a/src/CoreEx/Extensions.HttpClient.cs b/src/CoreEx/Extensions.HttpClient.cs index 3668f706a..53331158a 100644 --- a/src/CoreEx/Extensions.HttpClient.cs +++ b/src/CoreEx/Extensions.HttpClient.cs @@ -3,163 +3,72 @@ namespace CoreEx; public static partial class Extensions { /// - /// Creates an for the specified and . + /// Creates a single-use for the specified method and URI; no request is sent until a terminal method is called. /// - /// The . - /// The . + /// The HTTP client. + /// The HTTP method. /// The request URI. - /// An . + /// The request builder. public static HttpRequestBuilder Send(this HttpClient client, HttpMethod method, string? requestUri) => Send(client, method, requestUri, null); /// - /// Creates an for the specified and . + /// Creates a single-use with optional low-level request configuration. /// - /// The . - /// The . + /// The HTTP client. + /// The HTTP method. /// The request URI. - /// An optional action to configure the . - /// An . + /// An optional action to configure the request before sending. + /// The request builder. public static HttpRequestBuilder Send(this HttpClient client, HttpMethod method, string? requestUri, Action? configureRequest) => new(client, method, requestUri, configureRequest); - /// - /// Creates an for the specified and , serializing the specified to JSON. - /// - /// The value . - /// The . - /// The . + /// Creates a single-use GET request builder without sending the request. + /// The HTTP client. /// The request URI. - /// The value to serialize to JSON. - /// An . - public static HttpRequestBuilder SendValue(this HttpClient client, HttpMethod method, string? requestUri, T value) - => SendValue(client, method, requestUri, value, null, null, null); + /// The request builder. + public static HttpRequestBuilder Get(this HttpClient client, string? requestUri) => client.Send(HttpMethod.Get, requestUri); - /// - /// Creates an for the specified and , serializing the specified to JSON. - /// - /// The value . - /// The . - /// The . + /// Creates a single-use POST request builder without sending the request. + /// The HTTP client. /// The request URI. - /// The value to serialize to JSON. - /// An optional action to configure the . - /// An . - public static HttpRequestBuilder SendValue(this HttpClient client, HttpMethod method, string? requestUri, T value, Action? configureRequest) - => SendValue(client, method, requestUri, value, null, configureRequest, null); + /// The request builder. + public static HttpRequestBuilder Post(this HttpClient client, string? requestUri) => client.Send(HttpMethod.Post, requestUri); - /// - /// Creates an for the specified and , serializing the specified to JSON. - /// - /// The value . - /// The . - /// The . + /// Creates a single-use PUT request builder without sending the request. + /// The HTTP client. /// The request URI. - /// The value to serialize to JSON. - /// The media type. - /// An optional action to configure the . - /// An . - public static HttpRequestBuilder SendValue(this HttpClient client, HttpMethod method, string? requestUri, T value, string? mediaType, Action? configureRequest) - => SendValue(client, method, requestUri, value, mediaType, configureRequest, null); + /// The request builder. + public static HttpRequestBuilder Put(this HttpClient client, string? requestUri) => client.Send(HttpMethod.Put, requestUri); - /// - /// Creates an for the specified and , serializing the specified to JSON. - /// - /// The value . - /// The . - /// The . + /// Creates a single-use PATCH request builder without choosing a body format or sending the request. + /// The HTTP client. /// The request URI. - /// The value to serialize to JSON. - /// The media type. - /// An optional action to configure the . - /// An optional to use when serializing the value to JSON; defaults to where not specified. - /// An . - public static HttpRequestBuilder SendValue(this HttpClient client, HttpMethod method, string? requestUri, T value, string? mediaType = null, Action? configureRequest = null, JsonSerializerOptions? jsonSerializerOptions = null) - => Send(client, method, requestUri, request => - { - mediaType ??= MediaTypeNames.Application.Json; - request.Content = JsonContent.Create(value, new MediaTypeHeaderValue(mediaType), options: jsonSerializerOptions ?? JsonDefaults.SerializerOptions); - configureRequest?.Invoke(request); - }); + /// The request builder. + public static HttpRequestBuilder Patch(this HttpClient client, string? requestUri) => client.Send(HttpMethod.Patch, requestUri); - /// - /// Sends an HTTP request with the specified and and returns the response. - /// - /// The . - /// The . + /// Creates a single-use DELETE request builder without sending the request. + /// The HTTP client. + /// The request URI. + /// The request builder. + public static HttpRequestBuilder Delete(this HttpClient client, string? requestUri) => client.Send(HttpMethod.Delete, requestUri); + + /// Sends a request and returns the response, which the caller must dispose. + /// The HTTP client. + /// The HTTP method. /// The request URI. - /// A cancellation token. - /// The . + /// The cancellation token. + /// The response. public static Task SendAsync(this HttpClient client, HttpMethod method, string? requestUri, CancellationToken cancellationToken = default) => SendAsync(client, method, requestUri, null, cancellationToken); - /// - /// Sends an HTTP request with the specified and and returns the response. - /// - /// The . - /// The . + /// Sends a request with optional configuration and returns the response, which the caller must dispose. + /// The HTTP client. + /// The HTTP method. /// The request URI. - /// An optional action to configure the . - /// A cancellation token. - /// The . + /// An optional action to configure the request. + /// The cancellation token. + /// The response. public static Task SendAsync(this HttpClient client, HttpMethod method, string? requestUri, Action? configureRequest, CancellationToken cancellationToken = default) => Send(client, method, requestUri, configureRequest).GetResponseAsync(cancellationToken); - - /// - /// Sends an HTTP request with the specified and and returns the response, serializing the specified to JSON. - /// - /// The value . - /// The . - /// The . - /// The request URI. - /// The value to serialize to JSON. - /// A cancellation token. - /// The . - public static Task SendValueAsync(this HttpClient client, HttpMethod method, string? requestUri, T value, CancellationToken cancellationToken = default) - => SendValueAsync(client, method, requestUri, value, null, null, null, cancellationToken); - - /// - /// Sends an HTTP request with the specified and and returns the response, serializing the specified to JSON. - /// - /// The value . - /// The . - /// The . - /// The request URI. - /// The value to serialize to JSON. - /// An optional action to configure the . - /// A cancellation token. - /// The . - public static Task SendValueAsync(this HttpClient client, HttpMethod method, string? requestUri, T value, Action? configureRequest, CancellationToken cancellationToken = default) - => SendValueAsync(client, method, requestUri, value, null, configureRequest, null, cancellationToken); - - /// - /// Sends an HTTP request with the specified and and returns the response, serializing the specified to JSON. - /// - /// The value . - /// The . - /// The . - /// The request URI. - /// The value to serialize to JSON. - /// The media type. - /// An optional action to configure the . - /// A cancellation token. - /// The . - public static Task SendValueAsync(this HttpClient client, HttpMethod method, string? requestUri, T value, string? mediaType, Action? configureRequest, CancellationToken cancellationToken = default) - => SendValueAsync(client, method, requestUri, value, mediaType, configureRequest, null, cancellationToken); - - /// - /// Sends an HTTP request with the specified and and returns the response, serializing the specified to JSON. - /// - /// The value . - /// The . - /// The . - /// The request URI. - /// The value to serialize to JSON. - /// The media type. - /// An optional action to configure the . - /// An optional to use when serializing the value to JSON; defaults to where not specified. - /// A cancellation token. - /// The . - public static Task SendValueAsync(this HttpClient client, HttpMethod method, string? requestUri, T value, string? mediaType, Action? configureRequest = null, JsonSerializerOptions? jsonSerializerOptions = null, CancellationToken cancellationToken = default) - => SendValue(client, method, requestUri, value, mediaType, configureRequest, jsonSerializerOptions).GetResponseAsync(cancellationToken); } diff --git a/src/CoreEx/Extensions.HttpRequestMessage.cs b/src/CoreEx/Extensions.HttpRequestMessage.cs index 1b835a74c..ebe17a486 100644 --- a/src/CoreEx/Extensions.HttpRequestMessage.cs +++ b/src/CoreEx/Extensions.HttpRequestMessage.cs @@ -8,6 +8,37 @@ namespace CoreEx; public static partial class Extensions { + /// Sets the body to the JSON representation of the value, including JSON null. Rejects an existing body. + /// The value type. + /// The HTTP request. + /// The value to serialize. + /// The media type; defaults to application/json. + /// The serializer options; defaults to . + /// The request for fluent chaining. + public static HttpRequestMessage WithJson(this HttpRequestMessage request, T value, string? mediaType = null, JsonSerializerOptions? jsonSerializerOptions = null) + { + request.ThrowIfNull(); + if (request.Content is not null) + throw new InvalidOperationException("The HTTP request body has already been assigned."); + + request.Content = JsonContent.Create(value, new MediaTypeHeaderValue(mediaType ?? MediaTypeNames.Application.Json), options: jsonSerializerOptions ?? JsonDefaults.SerializerOptions); + return request; + } + + /// Sets the body, rejecting an existing body. The request owns and disposes accepted content. + /// The HTTP request. + /// The content; the caller retains ownership if assignment is rejected. + /// The request for fluent chaining. + public static HttpRequestMessage WithContent(this HttpRequestMessage request, HttpContent content) + { + request.ThrowIfNull(); + if (request.Content is not null) + throw new InvalidOperationException("The HTTP request body has already been assigned."); + + request.Content = content.ThrowIfNull(); + return request; + } + /// /// Adds the with the specified (where no ). /// diff --git a/src/CoreEx/Http/HttpRequestBuilder.cs b/src/CoreEx/Http/HttpRequestBuilder.cs index fbec4dc7d..c0bba61a4 100644 --- a/src/CoreEx/Http/HttpRequestBuilder.cs +++ b/src/CoreEx/Http/HttpRequestBuilder.cs @@ -7,8 +7,12 @@ namespace CoreEx.Http; /// -/// Builds and executes an HTTP request, providing fluent terminal methods that handle response consumption and disposal. +/// Builds and executes a single HTTP request, providing fluent terminal methods that handle response consumption and disposal. /// +/// +/// Configure before execution; concurrent configuration is not supported. The first execution attempt consumes the builder even on failure or cancellation. +/// Subsequent execution or configuration throws . HTTP resilience retries remain part of the same execution. +/// /// The . /// The . /// The request URI. @@ -18,8 +22,10 @@ public sealed class HttpRequestBuilder(HttpClient client, HttpMethod method, str private readonly HttpClient _client = client.ThrowIfNull(); private readonly HttpMethod _method = method.ThrowIfNull(); private readonly string? _requestUri = requestUri; - private Action? _configureRequest = configureRequest; + private readonly List> _configureRequests = configureRequest is null ? [] : [configureRequest]; private Dictionary>>? _statusResults; + private Func? _createContent; + private int _started; /// /// Gets the . @@ -43,12 +49,73 @@ public sealed class HttpRequestBuilder(HttpClient client, HttpMethod method, str /// The instance to support fluent chaining. public HttpRequestBuilder Configure(Action configureRequest) { + EnsureNotStarted(); configureRequest.ThrowIfNull(); - var existing = _configureRequest; - _configureRequest = existing is null ? configureRequest : (r => { existing(r); configureRequest(r); }); + _configureRequests.Add(configureRequest); return this; } + /// + /// Sets the body to the JSON representation of the value, including JSON null. Only one body may be assigned. + /// + /// The value type. + /// The value to serialize. + /// The media type; defaults to . + /// The serializer options; defaults to . + /// The builder for fluent chaining. + public HttpRequestBuilder WithJson(T value, string? mediaType = null, JsonSerializerOptions? jsonSerializerOptions = null) + { + EnsureBodyNotAssigned(); + var contentType = new MediaTypeHeaderValue(mediaType ?? MediaTypeNames.Application.Json); + _createContent = () => JsonContent.Create(value, contentType, options: jsonSerializerOptions ?? JsonDefaults.SerializerOptions); + return this; + } + + /// + /// Sets the request body. Only one body may be assigned. + /// + /// The content, disposed with the request when executed. The caller must dispose it if the builder is never executed or assignment is rejected. + /// The builder for fluent chaining. + public HttpRequestBuilder WithContent(HttpContent content) + { + EnsureBodyNotAssigned(); + content.ThrowIfNull(); + _createContent = () => content; + return this; + } + + /// Rejects configuration after execution has started. + private void EnsureNotStarted() + { + if (Volatile.Read(ref _started) != 0) + throw new InvalidOperationException("The HTTP request builder has already been consumed."); + } + + /// Rejects a second body assignment. + private void EnsureBodyNotAssigned() + { + EnsureNotStarted(); + if (_createContent is not null) + throw new InvalidOperationException("The HTTP request body has already been assigned."); + } + + /// Applies low-level configuration without allowing an existing body to be replaced. + private static void ApplyConfiguration(HttpRequestMessage request, Action configureRequest) + { + var content = request.Content; + try + { + configureRequest(request); + if (content is not null && !ReferenceEquals(content, request.Content)) + throw new InvalidOperationException("Request configuration cannot replace an assigned HTTP request body."); + } + finally + { + if (content is not null && !ReferenceEquals(content, request.Content)) + content.Dispose(); + } + } + /// /// Adds the header with the specified (where not ). /// @@ -150,6 +217,7 @@ public HttpRequestBuilder Configure(Action configureRequest) /// The instance to support fluent chaining. public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Result result) { + EnsureNotStarted(); _statusResults ??= []; _statusResults[statusCode] = (_, _) => Task.FromResult(result); return this; @@ -163,6 +231,7 @@ public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Result res /// The instance to support fluent chaining. public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Func result) { + EnsureNotStarted(); result.ThrowIfNull(); _statusResults ??= []; _statusResults[statusCode] = (_, _) => Task.FromResult(result()); @@ -177,6 +246,7 @@ public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, FuncThe instance to support fluent chaining. public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Func result) { + EnsureNotStarted(); result.ThrowIfNull(); _statusResults ??= []; _statusResults[statusCode] = (response, _) => Task.FromResult(result(response)); @@ -191,6 +261,7 @@ public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, FuncThe instance to support fluent chaining. public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, Func> result) { + EnsureNotStarted(); result.ThrowIfNull(); _statusResults ??= []; _statusResults[statusCode] = result; @@ -235,11 +306,17 @@ public HttpRequestBuilder WithStatusResult(HttpStatusCode statusCode, FuncThe . public async Task GetResponseAsync(CancellationToken cancellationToken = default) { + if (Interlocked.Exchange(ref _started, 1) != 0) + throw new InvalidOperationException("The HTTP request builder has already been consumed."); + using var request = new HttpRequestMessage(_method, _requestUri); + request.Content = _createContent?.Invoke(); if (_client.BaseAddress is not null && request.RequestUri?.IsAbsoluteUri != true) request.RequestUri = new Uri(_client.BaseAddress, _requestUri ?? string.Empty); - _configureRequest?.Invoke(request); + foreach (var configure in _configureRequests) + ApplyConfiguration(request, configure); + return await _client.SendAsync(request, cancellationToken).ConfigureAwait(false); } @@ -458,5 +535,9 @@ public async Task GetValueAsync(JsonSerializerOptions? jsonSerializerOptio /// The response value type. /// The response value validator. /// A typed wrapper that validates when a value is consumed. - public ValidatedHttpRequestBuilder WithValidator(Validation.IValidator validator) => new(this, validator); + public ValidatedHttpRequestBuilder WithValidator(Validation.IValidator validator) + { + EnsureNotStarted(); + return new(this, validator); + } } diff --git a/src/CoreEx/Http/README.md b/src/CoreEx/Http/README.md index 6af16df89..c14945d91 100644 --- a/src/CoreEx/Http/README.md +++ b/src/CoreEx/Http/README.md @@ -25,7 +25,7 @@ The `Abstractions` sub-namespace defines additional lightweight types and interf | **[`ProblemDetails`](./Abstractions/ProblemDetails.cs)** | RFC 7807 problem-details record with `Type`, `Title`, `Status`, `Detail`, `Instance`, and an `Extensions` dictionary; includes `GetValidationErrors(string? key)`. | | **[`ProblemDetailsException`](./ProblemDetailsException.cs)** | Exception wrapping a `ProblemDetails`; provides `ToException()` to convert it back to a typed CoreEx semantic exception. | | **[`HttpNames`](./HttpNames.cs)** | Static class of configurable string properties for all standard paging query-string names, response-header names, and the idempotency-key header name. | -| **[`HttpRequestBuilder`](./HttpRequestBuilder.cs)** | Fluent HTTP request builder returned by `client.Send(...)` and `client.SendValue(...)`; executes requests and manages automatic response disposal across typed deserialization, ROP result conversion, validation, and error-handling methods. | +| **[`HttpRequestBuilder`](./HttpRequestBuilder.cs)** | Single-use fluent HTTP request builder returned by `client.Send(...)` or the verb shortcuts; executes requests and manages automatic response disposal across typed deserialization, ROP result conversion, validation, and error-handling methods. | | **[`ValidatedHttpRequestBuilder`](./ValidatedHttpRequestBuilderT.cs)** | Wrapper returned by `builder.WithValidator(validator)` that executes the request, validates the deserialized response value, and automatically disposes the response. | | **[`ValidatedHttpResponse`](./ValidatedHttpResponseT.cs)** | Thin wrapper returned by `HttpResponseMessage.WithValidator(validator)`; validates successful non-null response values when consumed via `ToResult*` or `GetValue*`. Does not own the response. | | **[`HttpResult`](./HttpResult.cs)** | Readonly struct pairing an `HttpResponseMessage` with either a deserialized value or a `ProblemDetails` from the response; exposes `IsSuccess`, `StatusCode`, and `ThrowOnError()`. | @@ -34,29 +34,30 @@ The `Abstractions` sub-namespace defines additional lightweight types and interf ## Sending requests -The `CoreEx` namespace provides fluent synchronous builder methods `HttpClient.Send(...)` and `SendValue(...)` -returning an `HttpRequestBuilder`, as well as direct asynchronous `SendAsync` and `SendValueAsync` overloads. +The `CoreEx` namespace provides `HttpClient.Send(method, uri)` and the `Get`, `Post`, `Put`, `Patch`, and `Delete` +shortcuts returning an `HttpRequestBuilder`, as well as direct asynchronous `SendAsync` overloads. Relative URIs are resolved against `HttpClient.BaseAddress` before the configuration callback, so `WithQuery`, `WithPaging`, and header helpers can be applied there. Absolute URIs do not require a base address. ### Fluent request execution and automatic response disposal -`client.Send(...)` and `client.SendValue(...)` return an `HttpRequestBuilder`. Calling any terminal consumption +`client.Send(...)` and the verb shortcuts only build a request. Calling any terminal consumption method executes the request and automatically disposes the underlying `HttpResponseMessage`: ```csharp // Single await with automatic response disposal: -var product = await client.Send(HttpMethod.Get, "api/products/123") +var product = await client.Get("api/products/123") .GetValueAsync(cancellationToken).ConfigureAwait(false); // Fluent query, paging, and headers directly chained: -var products = await client.Send(HttpMethod.Get, "api/products") +var products = await client.Get("api/products") .WithQuery("startswith(sku, 'ABC')") .WithPaging(take: 10) .GetValueAsync(cancellationToken).ConfigureAwait(false); // Post JSON payload with typed response: -var created = await client.SendValue(HttpMethod.Post, "api/baskets", newBasket) +var created = await client.Post("api/baskets") + .WithJson(newBasket) .GetValueAsync(cancellationToken).ConfigureAwait(false); // With custom status mapping (short-circuits validation/deserialization on match): @@ -75,9 +76,33 @@ using var response = await client.Send(HttpMethod.Get, "api/products") .GetResponseAsync(cancellationToken).ConfigureAwait(false); ``` +### Request bodies and single-use execution + +Use `WithJson(value, mediaType: ..., jsonSerializerOptions: ...)` for a JSON body. It uses +`JsonDefaults.SerializerOptions` and `application/json` by default. A null value sends JSON `null`; +a string sends a JSON string, not raw text. For explicit raw text, form data, multipart content, or +streams, use `WithContent(HttpContent)`. Matching helpers are available on `HttpRequestMessage`. + +Only one body may be assigned: repeated `WithJson`, repeated `WithContent`, or mixing the two throws +`InvalidOperationException`. Low-level `Configure(...)` callbacks may supply a body only when no body +has already been assigned; replacing or removing an existing body is rejected before sending. +Callbacks may still modify content headers. `Patch(uri)` only selects the HTTP verb; use +`.WithJson(patch).WithMergePatchJsonContentType()` to explicitly select JSON Merge Patch. + +Each builder allows exactly one execution attempt, even if it fails or is cancelled. A second or +concurrent terminal call, or configuration after execution begins, throws `InvalidOperationException`. +Configure the builder before executing it; concurrent configuration is not supported. The validated +builder shares the same single-use request. Retries within the HTTP resilience pipeline are not new +builder executions. + +Accepted `WithContent` content is disposed with the request during execution, including failures. +The caller must dispose content if assignment is rejected or the builder is never executed. +JSON content is created when execution begins. Terminal consumption methods dispose the response; +`GetResponseAsync` and `SendAsync` return a raw response that the caller must dispose. + ### Direct asynchronous methods -For direct response inspection, `SendAsync` and `SendValueAsync` return the raw `HttpResponseMessage` (which the caller must dispose): +For direct response inspection, `SendAsync` returns the raw `HttpResponseMessage` (which the caller must dispose): ```csharp using var response = await client.SendAsync(HttpMethod.Get, "api/products", diff --git a/src/CoreEx/README.md b/src/CoreEx/README.md index 7fa59093e..0fb362f62 100644 --- a/src/CoreEx/README.md +++ b/src/CoreEx/README.md @@ -33,7 +33,7 @@ All other CoreEx packages (`CoreEx.Events`, `CoreEx.Database`, `CoreEx.AspNetCor - 🃏 **Wildcard parsing**: `Wildcard` provides standardized `*` and `?` wildcard parsing, validation, and `Regex` compilation for consistent wildcard-based filtering. - 🏷️ **Reference data interfaces**: Core `IReferenceData`, `IReferenceDataCollection`, and `ReferenceDataOrchestrator` types establish the reference data contract used by `CoreEx.RefData` and data-layer packages. - 📐 **Schema versioning**: `SchemaAttribute` and `Schema` utilities for associating a semantic version with entity types, supporting schema-aware messaging and validation. -- 🌐 **HTTP client and request builder**: `client.Send(...)` and `client.SendValue(...)` return an `HttpRequestBuilder` providing fluent request setup (`WithQuery`, `WithPaging`, headers) and terminal execution with automatic response disposal (`GetValueAsync`, `ToResultAsync`, `GetStringAsync`, etc.). `builder.AddNamedHttpClient` / `AddTypedHttpClient` configure service discovery, resilience, and idempotency key handling. For SDK-owned client names, `builder.AddNamedHttpClient("Sdk_httpClient", configureClient: null, configurationSectionName: "Service")` augments the SDK registration using `Service:BaseAddress` and `Service:Resilience`; existing overloads continue using the client name as the configuration section. +- 🌐 **HTTP client and request builder**: `client.Send(method, uri)` or `Get`/`Post`/`Put`/`Patch`/`Delete` return a single-use `HttpRequestBuilder`. Add exactly one body via `WithJson(value)` (JSON) or `WithContent(content)`, configure queries/paging/headers, then execute a terminal method (`GetValueAsync`, `ToResultAsync`, `GetStringAsync`, etc.) with automatic response disposal except when returning the raw response. Reuse, configuration after execution begins, or duplicate body assignment throws `InvalidOperationException`. `Patch` does not imply JSON Merge Patch. See the [HTTP guide](./Http/README.md#request-bodies-and-single-use-execution) for ownership and execution semantics. `builder.AddNamedHttpClient` / `AddTypedHttpClient` configure service discovery, resilience, and idempotency key handling. For SDK-owned client names, `builder.AddNamedHttpClient("Sdk_httpClient", configureClient: null, configurationSectionName: "Service")` augments the SDK registration using `Service:BaseAddress` and `Service:Resilience`; existing overloads continue using the client name as the configuration section. ## Key types diff --git a/tests/CoreEx.Test.Unit/GlobalUsing.cs b/tests/CoreEx.Test.Unit/GlobalUsing.cs index 63a474677..fbee4f0db 100644 --- a/tests/CoreEx.Test.Unit/GlobalUsing.cs +++ b/tests/CoreEx.Test.Unit/GlobalUsing.cs @@ -1,3 +1,9 @@ +global using CoreEx.Data; +global using CoreEx.Http; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Hosting; +global using System.Net; +global using System.Net.Http.Headers; +global using System.Text; +global using System.Text.Json; diff --git a/tests/CoreEx.Test.Unit/Http/HttpClientExtensionsTests.cs b/tests/CoreEx.Test.Unit/Http/HttpClientExtensionsTests.cs index 2078b915e..3940206d8 100644 --- a/tests/CoreEx.Test.Unit/Http/HttpClientExtensionsTests.cs +++ b/tests/CoreEx.Test.Unit/Http/HttpClientExtensionsTests.cs @@ -32,7 +32,7 @@ public async Task SendAsync_AbsoluteUriDoesNotRequireBaseAddress() [TestCase(null, "application/json")] [TestCase("application/merge-patch+json", "application/merge-patch+json")] - public async Task SendValue_PreservesContentUntilCompletionAndUsesCoreExJson(string? mediaType, string expectedMediaType) + public async Task WithJson_PreservesContentUntilCompletionAndUsesCoreExJson(string? mediaType, string expectedMediaType) { var payload = new { ProductId = "123", Quantity = 2 }; using var handler = new InspectHandler(async (request, ct) => @@ -44,7 +44,7 @@ public async Task SendValue_PreservesContentUntilCompletionAndUsesCoreExJson(str return new HttpResponseMessage(System.Net.HttpStatusCode.OK); }); using var client = new HttpClient(handler) { BaseAddress = new Uri("https://example.com/") }; - using var response = await client.SendValueAsync(HttpMethod.Post, "/api", payload, mediaType: mediaType); + using var response = await client.Post("/api").WithJson(payload, mediaType: mediaType).SendAsync(); response.StatusCode.Should().Be(System.Net.HttpStatusCode.OK); } diff --git a/tests/CoreEx.Test.Unit/Http/HttpRequestBodyTests.cs b/tests/CoreEx.Test.Unit/Http/HttpRequestBodyTests.cs new file mode 100644 index 000000000..646eea8ff --- /dev/null +++ b/tests/CoreEx.Test.Unit/Http/HttpRequestBodyTests.cs @@ -0,0 +1,327 @@ +namespace CoreEx.Test.Unit.Http; + +[TestFixture] +public class HttpRequestBodyTests +{ + [TestCase("GET")] + [TestCase("POST")] + [TestCase("PUT")] + [TestCase("PATCH")] + [TestCase("DELETE")] + public async Task VerbShortcuts_BuildWithoutSending_AndDoNotImplyContent(string verb) + { + var calls = 0; + using var handler = new InspectHandler((request, _) => + { + calls++; + request.Method.Method.Should().Be(verb); + request.RequestUri.Should().Be(new Uri("https://example.com/api")); + request.Content.Should().BeNull(); + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK)); + }); + using var client = CreateClient(handler); + var builder = verb switch + { + "GET" => client.Get("/api"), + "POST" => client.Post("/api"), + "PUT" => client.Put("/api"), + "PATCH" => client.Patch("/api"), + _ => client.Delete("/api") + }; + + calls.Should().Be(0); + using var response = await builder.SendAsync(); + calls.Should().Be(1); + } + + [TestCase(null, "null")] + [TestCase("hello", "\"hello\"")] + public async Task WithJson_SerializesNullAndStringsAsJson(string? value, string expected) + { + using var handler = new InspectHandler(async (request, ct) => + { + (await request.Content!.ReadAsStringAsync(ct)).Should().Be(expected); + request.Content.Headers.ContentType!.MediaType.Should().Be("application/json"); + return new HttpResponseMessage(System.Net.HttpStatusCode.OK); + }); + using var client = CreateClient(handler); + using var response = await client.Post("/api").WithJson(value).SendAsync(); + } + + [Test] + public async Task WithJson_UsesExplicitSerializerOptionsAndMergePatchIsOptIn() + { + using var handler = new InspectHandler(async (request, ct) => + { + (await request.Content!.ReadAsStringAsync(ct)).Should().Be("{\"Name\":\"Widget\"}"); + request.Content.Headers.ContentType!.MediaType.Should().Be("application/merge-patch+json"); + return new HttpResponseMessage(System.Net.HttpStatusCode.OK); + }); + using var client = CreateClient(handler); + using var response = await client.Patch("/api") + .WithJson(new { Name = "Widget" }, jsonSerializerOptions: new System.Text.Json.JsonSerializerOptions()) + .WithMergePatchJsonContentType() + .SendAsync(); + } + + [TestCase(false, false)] + [TestCase(false, true)] + [TestCase(true, false)] + [TestCase(true, true)] + public void Body_CanOnlyBeAssignedOnce(bool firstIsContent, bool secondIsContent) + { + using var client = new HttpClient(); + using var first = new TrackedContent(); + using var second = new TrackedContent(); + var builder = client.Post("/api"); + if (firstIsContent) + builder.WithContent(first); + else + builder.WithJson(null); + + Action assign = () => + { + if (secondIsContent) + builder.WithContent(second); + else + builder.WithJson("second"); + }; + + assign.Should().Throw(); + first.IsDisposed.Should().BeFalse(); + second.IsDisposed.Should().BeFalse(); + } + + [TestCase(false, false)] + [TestCase(false, true)] + [TestCase(true, false)] + [TestCase(true, true)] + public void RequestMessage_BodyExtensionsRejectEveryDuplicateCombination(bool firstIsContent, bool secondIsContent) + { + using var request = new HttpRequestMessage(); + using var first = new TrackedContent(); + using var second = new TrackedContent(); + if (firstIsContent) + request.WithContent(first); + else + request.WithJson(null); + + Action assign = () => + { + if (secondIsContent) + request.WithContent(second); + else + request.WithJson("second"); + }; + + assign.Should().Throw(); + second.IsDisposed.Should().BeFalse(); + } + + [TestCase(false)] + [TestCase(true)] + public async Task WithContent_PreservesRawContentAndDisposesIt_EvenOnConfigurationFailure(bool fail) + { + using var content = new TrackedContent(); + using var handler = new InspectHandler(async (request, ct) => + { + request.Content.Should().BeSameAs(content); + content.IsDisposed.Should().BeFalse(); + (await request.Content.ReadAsStringAsync(ct)).Should().Be("raw"); + return new HttpResponseMessage(System.Net.HttpStatusCode.OK); + }); + using var client = CreateClient(handler); + var builder = client.Post("/api").WithContent(content); + if (fail) + builder.Configure(_ => throw new ArgumentException("Configuration failed.")); + + if (fail) + { + Func send = () => builder.SendAsync(); + await send.Should().ThrowAsync(); + } + else + { + using var response = await builder.SendAsync(); + } + + content.IsDisposed.Should().BeTrue(); + } + + [TestCase(false)] + [TestCase(true)] + public async Task Configure_CannotOverrideFluentBody_RegardlessOfRegistrationOrder(bool configureFirst) + { + using var original = new TrackedContent(); + using var replacement = new TrackedContent(); + using var handler = new InspectHandler((_, _) => throw new AssertionException("Conflicting body must not be sent.")); + using var client = CreateClient(handler); + var builder = client.Post("/api"); + if (configureFirst) + builder.Configure(r => r.Content = replacement); + + builder.WithContent(original); + if (!configureFirst) + builder.Configure(r => r.Content = replacement); + + Func send = () => builder.SendAsync(); + await send.Should().ThrowAsync(); + original.IsDisposed.Should().BeTrue(); + replacement.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task Configure_CannotRemoveBody() + { + using var content = new TrackedContent(); + using var client = new HttpClient(); + var builder = client.Post("/api").WithContent(content).Configure(r => r.Content = null); + Func send = () => builder.SendAsync(); + await send.Should().ThrowAsync(); + content.IsDisposed.Should().BeTrue(); + } + + [Test] + public async Task Configure_CanSupplyBodyWithoutFluentAssignment() + { + using var handler = new InspectHandler(async (request, ct) => + { + (await request.Content!.ReadAsStringAsync(ct)).Should().Be("\"value\""); + return new HttpResponseMessage(System.Net.HttpStatusCode.OK); + }); + using var client = CreateClient(handler); + using var response = await client.Post("/api").Configure(r => r.WithJson("value")).SendAsync(); + } + + [TestCase("success")] + [TestCase("failure")] + [TestCase("cancelled")] + [TestCase("configuration")] + public async Task Builder_IsConsumedOnFirstAttempt(string outcome) + { + using var handler = new InspectHandler((_, _) => outcome == "failure" + ? Task.FromException(new HttpRequestException("Failed.")) + : Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK))); + using var client = CreateClient(handler); + var builder = client.Get("/api"); + using var cancellation = new CancellationTokenSource(); + if (outcome == "cancelled") + cancellation.Cancel(); + if (outcome == "configuration") + builder.Configure(_ => throw new ArgumentException("Failed.")); + + if (outcome == "success") + { + using var response = await builder.SendAsync(); + } + else + { + Func first = () => builder.SendAsync(cancellation.Token); + if (outcome == "cancelled") + await first.Should().ThrowAsync(); + else if (outcome == "configuration") + await first.Should().ThrowAsync(); + else + await first.Should().ThrowAsync(); + } + + Func second = () => builder.GetResponseAsync(); + await second.Should().ThrowAsync(); + Action configure = () => builder.WithPaging(take: 1); + Action body = () => builder.WithJson("late"); + using var lateContent = new TrackedContent(); + Action content = () => builder.WithContent(lateContent); + Action status = () => builder.WithStatusResult(System.Net.HttpStatusCode.OK, CoreEx.Results.Result.Success); + configure.Should().Throw(); + body.Should().Throw(); + content.Should().Throw(); + status.Should().Throw(); + lateContent.IsDisposed.Should().BeFalse(); + } + + [TestCase(false)] + [TestCase(true)] + public async Task WithContent_IsDisposedOnSendFailureOrCancellation(bool cancel) + { + using var content = new TrackedContent(); + using var handler = new InspectHandler((_, _) => Task.FromException(new HttpRequestException("Failed."))); + using var client = CreateClient(handler); + using var cancellation = new CancellationTokenSource(); + if (cancel) + cancellation.Cancel(); + + Func send = () => client.Post("/api").WithContent(content).SendAsync(cancellation.Token); + if (cancel) + await send.Should().ThrowAsync(); + else + await send.Should().ThrowAsync(); + + content.IsDisposed.Should().BeTrue(); + } + + [Test] + public void NullContent_IsRejectedWithoutConsumingBodySlot() + { + using var client = new HttpClient(); + var builder = client.Post("/api"); + Action assign = () => builder.WithContent(null!); + assign.Should().Throw(); + builder.WithJson("valid").Should().BeSameAs(builder); + + using var request = new HttpRequestMessage(); + Action assignMessage = () => request.WithContent(null!); + assignMessage.Should().Throw(); + request.WithJson("valid").Should().BeSameAs(request); + } + + [Test] + public async Task ConcurrentExecution_OnlySendsOnce_AndRejectsConfigurationWhileSending() + { + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var calls = 0; + using var handler = new InspectHandler(async (_, ct) => + { + Interlocked.Increment(ref calls); + started.SetResult(); + await release.Task.WaitAsync(ct); + return new HttpResponseMessage(System.Net.HttpStatusCode.OK); + }); + using var client = CreateClient(handler); + var builder = client.Get("/api"); + var first = builder.SendAsync(); + try + { + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Func second = () => builder.ToResultAsync(); + await second.Should().ThrowAsync(); + Action configure = () => builder.Configure(_ => { }); + configure.Should().Throw(); + } + finally + { + release.SetResult(); + using var response = await first; + } + + calls.Should().Be(1); + } + + private static HttpClient CreateClient(HttpMessageHandler handler) => new(handler) { BaseAddress = new Uri("https://example.com/") }; + + private sealed class InspectHandler(Func> inspect) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => inspect(request, cancellationToken); + } + + private sealed class TrackedContent() : StringContent("raw") + { + public bool IsDisposed { get; private set; } + + protected override void Dispose(bool disposing) + { + IsDisposed = true; + base.Dispose(disposing); + } + } +} diff --git a/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs b/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs index 3edf2c0a7..b60b1c370 100644 --- a/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs +++ b/tests/CoreEx.Test.Unit/Http/HttpRequestBuilderTests.cs @@ -1,10 +1,3 @@ -using System.Net; -using System.Net.Http.Headers; -using System.Text; -using System.Text.Json; -using CoreEx.Data; -using CoreEx.Http; - namespace CoreEx.Test.Unit.Http; [TestFixture] @@ -60,6 +53,29 @@ private sealed class WidgetValidationResult(Widget? value, bool hasErrors = fals private static HttpClient CreateClient(HttpMessageHandler handler) => new(handler) { BaseAddress = new Uri("https://example.com/") }; + [Test] + public async Task ValidatedBuilder_SharesSingleUseExecution_AndRejectsLateConfiguration() + { + using var handler = new InspectHandler((_, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"name\":\"Widget\"}", Encoding.UTF8, "application/json") + })); + using var client = CreateClient(handler); + var builder = client.Get("/api"); + var validator = new WidgetValidator((w, _) => Task.FromResult>(new WidgetValidationResult(w))); + var validated = builder.WithValidator(validator); + + (await validated.GetValueAsync()).Name.Should().Be("Widget"); + Func repeatValidated = () => validated.ToResultAsync(); + Func repeatBuilder = () => builder.SendAsync(); + await repeatValidated.Should().ThrowAsync(); + await repeatBuilder.Should().ThrowAsync(); + Action lateValidator = () => builder.WithValidator(validator); + Action lateStatus = () => validated.WithStatusResult(HttpStatusCode.NotFound, CoreEx.Results.Result.NotFoundError()); + lateValidator.Should().Throw(); + lateStatus.Should().Throw(); + } + [Test] public void Properties_And_ConfigureChaining() { @@ -276,7 +292,7 @@ public async Task ThrowOnBusinessExceptionAsync_ThrowsOnBusinessException_AndDis } [Test] - public async Task SendValue_SerializesPayload_AndSupportsFluentChaining() + public async Task WithJson_SerializesPayload_AndSupportsFluentChaining() { var payload = new Widget { Name = "PayloadWidget" }; DisposableResponse? lastResponse = null; @@ -294,7 +310,7 @@ public async Task SendValue_SerializesPayload_AndSupportsFluentChaining() }); using var client = CreateClient(handler); - var created = await client.SendValue(HttpMethod.Post, "/api/widgets", payload) + var created = await client.Post("/api/widgets").WithJson(payload) .GetValueAsync(); created.Name.Should().Be("CreatedWidget"); @@ -438,7 +454,7 @@ public async Task FluentWithMethods_WithPayloadAndIfNoneMatch() }); using var client = CreateClient(handler); - await client.SendValue(HttpMethod.Put, "/api/widgets/1", new Widget { Name = "test" }) + await client.Put("/api/widgets/1").WithJson(new Widget { Name = "test" }) .WithIfNoneMatch("xyz") .WithMergePatchJsonContentType() .WithPaging(new PagingArgs(5, 15, true)) From 4b90d42006f8a00919d40d57396377a695983e2d Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 20 Oct 2026 11:48:13 -0700 Subject: [PATCH 15/25] Fix MessagePack audit failure during CI restore Pin patched MessagePack in CoreEx.UnitTesting and the sample/template MockHost so transitive consumers resolve 2.5.307. Include remaining scaffold descriptions and AppHost cleanup. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .claude/commands/coreex-scaffold.md | 2 +- .github/skills/coreex-scaffold/SKILL.md | 2 +- CHANGELOG.md | 1 + .../Contoso.Aspire.MockHost/Contoso.Aspire.MockHost.csproj | 1 + samples/aspire/Contoso.Aspire/AppHost.cs | 7 ++++++- .../app-name.Aspire.MockHost.csproj | 2 ++ src/CoreEx.UnitTesting/CoreEx.UnitTesting.csproj | 2 ++ 7 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.claude/commands/coreex-scaffold.md b/.claude/commands/coreex-scaffold.md index bf4eb1470..19fa04351 100644 --- a/.claude/commands/coreex-scaffold.md +++ b/.claude/commands/coreex-scaffold.md @@ -1,5 +1,5 @@ --- -description: "Guide a developer through CoreEx solution shaping after bootstrap, using a short plain-English interview that turns user answers into safe dotnet new template inputs. USE FOR: bootstrap-only repos, deciding API-only vs API plus relay vs API plus subscriber, choosing SQL Server vs Postgres vs Cosmos vs no database, choosing refdata/outbox/DDD/ROP options, installing CoreEx.Template, checking current solution shape, adding missing Api/Relay/Subscribe hosts to an existing repo, adding a .NET Aspire AppHost for local multi-host orchestration and dashboard visibility, and optionally preparing a first local runnable state with local dependency assets plus database/code-generation steps. DO NOT USE FOR: unrelated runtime debugging, bootstrap creation, or forcing root re-scaffolding over an existing solution. INVOKES: workspace inspection, ask-questions style interviews, dotnet new install/list, dry-run validation, solution wiring, optional local dependency asset creation, focused build/test validation, and either template generation or manual retrofit work depending on repo shape." +description: "Guide CoreEx solution scaffolding and retrofit decisions after bootstrap. USE FOR: choosing coreex templates and options (API, Relay, Subscribe, database, messaging, refdata, DDD, ROP), checking the current solution shape, adding missing hosts or an Aspire AppHost, or preparing a first local runnable state. DO NOT USE FOR: unrelated runtime debugging, creating the initial AI bootstrap (use coreex-bootstrap), or re-scaffolding an existing solution. INVOKES: workspace inspection, a one-question interview, pinned template installation, dry-run validation, template generation or retrofit, solution wiring, and focused build/test validation." allowed-tools: [Read, Glob, Grep, Edit, Write, Bash] --- diff --git a/.github/skills/coreex-scaffold/SKILL.md b/.github/skills/coreex-scaffold/SKILL.md index 3e046886b..9d558376c 100644 --- a/.github/skills/coreex-scaffold/SKILL.md +++ b/.github/skills/coreex-scaffold/SKILL.md @@ -1,6 +1,6 @@ --- name: coreex-scaffold -description: "Guide a developer through CoreEx solution shaping after bootstrap, using a short plain-English interview that turns user answers into safe dotnet new template inputs. USE FOR: bootstrap-only repos, deciding API-only vs API plus relay vs API plus subscriber, choosing SQL Server vs Postgres vs Cosmos vs no database, choosing refdata/outbox/DDD/ROP options, installing CoreEx.Template, checking current solution shape, adding missing Api/Relay/Subscribe hosts to an existing repo, adding a .NET Aspire AppHost for local multi-host orchestration and dashboard visibility, and optionally preparing a first local runnable state with local dependency assets plus database/code-generation steps. DO NOT USE FOR: unrelated runtime debugging, bootstrap creation, or forcing root re-scaffolding over an existing solution. INVOKES: workspace inspection, ask-questions style interviews, dotnet new install/list, dry-run validation, solution wiring, optional local dependency asset creation, focused build/test validation, and either template generation or manual retrofit work depending on repo shape." +description: "Guide CoreEx solution scaffolding and retrofit decisions after bootstrap. USE FOR: choosing coreex templates and options (API, Relay, Subscribe, database, messaging, refdata, DDD, ROP), checking the current solution shape, adding missing hosts or an Aspire AppHost, or preparing a first local runnable state. DO NOT USE FOR: unrelated runtime debugging, creating the initial AI bootstrap (use coreex-bootstrap), or re-scaffolding an existing solution. INVOKES: workspace inspection, a one-question interview, pinned template installation, dry-run validation, template generation or retrofit, solution wiring, and focused build/test validation." argument-hint: "Optional: base solution name, whether this is new or retrofit, required hosts, database choice, messaging needs, and whether to add an Aspire AppHost for local orchestration." tags: ["coreex", "scaffolding", "retrofit", "template", "hosts", "aspire", "apphost", "orchestration"] --- diff --git a/CHANGELOG.md b/CHANGELOG.md index f284f62e1..378b598bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Represents the **NuGet** versions. - *Behavior change:* Added a string-specific `Required()` validation overload that rejects null, empty and whitespace values with a `ValidationException`. Existing string call sites now reject empty/whitespace strings rather than only null. - *Maintenance:* Updated framework dependencies and corresponding `CoreEx.Template` dependencies, including Aspire 13.6.1, EF/.NET servicing packages and FusionCache 2.9.0. The SQL Server package now explicitly references `System.Security.Cryptography.Pkcs` 10.0.13; the Aspire template's MessagePack override is updated to 2.5.307. - *Fixed:* Named/typed HTTP client registration now binds standard resilience settings from the nested `Resilience` configuration section rather than the parent client section. +- *Fixed:* `CoreEx.UnitTesting` and the sample/template MockHost now explicitly reference MessagePack 2.5.307 so their transitive Aspire dependency cannot resolve the vulnerable 2.5.302 version; a central version entry alone does not pin transitive dependencies. - *Fixed:* SQL Server and PostgreSQL unit-of-work root transactions now honor `DatabaseArgs.IsolationLevel`; nested invocations use savepoints and inherit the existing transaction's isolation level. - *Fixed:* `HttpRequestMessage.WithQuery(QueryArgs)` (and therefore `HttpRequestBuilder.WithQuery(QueryArgs)`) previously ignored `IsIncludeText` and `IsIncludeInactive`; these are now sent as `$text=true` and `$inactive=true` respectively. diff --git a/samples/aspire/Contoso.Aspire.MockHost/Contoso.Aspire.MockHost.csproj b/samples/aspire/Contoso.Aspire.MockHost/Contoso.Aspire.MockHost.csproj index 513fc7705..ebe838192 100644 --- a/samples/aspire/Contoso.Aspire.MockHost/Contoso.Aspire.MockHost.csproj +++ b/samples/aspire/Contoso.Aspire.MockHost/Contoso.Aspire.MockHost.csproj @@ -11,6 +11,7 @@ + diff --git a/samples/aspire/Contoso.Aspire/AppHost.cs b/samples/aspire/Contoso.Aspire/AppHost.cs index 6a0495c4b..ce4216116 100644 --- a/samples/aspire/Contoso.Aspire/AppHost.cs +++ b/samples/aspire/Contoso.Aspire/AppHost.cs @@ -16,7 +16,7 @@ var redis = builder.AddExternalConnectionString("redis").WithIconName("Database"); var serviceBus = builder.AddExternalConnectionString("ServiceBus").WithIconName("MailMultiple"); var cosmos = builder.AddExternalConnectionString("Cosmos", endpointKey: "AccountEndpoint").WithIconName("DatabaseMultiple"); -var ollama = builder.AddExternalService("ollama", builder.Configuration["Ollama:BaseAddress"].ThrowIfNullOrEmpty()).WithHttpHealthCheck("/api/tags").WithIconName("BrainCircuitRegular"); +var ollama = builder.AddExternalService("ollama", builder.Configuration["Ollama:BaseAddress"].ThrowIfNullOrEmpty()).WithHttpHealthCheck("/api/tags"); // Customers domain. var customersApi = builder.AddProject("customers-api").WithReference(cosmos).WithReference(redis).AddEndpoints("/health/ready/detailed"); @@ -24,13 +24,17 @@ // Products domain. var productSearchMode = builder.Configuration["Products:Search:Mode"] ?? "Hybrid"; + var productsApi = builder.AddProject("products-api").WithReference(postgres).WithReference(redis).AddEndpoints("/health/ready/detailed"); productsApi.WithEnvironment("Products__Search__Mode", productSearchMode); productsApi.WithReference(ollama).WithEnvironment("Ollama__BaseAddress", "http://ollama").WaitFor(ollama); + builder.AddProject("products-relay").WithReference(postgres).WithReference(serviceBus).AddEndpoints("/health/ready/detailed").AddHostedServiceSupport(); + var productsSubscribe = builder.AddProject("products-subscribe").WithReference(postgres).WithReference(redis).WithReference(serviceBus).AddEndpoints("/health/ready/detailed").AddHostedServiceSupport(); productsSubscribe.WithEnvironment("Products__Search__Mode", productSearchMode); productsSubscribe.WithReference(ollama).WithEnvironment("Ollama__BaseAddress", "http://ollama"); + foreach (var setting in builder.Configuration.GetSection("Ollama:Resilience").AsEnumerable().Where(x => x.Value is not null)) { productsApi.WithEnvironment(setting.Key.Replace(":", "__"), setting.Value.ThrowIfNull()); @@ -77,6 +81,7 @@ var orderWorkflowWorker = builder.AddProject("order-workflow-worker").AddEndpoints("/health").WithUrlForEndpoint("https", ep => { ep.Url = "http://localhost:8082"; ep.DisplayText = "DTS Dashboard"; }); builder.AddProject("orders-api").WithReference(sqlServer).WithReference(redis).WaitFor(orderWorkflowWorker).AddEndpoints("/health/ready/detailed"); +// Run host. var app = builder.Build(); await app.StartAsync(); diff --git a/src/CoreEx.Template/content/CoreEx.Aspire/aspire/app-name.Aspire.MockHost/app-name.Aspire.MockHost.csproj b/src/CoreEx.Template/content/CoreEx.Aspire/aspire/app-name.Aspire.MockHost/app-name.Aspire.MockHost.csproj index 8748f23c7..52ee5e06a 100644 --- a/src/CoreEx.Template/content/CoreEx.Aspire/aspire/app-name.Aspire.MockHost/app-name.Aspire.MockHost.csproj +++ b/src/CoreEx.Template/content/CoreEx.Aspire/aspire/app-name.Aspire.MockHost/app-name.Aspire.MockHost.csproj @@ -22,6 +22,8 @@ + + diff --git a/src/CoreEx.UnitTesting/CoreEx.UnitTesting.csproj b/src/CoreEx.UnitTesting/CoreEx.UnitTesting.csproj index 4b6cb00f7..f712f56f6 100644 --- a/src/CoreEx.UnitTesting/CoreEx.UnitTesting.csproj +++ b/src/CoreEx.UnitTesting/CoreEx.UnitTesting.csproj @@ -13,6 +13,8 @@ + + From 13cab2308b7b30328bd543ec8acc385bd91a7d2e Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 20 Oct 2026 12:47:32 -0700 Subject: [PATCH 16/25] Fix hybrid product search on .NET 8 Use list-based candidate membership to avoid span expressions during EF query evaluation, and document the compatibility constraint. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- samples/docs/search.md | 2 ++ .../Repositories/ProductQueryRepository.cs | 14 ++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/samples/docs/search.md b/samples/docs/search.md index dd1632579..e92cc3e5b 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -617,6 +617,8 @@ Ranked search must define its result window and count semantics: For hybrid mode, the branch candidate lists are independent of requested paging, `$skip` and `$take` are applied only after fusion, and `$count` is the deduplicated candidate-union size (maximum 200). It is not a total match count or the complete eligible catalog. Semantic and FullText keep their existing count behavior. No branch is silently dropped: an unready lexical/semantic projection or failed query embedding makes Hybrid unavailable (HTTP 503), rather than returning a partial branch. Empty lists are valid when both indexes are ready; weak semantic matches are not assigned a confidence score. +The final product lookup uses a materialized `List` and its instance `Contains` inside the EF predicate. Keep this list-based: array `Contains` can introduce a span conversion into the expression tree that EF cannot evaluate on .NET 8. The existing hybrid-search API tests cover this lookup across all supported target frameworks in CI. + ## Model hosting and operational costs A locally runnable, pinned model is attractive for a self-contained learning sample, but "local" is not "free". Downloads, licensing, RAM, CPU/GPU needs, startup time, and query latency still matter. diff --git a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs index 8f7d4497e..6c1be8ee8 100644 --- a/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs +++ b/samples/src/Contoso.Products.Infrastructure/Repositories/ProductQueryRepository.cs @@ -134,8 +134,10 @@ join match in _searchIndex.Matches(vector.ThrowIfNull()) on candidate.Product.Id var fused = ProductSearchFusion.Fuse( lexical.Select(item => new ProductSearchFusion.LexicalCandidate(item.Id, item.ExactSku)).ToArray(), semantic); - var ids = fused.Select(candidate => candidate.Id).ToArray(); - var mapped = ids.Length == 0 + + // List.Contains avoids span-based array expressions that EF cannot evaluate on .NET 8. + var ids = fused.Select(candidate => candidate.Id).ToList(); + var mapped = ids.Count == 0 ? [] : await q.Where(candidate => ids.Contains(candidate.Product.Id)) .ToMappedItemsAsync(x => new Contracts.ProductLite @@ -153,16 +155,16 @@ join match in _searchIndex.Matches(vector.ThrowIfNull()) on candidate.Product.Id IsNonStocked = x.Product.IsNonStocked, QtyOnHand = x.QtyOnHand }, cancellationToken).ConfigureAwait(false); - var byId = mapped.ToDictionary( - product => product.Id ?? throw new InvalidOperationException("A product search candidate must have an identifier."), - StringComparer.Ordinal); + + var byId = mapped.ToDictionary(product => product.Id ?? throw new InvalidOperationException("A product search candidate must have an identifier."), StringComparer.Ordinal); var effectivePaging = paging ?? new PagingArgs(); IEnumerable pageIds = effectivePaging.IsNone ? fused.Select(candidate => candidate.Id) : fused.Skip(effectivePaging.Skip).Take(effectivePaging.Take).Select(candidate => candidate.Id); + var result = new ItemsResult(paging) { - Items = pageIds.Select(id => byId[id]).ToArray() + Items = [.. pageIds.Select(id => byId[id])] }; if (paging?.IsCountRequested is true) From e9e32f2fcefdd969679c95304bf4ae46362cadb8 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Tue, 20 Oct 2026 13:43:07 -0700 Subject: [PATCH 17/25] Preserve GraphQL search validation details Retain SEARCH_PARSE_ERROR while exposing structured search validation messages, add regression coverage and release notes, and include manual MCP validation guidance. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- CHANGELOG.md | 1 + samples/docs/search.md | 23 ++++++++++- src/CoreEx.Data.GraphQL/GraphQLEngine.cs | 8 ++-- src/CoreEx.Data.GraphQL/README.md | 4 +- .../GraphQLEngineTests.cs | 40 +++++++++++++++++++ 5 files changed, 70 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 378b598bd..56355bf54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Represents the **NuGet** versions. - *Behavior change:* Added a string-specific `Required()` validation overload that rejects null, empty and whitespace values with a `ValidationException`. Existing string call sites now reject empty/whitespace strings rather than only null. - *Maintenance:* Updated framework dependencies and corresponding `CoreEx.Template` dependencies, including Aspire 13.6.1, EF/.NET servicing packages and FusionCache 2.9.0. The SQL Server package now explicitly references `System.Security.Cryptography.Pkcs` 10.0.13; the Aspire template's MessagePack override is updated to 2.5.307. - *Fixed:* Named/typed HTTP client registration now binds standard resilience settings from the nested `Resilience` configuration section rather than the parent client section. +- *Fixed:* GraphQL-lite search parsing errors now preserve actionable per-property validation details in `extensions.messages["$search"]` while retaining the `SEARCH_PARSE_ERROR` code, including unsupported search, length/term limits and custom validation failures. - *Fixed:* `CoreEx.UnitTesting` and the sample/template MockHost now explicitly reference MessagePack 2.5.307 so their transitive Aspire dependency cannot resolve the vulnerable 2.5.302 version; a central version entry alone does not pin transitive dependencies. - *Fixed:* SQL Server and PostgreSQL unit-of-work root transactions now honor `DatabaseArgs.IsolationLevel`; nested invocations use savepoints and inherit the existing transaction's isolation level. - *Fixed:* `HttpRequestMessage.WithQuery(QueryArgs)` (and therefore `HttpRequestBuilder.WithQuery(QueryArgs)`) previously ignored `IsIncludeText` and `IsIncludeInactive`; these are now sent as `$text=true` and `$inactive=true` respectively. diff --git a/samples/docs/search.md b/samples/docs/search.md index e92cc3e5b..0f34bbd06 100644 --- a/samples/docs/search.md +++ b/samples/docs/search.md @@ -12,7 +12,7 @@ The goal is not merely to store some numbers and label the result "AI". It is to |---|---|---| | CoreEx `QueryArgs.Search` and search parser configuration | Implemented | REST, HTTP clients, OpenAPI, and GraphQL-lite can carry search text. | | Products semantic search | Implemented; default for `$search` | Exact cosine-distance ranking replaces the earlier keyword matching. Requests without `$search` retain ordinary filtering and ordering. | -| MCP `search_products` forwarding to Products | Implemented | The tool forwards `$search`, optional category/subcategory/brand/price filters, and paging to the Products API; it does not query the database. | +| MCP product-search tools | Implemented | `search_products` forwards `$search`, optional category/subcategory/brand/price filters, and paging to the Products API; `get_product_search_options` exposes live category, subcategory and brand codes. MCP does not query the database. | | Ranked-search mode plumbing | Implemented | CoreEx validates and carries ranked search intent; the repository must execute it. | | Canonical product search document | Implemented | Versioned, whitespace-normalized documents and source hashes exclude price, status, inventory and business metadata. | | Product descriptions and price filtering | Implemented | Optional descriptive content supports CRUD; decimal price constraints use existing REST and GraphQL-lite query configuration. | @@ -372,11 +372,32 @@ Keep deterministic tool behavior checks separate from protocol and model evaluat |---|---|---| | Direct MCP tool tests | Calls `CommerceTools` directly with a mocked Products HTTP handler; checks query forwarding, special-character handling, combined filters, culture-invariant prices, validation, API failures, live filter options, and the MCP input schema. This does not start the MCP server, database, or embedding model. | `dotnet test samples/aspire/Contoso.Test.Aspire --filter "FullyQualifiedName~ProductSearchToolTests"` | | Streamable HTTP protocol E2E | Starts the real Aspire topology, connects through the MCP client SDK, discovers tools and output schemas, then exercises semantic product search and the basket lifecycle through MCP. This verifies the MCP transport and real cross-domain calls, not just the static tool methods. | `dotnet test samples/aspire/Contoso.Test.Aspire --filter "FullyQualifiedName~Contoso_Mcp_FullBasketLifecycle"` | +| Interactive MCP client check | Uses MCP Inspector or Copilot CLI against the running local Streamable HTTP server. Verifies that a client can discover tools, use live filter codes, select structured constraints, and ground its response in returned products. This is exploratory, model-dependent validation, not a deterministic regression test. | Follow the steps below. | The E2E fixture migrates/seeds Products, Shopping, and Customers, clears Redis, rebuilds the real-model search projection, and resets Service Bus before starting the hosts. Run it only against disposable local infrastructure; it resets shared local test services. Locally, start the [Compose search profile and install the pinned model](local-dev.md#optional-local-embedding-model) first. CI includes this test in `CoreEx.Samples.Test.E2E.slnf`; the workflow starts the search profile and pulls the model before running Aspire E2E tests. These tests verify tool behavior when called and the real MCP protocol flow. They do not test whether an LLM chooses to call `search_products` for a natural-language prompt; that is a separate, nondeterministic agent evaluation, not a deterministic CI test. +#### Manually validate with MCP Inspector + +Start `Contoso.Aspire` and copy the `contoso-mcp` HTTP endpoint from the Aspire Dashboard, appending `/mcp` (for example, `http://localhost:7235/mcp`). In [MCP Inspector](https://github.com/modelcontextprotocol/inspector), select **Streamable HTTP**, enter the endpoint, and connect. A successful connection only verifies transport; continue to the tools view and list the available tools. + +Call `get_product_search_options` first and inspect its live category, subcategory and brand codes. Then invoke `search_products` with `searchTerm` = `descending with purpose`, `categoryCode` = `B`, `subCategoryCode` = `EM`, `maxPrice` = `10000`, and `take` = `10`. Leave other optional arguments unset. Verify every returned product matches the requested category/subcategory and is priced at or below the maximum; fewer than ten results are valid. Optionally add `brandCode` = `YETI` to verify that multiple filters are combined with AND semantics. Inspector tests explicit tool calls; it does not test an AI model's ability to select tools from a natural-language request. + +#### Manually validate tool selection with Copilot CLI + +With Aspire still running, register the dashboard endpoint from PowerShell (replace the port with the current `contoso-mcp` endpoint): + +```powershell +copilot mcp add --transport http coreex-contoso "http://localhost:/mcp" +``` + +In a **new Copilot CLI conversation**, use `/mcp` to confirm the server and tools are available. Then ask a natural-language request without procedural coaching, for example: “Find me a dentist-worthy electric mountain bike for aggressive descending, under $10,000. Show prices and product IDs.” Inspect the actual tool calls, not just the final prose: the client should use `get_product_search_options` when it needs current codes, pass the electric-bike subcategory (`EM`) and `maxPrice: 10000` as structured filters, and leave “dentist-worthy” and descending intent in `searchTerm` rather than inventing a Yeti-only constraint. If the client already has valid reference codes, an extra discovery call is not required on every request. + +Compare the response with the tool's returned products and descriptions. The model may summarize retrieved evidence, but a ranking result alone does not prove unsupported claims such as “best for descending”; require a product lookup or other supporting data before presenting such claims as fact. A different product order or fewer matches can be valid when the search text changes, provided the explicit filters are satisfied. Do not ask the client to create or alter a basket during this read-only search check. Remove the CLI registration afterwards with `copilot mcp remove coreex-contoso`. + +`Contoso.Mcp` has no authentication and is intended only for local development; use it only on trusted local infrastructure and never expose it to an untrusted network. Avoid destructive setup or checkout operations during search validation. + Search does not resolve authentication or authorization. The sample's existing authentication gaps remain separate concerns. ## Keeping search current diff --git a/src/CoreEx.Data.GraphQL/GraphQLEngine.cs b/src/CoreEx.Data.GraphQL/GraphQLEngine.cs index fc5f159c5..10ae9fab3 100644 --- a/src/CoreEx.Data.GraphQL/GraphQLEngine.cs +++ b/src/CoreEx.Data.GraphQL/GraphQLEngine.cs @@ -364,7 +364,7 @@ private Task ExecuteItemRootAsync(GraphQLItemRoot root, GraphQLField field, stri KeyNotFoundException => NewError(ex.Message, [alias], "ARGUMENT_ERROR"), QueryFilterParserException => NewError(ex.Message, [alias], "FILTER_PARSE_ERROR"), QueryOrderByParserException => NewError(ex.Message, [alias], "ORDERBY_PARSE_ERROR"), - QuerySearchParserException => NewError(ex.Message, [alias], "SEARCH_PARSE_ERROR"), + QuerySearchParserException qspex => MapValidationException(qspex, alias, "SEARCH_PARSE_ERROR"), ValidationException vex => MapValidationException(vex, alias), NotFoundException => MapKnownExtendedException(ex, alias, "NOT_FOUND"), ConflictException => MapKnownExtendedException(ex, alias, "CONFLICT_ERROR"), @@ -393,13 +393,13 @@ private static GraphQLEngineError MapKnownExtendedException(Exception ex, string /// Maps a to a , including a per-property messages extension (mirroring the REST ValidationProblem shape) /// where is populated - so structured per-property detail is not lost behind the single top-level . /// - private static GraphQLEngineError MapValidationException(ValidationException vex, string alias) + private static GraphQLEngineError MapValidationException(ValidationException vex, string alias, string code = "VALIDATION_ERROR") { if (vex.ShouldBeLogged) LogException(vex); if (vex.Messages is not { Count: > 0 }) - return NewError(vex.Message, [alias], "VALIDATION_ERROR"); + return NewError(vex.Message, [alias], code); var messages = new Dictionary(); foreach (var group in from m in vex.Messages.GetMessagesForType(MessageType.Error).Where(x => x.Property is not null && x.Text is not null) @@ -409,7 +409,7 @@ group m by m.Property into g messages.Add(group.Property!, [.. group.Messages.Select(m => m.Text!.ToString()!)]); } - return NewError(vex.Message, [alias], "VALIDATION_ERROR", new Dictionary { ["messages"] = messages }); + return NewError(vex.Message, [alias], code, new Dictionary { ["messages"] = messages }); } /// diff --git a/src/CoreEx.Data.GraphQL/README.md b/src/CoreEx.Data.GraphQL/README.md index 07cb6e512..85f4c3f77 100644 --- a/src/CoreEx.Data.GraphQL/README.md +++ b/src/CoreEx.Data.GraphQL/README.md @@ -50,7 +50,9 @@ The engine is deliberately **transport-agnostic**: it references only `CoreEx.Da already-materialized result via `JsonFilter`, not via per-field resolvers. - 🧾 **GraphQL-shaped errors**: `GraphQLArgumentTranslationException`, `QueryFilterParserException`, `QueryOrderByParserException`, `QuerySearchParserException`, `ValidationException`, `NotFoundException`, and unknown-field errors are - mapped to `{ message, path, extensions.code }` error objects. + mapped to `{ message, path, extensions.code }` error objects. Validation errors also include per-property + `extensions.messages` details. Search parsing errors retain `SEARCH_PARSE_ERROR` and expose the actionable + reason in `extensions.messages["$search"]` (for example, unsupported search, length/term limits, or custom validation). - 🔍 **Spec-compliant introspection**: `__schema`/`__type(name:)` (plus `__typename`) implement the official GraphQL introspection schema, built once from the registered roots (see `Internal.GraphQLIntrospectionSchemaBuilder`) and exposed identically via `IGraphQLEngine.GetSchemaAsync()`. Each query root's `where`/`orderBy` arguments diff --git a/tests/CoreEx.Data.GraphQL.Test.Unit/GraphQLEngineTests.cs b/tests/CoreEx.Data.GraphQL.Test.Unit/GraphQLEngineTests.cs index 75d7cde16..8835932b1 100644 --- a/tests/CoreEx.Data.GraphQL.Test.Unit/GraphQLEngineTests.cs +++ b/tests/CoreEx.Data.GraphQL.Test.Unit/GraphQLEngineTests.cs @@ -818,6 +818,46 @@ public async Task ExecuteAsync_ResolverThrowsValidationExceptionWithMessages_Inc messages["age"].Should().Equal("Age must be greater than zero.", "Age must be a whole number."); } + [TestCase("unsupported", "Search statement is not currently supported.")] + [TestCase("length", "Statement must not exceed 5 characters in length.")] + [TestCase("terms", "Statement must not contain more than 1 distinct terms.")] + [TestCase("custom", "Use a more specific search.")] + public async Task ExecuteAsync_QueryRoot_SearchParseError_PreservesValidationDetailsAsync(string scenario, string expectedMessage) + { + var config = QueryArgsConfig.Create(); + if (scenario != "unsupported") + { + config.WithSearch(search => + { + search.AddKeywordField(nameof(Person.Name)); + if (scenario == "length") + search.WithMaxLength(5); + else if (scenario == "terms") + search.WithMaxTerms(1); + else + search.WithValidator(_ => throw new QuerySearchParserException("Use a more specific search.")); + }); + } + + var engine = CreateEngine(options => options.AddQuery("searchPeople", config, (qa, pa, ct) => + { + config.Parse(qa).ThrowOnError(); + return Task.FromResult>(new ItemsResult(_people, pa)); + })); + + var result = await engine.ExecuteAsync("{ found: searchPeople(search: \"Alice Bob\") { edges { node { id } } } }"); + + result.HasErrors.Should().BeTrue(); + var error = result.Errors!.Single(); + error.Message.Should().Be("A query search parsing error occurred."); + error.Path.Should().Equal("found"); + error.Extensions!["code"].Should().Be("SEARCH_PARSE_ERROR"); + var messages = error.Extensions["messages"].Should().BeOfType>().Subject; + messages.Should().ContainSingle(); + messages["$search"].Should().Equal(expectedMessage); + result.Data!.Value.GetProperty("found").ValueKind.Should().Be(JsonValueKind.Null); + } + [Test] public async Task ExecuteAsync_ResolverThrowsValidationExceptionWithoutMessages_ProducesPlainValidationErrorAsync() { From 0d7aa5a6b5d3514843250e1a3ae76e12c1522e2e Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Wed, 21 Oct 2026 10:49:58 -0700 Subject: [PATCH 18/25] Document reference-data code conversion Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- ...oreex-application-services.instructions.md | 2 ++ .../coreex-contracts.instructions.md | 33 +++++++++++++++++++ .../coreex-domain.instructions.md | 2 ++ .../coreex-repositories.instructions.md | 2 ++ .github/skills/coreex-aggregate/SKILL.md | 1 + .github/skills/coreex-app-service/SKILL.md | 1 + .github/skills/coreex-contract/SKILL.md | 1 + .../coreex-contract/references/workflow.md | 1 + .github/skills/coreex-refdata/SKILL.md | 2 ++ .../coreex-refdata/references/workflow.md | 1 + .github/skills/coreex-repository/SKILL.md | 1 + samples/docs/contracts-layer.md | 19 +++++++++++ src/CoreEx.RefData/AGENTS.md | 17 +++++++++- src/CoreEx.RefData/README.md | 15 +++++++++ 14 files changed, 97 insertions(+), 1 deletion(-) diff --git a/.github/instructions/coreex-application-services.instructions.md b/.github/instructions/coreex-application-services.instructions.md index e2e3decb6..dab0f9fb2 100644 --- a/.github/instructions/coreex-application-services.instructions.md +++ b/.github/instructions/coreex-application-services.instructions.md @@ -463,6 +463,8 @@ var result = await new ProductPolicy(_productAdapter).EnsureExistsAsync(productI ## Application-Level Mapping +Pass code strings directly to concrete typed reference-data parameters/properties; map DTO destinations through their annotated `XxxCode` properties. Do not add conversion-only `RefData` helpers or redundant null checks. See [Reference-Data Code Conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion); conversion is not validation. + For collection projections, follow [Mapper Projection Nullability](/.github/instructions/coreex-conventions.instructions.md#mapper-projection-nullability); preserve the lambda selectors in the examples below. When a domain has a Domain layer, an `Application/Mapping/` sub-folder holds mappers that translate between the **Domain aggregate** and the **Contract**. This mapping is an Application-layer concern because it sits at the public surface boundary — it is not tied to any persistence technology. diff --git a/.github/instructions/coreex-contracts.instructions.md b/.github/instructions/coreex-contracts.instructions.md index 16f687c72..a2722c0f0 100644 --- a/.github/instructions/coreex-contracts.instructions.md +++ b/.github/instructions/coreex-contracts.instructions.md @@ -220,6 +220,39 @@ Do **not** apply it to optional properties, and do not use `System.Diagnostics.C Contracts are commonly shared between request and response bodies, so a server-assigned property is legitimately null on create. Never mark `Id`, `ETag` or `ChangeLog` with `[NonNullable]` — they are lifecycle fields that are null in requests, and non-nullable members in generated clients cause confusion. For any other `[ReadOnly]` property, apply `[NonNullable]` only when its containing type is never used as a request body (e.g. `BasketItem`, which is only returned within the read-only `Basket.Items`); otherwise leave it unmarked (e.g. `ProductBase.CategoryCode`). +## Reference-Data Code Conversion + +Concrete types decorated with `[ReferenceData]` receive a generated implicit conversion from `string?` to that concrete reference-data type. Use it when assigning a code to a typed property, returning a typed value, or passing a code to a typed method parameter: + +```csharp +Contracts.BookingStatus? status = source.BookingStatusCode; +Contracts.FareClass? fareClass = source.FareClassCode; +public static Contracts.BookingStatus Draft => DraftCode; // DraftCode is a non-null string constant. + +// CreateFrom's parameters and Money.Currency are concrete reference-data types. +Domain.BookingSegment.CreateFrom( + s.Id, + s.SegmentTypeCode, + s.FareClassCode, + s.Description, + s.Provider, + new Domain.ValueObjects.Money + { + Amount = s.PriceAmount, + Currency = s.PriceCurrencyCode + }, + s.StartDate, + s.EndDate); +``` + +Do not introduce application-specific `RefData` wrappers, manually construct code-only instances, or query a collection merely to convert a code. Redundant `code is null ? null : ...` guards are unnecessary. Explicit lookups remain appropriate for collection querying or when found/not-found semantics are required. + +Conversion delegates to the current orchestrator. A null code converts to null; an unresolved non-null code produces an invalid instance retaining its code, including when no current orchestrator is available. Conversion does not enforce requiredness, validity or activity; enforce these separately at the appropriate validation boundary. Do not suppress nullable warnings with `!` to bypass requiredness. + +When `var`, generic inference or an expression supplies no concrete reference-data target type, declare the type explicitly or cast at the conversion boundary. Code constrained only to `IReferenceData` cannot assume a concrete type's generated operator is available. + +On contracts, use the annotated partial-code-property pattern below: set `XxxCode` when mapping DTOs or arranging request data; consume reference-data behavior through the generated typed `Xxx` accessor. + ## Reference Data Properties Use `[ReferenceData]` on code properties that back a reference data relationship. Two conditions must both be met for the source generator to emit the navigation accessor: diff --git a/.github/instructions/coreex-domain.instructions.md b/.github/instructions/coreex-domain.instructions.md index 5a40a5877..c50400040 100644 --- a/.github/instructions/coreex-domain.instructions.md +++ b/.github/instructions/coreex-domain.instructions.md @@ -66,6 +66,8 @@ public sealed class Basket : Aggregate ### Factory Methods +Where parameters or value-object properties use concrete reference-data types, callers can pass code strings directly via generated implicit conversion. Do not add conversion-only helpers or weaken typed parameters to strings to accommodate mapping. See [Reference-Data Code Conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion) for resolution, nullability and validation semantics. + Provide two factory methods per aggregate: - `CreateNew(...)` — constructs a new aggregate with a generated ID, initial state, and calls `.AsNew()` to mark it as `PersistenceState.New`. diff --git a/.github/instructions/coreex-repositories.instructions.md b/.github/instructions/coreex-repositories.instructions.md index 53ecc3d68..e3843eda0 100644 --- a/.github/instructions/coreex-repositories.instructions.md +++ b/.github/instructions/coreex-repositories.instructions.md @@ -373,6 +373,8 @@ public Task> CreateAsync(Domain.Basket basket, Cancellatio ## Mapping +For concrete typed reference-data destinations and factory parameters, pass stored code strings directly (`Currency = source.CurrencyCode`, `CreateFrom(..., source.StatusCode, ...)`); use the generated implicit conversion, not custom `RefData` helpers or conversion-only collection lookups. Map contracts through their annotated `XxxCode` properties. See [Reference-Data Code Conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion) for null, resolution and validation semantics. + For collection projections through `.To.Map` / `.From.Map`, follow [Mapper Projection Nullability](/.github/instructions/coreex-conventions.instructions.md#mapper-projection-nullability). > **GlobalUsing requirement:** Mappers are declared in the `.Infrastructure.Mapping` namespace but referenced elsewhere in the project (repositories, adapters, `*EfDb`). Whenever you add a Contract ↔ Persistence mapper, ensure `global using .Infrastructure.Mapping;` is present in the Infrastructure project's `GlobalUsing.cs` — add it if missing. This prevents avoidable compilation errors when other classes reference the mapper (e.g. `ProductMapper.Default`). diff --git a/.github/skills/coreex-aggregate/SKILL.md b/.github/skills/coreex-aggregate/SKILL.md index 7e849712b..2281fbda7 100644 --- a/.github/skills/coreex-aggregate/SKILL.md +++ b/.github/skills/coreex-aggregate/SKILL.md @@ -59,6 +59,7 @@ then follows the corresponding pattern from `CoreEx.DomainDriven`. - Public mutation methods on the aggregate should return `Result`/`Result` for consistency with Application-layer `Result` pipelines — even though the internal guard throws - Child entity mutation methods are `internal` — only the owning aggregate may invoke them - No async I/O in domain classes — ever. Async belongs in Application services or Policies. +- Keep concrete reference-data factory parameters/value-object properties typed; callers pass code strings directly via [generated implicit conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion). Do not add conversion-only wrappers; requiredness, validity and activity remain separate guards. - No native domain-event dispatch (MediatR-style) — only integration events via `Aggregate.Events`, forwarded through `IUnitOfWork.Events` in the Application layer - Aggregates are the best unit-test target in the codebase (no injected dependencies) — cover every mutation method's happy path and rejection path with `WithGenericTester` + `Test.Scoped(...)`, calling the aggregate directly (no repository, no Application service); assert `OnCheckCanMutate()` guard failures as thrown exceptions, and failed-`Result` returns as `Result` assertions - A value object backed by a JSON database column (e.g. `Address` stored as `basket.ShippingAddressJson`) is authored **exactly like any other value object** — a plain `sealed record`/`record class` with invariant-enforcing `init` setters. It sits as the middle type in a three-type chain (`Contracts.Xxx` ↔ `Domain.ValueObjects.Xxx` ↔ `Persistence.Xxx`) bridged by two separate mappers (Application: Domain↔Contract; Infrastructure: Domain↔Persistence) — see [`coreex-domain.instructions.md#value-objects-backed-by-a-json-column`](/.github/instructions/coreex-domain.instructions.md#value-objects-backed-by-a-json-column) and the [`coreex-db-migration`](/.github/skills/coreex-db-migration/SKILL.md#json-columns) skill for the column/mapper setup diff --git a/.github/skills/coreex-app-service/SKILL.md b/.github/skills/coreex-app-service/SKILL.md index d435c0bb9..b625f5a06 100644 --- a/.github/skills/coreex-app-service/SKILL.md +++ b/.github/skills/coreex-app-service/SKILL.md @@ -61,6 +61,7 @@ Guides you through creating or modifying a CoreEx Application-layer service in ` - CQRS: mutations + `GetAsync` → `{Name}Service`; queries + `GetAsync` → `{Name}ReadService` (both have `GetAsync`) - **Before building Path C:** confirm `I{Name}Repository` already has `QueryAsync`/`QuerySchemaAsync` backed by a `{Name}QueryArgsConfig`. If not, stop and invoke `coreex-repository` first — never add filtering/ordering logic or a hand-rolled query in the service to work around a missing repository method - Always `.ConfigureAwait(false)` on every `await` +- For concrete typed reference-data parameters/properties, pass codes directly via [generated implicit conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion); map DTOs through annotated `XxxCode` properties. Do not add conversion-only helpers; conversion is not validation. - `Result` failure propagation: prefer a `Then*` chain; for a guard clause `return r;` (same type, or `Result` → `Result`) or `return r.AsResult();` (`Result` → `Result`) — **never** `Result.Fail(r.Error!)` / `return r.Error!;`, and don't touch `.Error` in service code - A Domain value object persisted via a JSON column (e.g. `Basket.ShippingAddress`) is mapped with a `BiDirectionMapper` in `Application/Mapping/` (not the uni-directional `Mapper` used for the root aggregate) — see [`coreex-application-services.instructions.md#json-backed-value-object-mapping`](/.github/instructions/coreex-application-services.instructions.md#json-backed-value-object-mapping) diff --git a/.github/skills/coreex-contract/SKILL.md b/.github/skills/coreex-contract/SKILL.md index 72d02f407..7cedd0635 100644 --- a/.github/skills/coreex-contract/SKILL.md +++ b/.github/skills/coreex-contract/SKILL.md @@ -45,6 +45,7 @@ Guides you through creating or modifying a hand-authored contract (DTO/entity) i **Key rules at a glance:** - `[Contract]` + `partial` on **all** contract classes by default - Only `[ReferenceData]` properties are `partial` — never make plain properties `partial` (CS9248) +- Map/request-arrange through annotated `XxxCode` properties; use generated typed accessors for reference-data behavior. Pass codes directly to concrete typed destinations via [implicit conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion), not custom conversion helpers; conversion is not validation. - `[ReadOnly(true)]` on all server-assigned fields (`Id`, `ETag`, `ChangeLog`, computed/derived) - `[NonNullable]` on any property that is mandatory/required (ultimately not nullable) but declared nullable (`string?`, `int?`) for DTO flexibility — drives `nullable: false` in OpenAPI; not the same as `System.Diagnostics.CodeAnalysis.NotNull`. Never on `Id`/`ETag`/`ChangeLog`; on other `[ReadOnly]` properties only when the type is never a request body - Every property needs a `` XML doc comment diff --git a/.github/skills/coreex-contract/references/workflow.md b/.github/skills/coreex-contract/references/workflow.md index 5b606bcc1..121bdcb82 100644 --- a/.github/skills/coreex-contract/references/workflow.md +++ b/.github/skills/coreex-contract/references/workflow.md @@ -262,6 +262,7 @@ Apply the 5-step type resolution. Emit the property with ``, correct ty 3. Add: `[ReferenceData]` `public partial string? {Name}Code { get; set; }` 4. Add `[Localization]` only if the auto-label would be wrong. 5. Rebuild — Roslyn emits the typed navigation property automatically. +6. Map/request-arrange through `{Name}Code`; consume behavior through `{Name}`. Pass code strings directly to concrete typed reference-data destinations using [generated implicit conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion), not custom helpers or conversion-only collection lookups. ### C3 — Add an interface diff --git a/.github/skills/coreex-refdata/SKILL.md b/.github/skills/coreex-refdata/SKILL.md index a32d3cacc..bfccf0ac0 100644 --- a/.github/skills/coreex-refdata/SKILL.md +++ b/.github/skills/coreex-refdata/SKILL.md @@ -35,6 +35,8 @@ Guides you through the complete end-to-end workflow for adding or modifying a re ## Quick Reference +- Consume codes through [generated implicit conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion) to concrete reference-data types, not custom `RefData` helpers. Contracts use annotated partial `XxxCode` properties, not plain code auto-properties; conversion does not validate requiredness, validity or activity. + > **Resolve project-wide choices from state before asking.** Read the solution-root `AGENTS.md` > **Feature Configuration** for `refdata-enabled` (gates this whole skill) and `data-provider` > (`SqlServer` / `Postgres` / `Cosmos` — drives whether the database step is DbEx or Cosmos provisioning, plus casing/seed shape). Only prompt for what is unrecorded; diff --git a/.github/skills/coreex-refdata/references/workflow.md b/.github/skills/coreex-refdata/references/workflow.md index f22027227..3c2bab10a 100644 --- a/.github/skills/coreex-refdata/references/workflow.md +++ b/.github/skills/coreex-refdata/references/workflow.md @@ -242,6 +242,7 @@ public partial string? BrandCode { get; set; } ``` 3. Rebuild — the Roslyn generator emits the typed `Brand` navigation property automatically. Do not hand-author it. +4. Set `BrandCode` when mapping/arranging DTOs and use `Brand` for typed behavior. For a concrete typed destination, use `Contracts.Brand? brand = source.BrandCode`; no helper or null guard is needed. Follow [Reference-Data Code Conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion) and validate separately. --- diff --git a/.github/skills/coreex-repository/SKILL.md b/.github/skills/coreex-repository/SKILL.md index 446422103..0bc070365 100644 --- a/.github/skills/coreex-repository/SKILL.md +++ b/.github/skills/coreex-repository/SKILL.md @@ -54,6 +54,7 @@ Guides you through creating or modifying a CoreEx Infrastructure-layer repositor - `AddReferenceDataField(field, model, ...)`: `field` is always the contract's generated nav property (`{Name}`, never `{Name}Code`) — a fixed, documented source-generator convention, **not** something to verify by exploring generated code - `ToMappedItemsResultAsync(mapper, paging, cancellationToken: cancellationToken)`: always pass `cancellationToken` **by name** — `autoCount` (`bool`, defaults `true`) sits before it in the signature, and a bare positional token there fails to compile - Always `.ConfigureAwait(false)` on every `await` +- In mappings, pass stored codes directly to concrete reference-data parameters/properties (`Currency = source.CurrencyCode`); do not add `RefData` helpers or conversion-only lookups. See [conversion semantics](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion). For full workflow and code examples see [`references/workflow.md`](references/workflow.md). diff --git a/samples/docs/contracts-layer.md b/samples/docs/contracts-layer.md index 236d9b374..39be968ca 100644 --- a/samples/docs/contracts-layer.md +++ b/samples/docs/contracts-layer.md @@ -102,6 +102,25 @@ public partial class UnitOfMeasure --- +### Consuming reference-data codes + +The generated concrete reference-data types support implicit conversion from `string?`. The Shopping [BasketMapper](../src/Contoso.Shopping.Infrastructure/Mapping/BasketMapper.cs) passes stored codes directly to typed domain parameters and value-object properties: + +```csharp +var pricing = new Domain.ValueObjects.ItemPricing +{ + UnitOfMeasure = item.UnitOfMeasureCode, + Quantity = item.Quantity, + UnitPrice = item.UnitPrice +}; +Contracts.BasketStatus? status = source.BasketStatusCode; +Contracts.DiscountCoupon? coupon = source.DiscountCouponCode; +``` + +Do not add `RefData` helpers or conversion-only collection lookups. Null stays null; unresolved non-null codes become invalid instances retaining their codes, including when no current orchestrator is available. Requiredness, validity and activity need separate validation. Use explicit lookups when found/not-found semantics or collection querying are needed. + +On contracts, declare `[ReferenceData] public partial string? XxxCode { get; set; }` within a `[Contract]` partial class. Map/request-arrange through `XxxCode`; consume typed behavior through the generated `Xxx` accessor. See the [reference-data guide](../../src/CoreEx.RefData/README.md#reference-data-code-conversion) for conversion details. + ## Source generation The `[Contract]` and `[ReferenceData]` class annotations drive the [`CoreEx.Generator`](../gen/CoreEx.Generator) Roslyn source generator. diff --git a/src/CoreEx.RefData/AGENTS.md b/src/CoreEx.RefData/AGENTS.md index 605aac00a..a880d7452 100644 --- a/src/CoreEx.RefData/AGENTS.md +++ b/src/CoreEx.RefData/AGENTS.md @@ -84,6 +84,21 @@ builder.Services.AddReferenceDataOrchestrator(sp => --- +## Consuming Codes + +Concrete `[ReferenceData]` types receive a generated implicit conversion from `string?`. Assign codes directly to typed properties, return values and method parameters: + +```csharp +BasketStatus? status = source.BasketStatusCode; +DiscountCoupon? coupon = source.DiscountCouponCode; +``` + +The generated operator uses the current orchestrator. Null converts to null; an unresolved non-null code produces an invalid instance retaining its code, including when no current orchestrator exists. Conversion does not enforce requiredness, validity or activity. Apply those checks separately at the appropriate boundary. + +Do not introduce `RefData` helpers, manually construct code-only instances, or call collection `GetByCode` merely for conversion. Use an explicit concrete type or cast when target typing/generic inference cannot supply it; an `IReferenceData` constraint alone does not expose a concrete type's operator. Explicit lookups are appropriate when collection operations or found/not-found semantics are needed. + +On contracts, use `[Contract]` + `partial` on the class and `[ReferenceData] public partial string? XxxCode { get; set; }` on each reference-data field. Map/request-arrange through `XxxCode`; consume behavior through the generated typed `Xxx` accessor. + ## ReferenceDataCodeCollection — Wire Serialization For properties that serialise as a list of string codes on the wire but expose typed reference data objects in code: @@ -119,7 +134,7 @@ public class MyService(IReferenceDataContext refDataContext) - Do not edit `*.g.cs` files — they are overwritten on every generation run. - Do not load reference data collections on every request — the orchestrator caches via `IHybridCache`; load functions are called only on a cache miss. - Do not access reference data in `static` constructors — the orchestrator must be resolved from DI at runtime. -- Do not use `ReferenceDataCodeCollection` for single-value fields — use a plain `Code` string property on the contract instead. +- Do not use `ReferenceDataCodeCollection` for single-value contract fields or replace them with plain code auto-properties; use `[ReferenceData] public partial string? XxxCode { get; set; }` inside a `[Contract]` partial class. ## Further Reading diff --git a/src/CoreEx.RefData/README.md b/src/CoreEx.RefData/README.md index 54b74fb08..0ad53460c 100644 --- a/src/CoreEx.RefData/README.md +++ b/src/CoreEx.RefData/README.md @@ -21,6 +21,21 @@ Two concrete base classes cover the most common identity types: `ReferenceData()` out of the box); `ReferenceDataOrchestratorHealthCheck` exposes a health-check endpoint reporting all registered reference data types. - 🔎 **OData-esque query/paging**: `ReferenceDataQuery` (the default `IReferenceDataQuery` implementation) filters, orders, and pages a reference data collection via `ReferenceDataQueryArgsConfig` — OData-esque `$filter` on `code`/`text` (all string operators, case-insensitive) and `$orderby` on `code`/`text`/`sortOrder`. Supply a `Func` to `ReferenceDataQuery(configSelector)` to use a different config per reference data type. +## Reference-data code conversion + +The Roslyn generator gives each concrete `[ReferenceData]` type an implicit conversion from `string?`: + +```csharp +BasketStatus? status = source.BasketStatusCode; +DiscountCoupon? coupon = source.DiscountCouponCode; +``` + +Pass code strings directly to concrete typed reference-data parameters and properties; no `RefData` wrapper, code-only construction or conversion-only collection lookup is needed. Use an explicitly typed variable or cast when the expression does not provide a concrete target type; generic `IReferenceData` constraints do not expose concrete generated operators. + +Conversion uses the current orchestrator. Null converts to null; unresolved non-null codes produce invalid instances retaining their codes, including when no current orchestrator exists. Requiredness, validity and activity must be checked separately. Use explicit collection lookups when querying the collection or requiring found/not-found semantics. + +For a single reference-data field on a contract, declare `[ReferenceData] public partial string? XxxCode { get; set; }` within a `[Contract]` partial class, not a plain code auto-property. Set `XxxCode` when mapping/arranging DTOs; use the generated typed `Xxx` accessor for reference-data behavior. + ## Key types | Type | Description | From b3a129383c12594e5aa85d356f2956a172d80211 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Wed, 21 Oct 2026 10:50:49 -0700 Subject: [PATCH 19/25] Pin stable .NET SDK and fix package publishing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .github/workflows/CI.yml | 6 ++++-- .github/workflows/product-search-relevance.yml | 2 +- CoreEx.slnx | 4 ++++ README.md | 2 ++ Version.props | 2 +- gen/CoreEx.Generator/CoreEx.Generator.csproj | 2 +- global.json | 7 +++++++ nuget-publish.ps1 | 10 ++++++---- src/Directory.Build.props | 2 +- 9 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 global.json diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e11656b85..ee4ebefab 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -10,6 +10,7 @@ on: - 'gen/**' - 'servicebus/**' - 'Directory.Packages.props' + - 'global.json' - 'docker-compose.yml' - '*.slnf' - '*.slnx' @@ -24,6 +25,7 @@ on: - 'gen/**' - 'servicebus/**' - 'Directory.Packages.props' + - 'global.json' - 'docker-compose.yml' - '*.slnf' - '*.slnx' @@ -53,10 +55,10 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: + global-json-file: global.json dotnet-version: | 8.x 9.x - 10.x - name: Cache NuGet packages uses: actions/cache@v4 @@ -212,10 +214,10 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: + global-json-file: global.json dotnet-version: | 8.x 9.x - 10.x - name: Cache NuGet packages uses: actions/cache@v4 diff --git a/.github/workflows/product-search-relevance.yml b/.github/workflows/product-search-relevance.yml index bf5a2cbc6..35a494531 100644 --- a/.github/workflows/product-search-relevance.yml +++ b/.github/workflows/product-search-relevance.yml @@ -29,7 +29,7 @@ jobs: - name: Set up .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 10.x + global-json-file: global.json - name: Cache NuGet packages uses: actions/cache@v4 diff --git a/CoreEx.slnx b/CoreEx.slnx index c0ed18e54..f0aa42ef4 100644 --- a/CoreEx.slnx +++ b/CoreEx.slnx @@ -131,11 +131,15 @@ + + + + diff --git a/README.md b/README.md index 376b3cb7d..30be55ba4 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,8 @@ The [Pattern Catalog](./samples/docs/patterns.md) is the best entry point: it in ## Build and Package Status +**Contributor SDK:** Install the stable .NET SDK specified in [`global.json`](./global.json) (10.0.402, with servicing patches in the same feature band allowed). Local builds, CI and [`nuget-publish.ps1`](./nuget-publish.ps1) use this SDK; preview SDKs installed alongside it are not selected. Run `dotnet --version` from the repository root to confirm. Publishing still packs all supported target frameworks in Release. Update the SDK baseline in `global.json` when adopting a newer feature band; CI installs that baseline automatically. + The build status is [![CI](https://github.com/Avanade/CoreEx/actions/workflows/CI.yml/badge.svg)](https://github.com/Avanade/CoreEx/actions/workflows/CI.yml) with the NuGet package status as follows, including links to the underlying source code and documentation: Project/Package | Description | Source diff --git a/Version.props b/Version.props index 2bbd6a4d4..15fe5e9e3 100644 --- a/Version.props +++ b/Version.props @@ -1,5 +1,5 @@ - 4.1.0 + 4.2.0 diff --git a/gen/CoreEx.Generator/CoreEx.Generator.csproj b/gen/CoreEx.Generator/CoreEx.Generator.csproj index bd52396bc..6d723c703 100644 --- a/gen/CoreEx.Generator/CoreEx.Generator.csproj +++ b/gen/CoreEx.Generator/CoreEx.Generator.csproj @@ -5,7 +5,7 @@ netstandard2.0 enable enable - preview + latest diff --git a/global.json b/global.json new file mode 100644 index 000000000..c59154467 --- /dev/null +++ b/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "10.0.402", + "rollForward": "latestPatch", + "allowPrerelease": false + } +} diff --git a/nuget-publish.ps1 b/nuget-publish.ps1 index a84125d47..bf01bf3b6 100644 --- a/nuget-publish.ps1 +++ b/nuget-publish.ps1 @@ -4,6 +4,7 @@ .DESCRIPTION Packs all supported target frameworks regardless of CoreExLocalTargetFramework. + Builds before packing, overriding GeneratePackageOnBuild so existing binaries are not reused without a build. Uses Release by default; the Visual Studio configuration does not affect this script. Preserves existing packages except when replacing the same package filename. Remote publishing uploads only packages produced by this invocation. @@ -27,7 +28,8 @@ List of projects to Build and publish. .PARAMETER DeleteFromlocalCache - Should the newly built packages be deleted from tle Nuget Cache Folder. + Removes matching newly packed package versions from the local NuGet cache by default. + Pass -DeleteFromlocalCache:$false to preserve cached copies. .PARAMETER NugetCacheFolder Path to Nuget Cache folder. @@ -51,7 +53,7 @@ param( [switch]$IncludeSource, [string]$NugetServer, [string]$ApiKey, - [switch]$DeleteFromlocalCache, + [switch]$DeleteFromlocalCache = $true, [string]$NugetCacheFolder = "$($env:USERPROFILE)\.nuget\packages", [String[]]$ProjectsToPublish = @( "src\CoreEx", @@ -85,11 +87,11 @@ if (!(Test-Path -Path $publishFolder)) { $stagingFolder = Join-Path ([IO.Path]::GetTempPath()) ("CoreEx-nuget-publish-" + [guid]::NewGuid().ToString('N')) New-Item -ItemType Directory -Path $stagingFolder -ErrorAction Stop | Out-Null -$packageDetailsRegexp = [regex] '(?.*?)\.(?\d*\.\d*\.\d*\.\d*.*)(?.+?)\.(?\d+\.\d+\.\d+(?:\.\d+)?(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$' try { foreach ($project in $projectsToPublish) { - $packArguments = @('pack', $project, '-o', $stagingFolder, '-c', $configuration, '-p:CoreExLocalTargetFramework=', '--force') + $packArguments = @('pack', $project, '-o', $stagingFolder, '-c', $configuration, '-p:CoreExLocalTargetFramework=', '-p:GeneratePackageOnBuild=false', '--force') if ($IncludeSource) { $packArguments += '--include-source' } if ($IncludeSymbols) { $packArguments += '--include-symbols' } diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 95f7f5e38..db2051bcf 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -7,7 +7,7 @@ $(CoreExTargetFrameworks) enable enable - preview + latest From 7a5eb50b3e58f503eef41f633f79d19d3faf7996 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Wed, 21 Oct 2026 12:20:24 -0700 Subject: [PATCH 20/25] Add predicate conditions to DependsOn validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .../coreex-validators.instructions.md | 19 ++++- .github/skills/coreex-validator/SKILL.md | 1 + .../coreex-validator/references/workflow.md | 15 +++- CHANGELOG.md | 3 + samples/docs/application-layer.md | 10 +++ src/CoreEx.Validation/AGENTS.md | 7 ++ .../Clauses/DependsOnClause.cs | 9 +- src/CoreEx.Validation/Clauses/README.md | 4 +- src/CoreEx.Validation/README.md | 2 +- .../ValidationExtensions.DependsOnClause.cs | 16 +++- .../Clauses/DependsOnClauseTests.cs | 83 ++++++++++++++++++- 11 files changed, 158 insertions(+), 11 deletions(-) diff --git a/.github/instructions/coreex-validators.instructions.md b/.github/instructions/coreex-validators.instructions.md index 93456a2c6..504de6947 100644 --- a/.github/instructions/coreex-validators.instructions.md +++ b/.github/instructions/coreex-validators.instructions.md @@ -158,7 +158,7 @@ The table above lists the most common rules; the full set of fluent extension me | Dictionaries | `Dictionary(...)` (with `WithKeyValidator(...)` / `WithValueValidator(...)`) | | Child / shared / external | `Entity(validator)` (child entity), `Common(commonValidator)` (shared value rules), `Interop(validator)` (external/FluentValidation) | | Always-error (guard with a clause) | `Error(text)`, `Duplicate()`, `NotFound()`, `Invalid()`, `Immutable()` | -| Clauses (conditional execution) | `When(...)`, `WhenValue(pred)`, `WhenHasValue()`, `WhenEntity(pred)`, `DependsOn(x => x.Other)` | +| Clauses (conditional execution) | `When(...)`, `WhenValue(pred)`, `WhenHasValue()`, `WhenEntity(pred)`, `DependsOn(x => x.Other)` / `DependsOn(x => x.Other, value => predicate)` | > **`Mandatory()` on a non-nullable value type checks `default`, not just null.** `Mandatory()` defaults to `mustNotBeDefault: true`, so on a non-nullable value type it **errors when the value equals `default(T)`** — `0` for `decimal`/`int`, `DateOnly.MinValue` for `DateOnly`, etc. (On a **nullable** type — `decimal?`, `DateOnly?` — it errors only on `null`, since a supplied `0` is non-default.) Implications: > - **Do not use `Mandatory()` on a non-nullable value type where the default is a legitimate value** (e.g. a `decimal Salary` where `0` is allowed) — it will wrongly reject `0`. Use a range rule instead, e.g. `GreaterThanOrEqualTo(0)` (allows `0`) or `GreaterThan(0)` (requires positive). To require presence *without* the default-check, pass `Mandatory(mustNotBeDefault: false)`. @@ -317,9 +317,9 @@ public partial string? SubCategoryCode { get; set; } // Produces: "Sub-category is required." (not "SubCategoryCode is required.") ``` -## DependsOn for Conditional Precision +## DependsOn for Conditional Rules and Precision -Use `.DependsOn(x => x.OtherProp)` to skip a rule when a dependent property is already invalid. This prevents misleading cascading errors: +Use `.DependsOn(x => x.OtherProp)` to skip a rule when a dependent property is unset/default or already invalid. This prevents misleading cascading errors: ```csharp Property(x => x.Quantity, c => c @@ -330,6 +330,19 @@ Property(x => x.Quantity, c => c .DependsOn(x => x.UnitOfMeasure)); ``` +The predicate overload adds a condition over the dependent property's value. The rule runs only when that property is non-default, has no validation error, **and** the predicate returns `true`: + +```csharp +// FareClass is required only when the valid SegmentType is a flight. +Property(x => x.FareClass) + .Mandatory() + .DependsOn(x => x.SegmentType, segmentType => segmentType!.IsFlight); +``` + +This is a good fit when the condition is naturally about the dependency itself. For a nullable dependency, `!` is justified in the predicate: `DependsOn` invokes it only after the dependency is non-default and has no validation error, though nullable flow analysis cannot infer that guard. The null-forgiving operator only affects compile-time analysis; it does not change validation behavior. Clauses apply to the rule they are attached to: in this example, both clauses guard `.Mandatory()`, so non-flight, unset, or invalid `SegmentType` values skip that mandatory check. Use `.WhenEntity(...)` when the condition needs other entity state or is not about the dependency value; do not replace a dependency guard if you also need its invalid-state short-circuit behavior. + +Use another `Property(x => x.SameProperty)` call to start a separate root rule chain for the same property when different rules need different conditions. This is a separate chain, not an isolated validation transaction: if an earlier chain reports an error for that property, later chains may be skipped by the existing-error guard. + ## Do Not - Do not use the `FluentValidation` NuGet package — `AbstractValidator` here is `CoreEx.Validation.AbstractValidator`, not FluentValidation. diff --git a/.github/skills/coreex-validator/SKILL.md b/.github/skills/coreex-validator/SKILL.md index f15a4c00f..5d1cb4308 100644 --- a/.github/skills/coreex-validator/SKILL.md +++ b/.github/skills/coreex-validator/SKILL.md @@ -55,6 +55,7 @@ Guides you through creating or modifying a CoreEx validator (`Application/Valida - Runtime-computed thresholds: use delegate overloads (`.LessThanOrEqualTo(_ => ...)`) — prefer over `OnValidateAsync` imperative logic - `context.HasErrors` guard before async I/O; `context.HasError(x => x.Prop)` for per-property guards - `context.AddError(x => x.Prop, ...)` — member-access expression, never `nameof(...)` +- Conditional rules on a dependency: `.DependsOn(x => x.Other, value => predicate)` skips its attached rule when the dependency is default/invalid and otherwise runs it only when the predicate is true. Clauses guard the rule they are attached to; use another `Property(...)` call for a separate root rule chain. Use `.WhenEntity(...)` for conditions based on broader entity state. - Message text argument is a `{2}` suffix substitution — not a full sentence; use `.Error("...")` to override the whole message - **Always generate or update the matching `{Validator}Tests`** in `*.Test.Unit/Validators/` — cover conditional/business rules and cross-field logic thoroughly, plus a representative error/success pair for simple rules; skip dedicated boundary tests for framework-guaranteed constraints like length rules (`MaximumLength`/`MinimumLength`/`Length`/`String`) diff --git a/.github/skills/coreex-validator/references/workflow.md b/.github/skills/coreex-validator/references/workflow.md index 2c6048940..b60a16858 100644 --- a/.github/skills/coreex-validator/references/workflow.md +++ b/.github/skills/coreex-validator/references/workflow.md @@ -65,10 +65,23 @@ public class {Name}Validator : Validator | Required + precision | `.PrecisionScale(null, 2).GreaterThanOrEqualTo(0, _ => "zero")` | | Run rule only when entity condition met | `.WhenEntity(e => e.StartDate.HasValue)` chained after the rule | | Run rule only when property value condition met | `.WhenValue(v => v != null)` chained after the rule (predicate on property value, not entity — use `WhenEntity` for entity conditions) | -| Skip when dependent prop invalid | `.DependsOn(x => x.OtherProp)` at end of chain | +| Skip when dependent prop invalid/default | `.DependsOn(x => x.OtherProp)` at end of chain | +| Run only when a valid dependency also meets a condition | `.DependsOn(x => x.OtherProp, value => predicate)` — the predicate is evaluated only after the dependency is non-default and has no validation error | | Compare to another property on the same entity | `.CompareProperty(CompareOperator.GreaterThanOrEqualTo, x => x.StartsOn)` — compares only when both have a value and are valid; add `.Mandatory()` first to require it | | Runtime-computed threshold | `.LessThanOrEqualTo(_ => DateOnly.FromDateTime(Runtime.UtcNow.UtcDateTime.AddYears(-16)), _ => "the minimum age of 16")` | +When a property's rules are conditional on the dependent property's value, use the predicate overload instead of repeating the dependency/null condition in `WhenEntity`: + +```csharp +Property(x => x.FareClass) + .Mandatory() + .DependsOn(x => x.SegmentType, segmentType => segmentType!.IsFlight); +``` + +Here `.Mandatory()` runs only when `SegmentType` is non-default, has no validation error, and `IsFlight` is true; clauses apply to the rule they are attached to. For a nullable dependency, `!` is justified because the predicate runs only after the runtime guard; it only affects nullable analysis and does not change validation behavior. Use `WhenEntity(...)` when the condition depends on other entity state; retain `DependsOn(...)` as a separate guard when you need its invalid-dependency short-circuit behavior. + +Call `Property(x => x.SameProperty)` again to start a separate root rule chain for the same property when another rule needs a different condition. This is not an isolated validation transaction: if an earlier chain reports an error for that property, later chains may be skipped by the existing-error guard. + **Mandatory() on non-nullable value types:** `Mandatory()` errors when the value equals `default(T)` — `0` for `decimal`/`int`, `DateOnly.MinValue` for `DateOnly`. If `0` is a legitimate value, do **not** use `Mandatory()` — use `.GreaterThanOrEqualTo(0)` (allows `0`) or `.GreaterThan(0)` (requires positive) instead. diff --git a/CHANGELOG.md b/CHANGELOG.md index 56355bf54..a4f220b3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,12 @@ Represents the **NuGet** versions. - *Enhancement:* Added host-builder `AddNamedHttpClient()` and its `AddHttpClient()` alias, sharing idempotency key handling, service discovery and standard resilience with `AddTypedHttpClient()`. A separate configuration-section overload supports augmenting SDK-owned named clients without replacing their registrations; base addresses may also be supplied by an existing registration or configuration callback. - *Enhancement:* Added `IEfDb.SqlQuery(SqlStatement, params object[])` for deferred, parameterized, composable EF queries over scalar or unmapped result types. Queries use the existing context and transaction, not the CoreEx Database command pipeline; CoreEx tenant and logical-delete filters are not automatically applied. - *Enhancement:* Added `ReferenceDataOrchestrator.PrefetchAsync(IEnumerable)` for typed reference-data prefetch, validating all requested types before loading, deduplicating them and honoring configured parallelism. Name-based prefetch now also checks cancellation immediately. +- *Enhancement:* Added a predicate overload for validation `DependsOn`, allowing a rule to run only when its dependency is non-default, has no validation error, and satisfies an additional condition. +- *Enhancement:* Updated the shipped AI guidance and reference-data documentation to explain implicit conversion from reference-data codes to concrete typed values, the annotated contract code-property pattern, and the separation of conversion from validation. - *Enhancement:* Added OData-style free-text `$search` support to the `QueryArgs` querying pipeline: `QueryArgs.Search`; `QueryArgsConfig.WithSearch(Action)` with `QuerySearchMode` (`Keyword` executed by `QueryExtensions.Where` as a provider-agnostic predicate where every term must match a configured keyword field; `FullText`/`Semantic`/`Hybrid` are relevance-ranked and executed by the repository via `QueryArgsParseResult.IsRankedSearch`/`SearchResult.Text`, with `$orderby` rejected); `QuerySearchParserException`; `WithSearch` on `HttpRequestMessage`/`HttpRequestBuilder`; `QueryAttribute.SupportsSearch` (NSwag `$search` parameter); and a GraphQL-lite `search` argument on list roots. `$search` is rejected where not configured. The Products sample (and its MCP `search_products` tool) now uses `$search` to demonstrate. - *Behavior change:* Added a string-specific `Required()` validation overload that rejects null, empty and whitespace values with a `ValidationException`. Existing string call sites now reject empty/whitespace strings rather than only null. - *Maintenance:* Updated framework dependencies and corresponding `CoreEx.Template` dependencies, including Aspire 13.6.1, EF/.NET servicing packages and FusionCache 2.9.0. The SQL Server package now explicitly references `System.Security.Cryptography.Pkcs` 10.0.13; the Aspire template's MessagePack override is updated to 2.5.307. +- *Maintenance:* Pinned local development, CI and package publishing to the stable .NET SDK in `global.json`; updated `nuget-publish.ps1` to build before packing and remove matching package versions from the local NuGet cache by default. - *Fixed:* Named/typed HTTP client registration now binds standard resilience settings from the nested `Resilience` configuration section rather than the parent client section. - *Fixed:* GraphQL-lite search parsing errors now preserve actionable per-property validation details in `extensions.messages["$search"]` while retaining the `SEARCH_PARSE_ERROR` code, including unsupported search, length/term limits and custom validation failures. - *Fixed:* `CoreEx.UnitTesting` and the sample/template MockHost now explicitly reference MessagePack 2.5.307 so their transitive Aspire dependency cannot resolve the vulnerable 2.5.302 version; a central version entry alone does not pin transitive dependencies. diff --git a/samples/docs/application-layer.md b/samples/docs/application-layer.md index e659f33e7..0e0a59f99 100644 --- a/samples/docs/application-layer.md +++ b/samples/docs/application-layer.md @@ -148,6 +148,16 @@ public class MovementRequestValidator : Validator } ``` +When a property's rules depend on a condition of the dependency itself, use the predicate overload to combine the non-default/valid dependency guard with that condition: + +```csharp +Property(x => x.FareClass) + .Mandatory() + .DependsOn(x => x.SegmentType, segmentType => segmentType!.IsFlight); +``` + +This makes `FareClass` mandatory only when `SegmentType` is valid and represents a flight. For conditions based on other entity properties, use `WhenEntity(...)`; the `DependsOn` predicate is intentionally scoped to the dependent property's value. + > **See also**: [`Validator`](../../src/CoreEx.Validation/Validator.cs) · [`IValidatorEx`](../../src/CoreEx.Validation/IValidatorEx.cs) --- diff --git a/src/CoreEx.Validation/AGENTS.md b/src/CoreEx.Validation/AGENTS.md index 6a6ad8a70..0d86e3af0 100644 --- a/src/CoreEx.Validation/AGENTS.md +++ b/src/CoreEx.Validation/AGENTS.md @@ -67,8 +67,15 @@ Property(p => p.DiscountCode) Property(p => p.ExpiresOn) .GreaterThan(_ => Runtime.UtcNow) .WhenHasValue(); + +// FareClass is required only when its valid dependency is a flight. +Property(p => p.FareClass) + .Mandatory() + .DependsOn(p => p.SegmentType, segmentType => segmentType?.IsFlight == true); ``` +The `DependsOn` predicate overload is evaluated only after the dependency has a non-default value and no validation error. Clauses guard the rule they are attached to, so `Mandatory()` is conditional in this example. Use another `Property(p => p.SameProperty)` call to add a separate root rule chain for the same property when different rules need different conditions; later chains may be skipped if an earlier one has already reported an error for that property. Use `WhenEntity(...)` when the condition depends on broader entity state; use `!` for nullable dependencies when relying on this runtime guard, because nullable flow analysis cannot infer it. + ### Comparing two properties Use `CompareProperty` to compare a property against another property on the **same entity** — do not hand-write it in `OnValidateAsync`. The comparison is skipped when either value is `null` or the compare-to property already has an error, so it only runs when both are present and valid. Chain `Mandatory()` first where the property must also be supplied. diff --git a/src/CoreEx.Validation/Clauses/DependsOnClause.cs b/src/CoreEx.Validation/Clauses/DependsOnClause.cs index faa79ee90..ad9bce8a2 100644 --- a/src/CoreEx.Validation/Clauses/DependsOnClause.cs +++ b/src/CoreEx.Validation/Clauses/DependsOnClause.cs @@ -11,6 +11,11 @@ public sealed class DependsOnClause(Expr { private readonly IPropertyRuntimeMetadata _dependsOn = RuntimeMetadata.GetForExpression(dependsOnExpression.ThrowIfNull()); + /// + /// Gets or initializes a predicate that determines whether the dependent property satisfies an additional condition. + /// + internal Func? AndAlso { get; init; } + /// public Task CheckAsync(PropertyContext context, CancellationToken cancellationToken) { @@ -23,6 +28,6 @@ public Task CheckAsync(PropertyContext context, Cancel return Task.FromResult(false); // Check depends on value to continue. - return Task.FromResult(!_dependsOn.IsDefault(context.Entity)); + return Task.FromResult(!_dependsOn.IsDefault(context.Entity) && AndAlso?.Invoke(_dependsOn.GetValue(context.Entity)) != false); } -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/Clauses/README.md b/src/CoreEx.Validation/Clauses/README.md index 08ce8b6a4..7a26dd4ab 100644 --- a/src/CoreEx.Validation/Clauses/README.md +++ b/src/CoreEx.Validation/Clauses/README.md @@ -11,7 +11,7 @@ Clauses are never instantiated directly by consumers. They are created and attac ## Key capabilities - ⏸ **Boolean/predicate guard (`WhenClause`)**: skips the rule when a static `bool`, a `Func`, a context-aware predicate, or a fully async `PredicateAsync` evaluates to `false`; covers entity-level, property-value-level, and presence-check variants. -- 🔗 **Sibling-property guard (`DependsOnClause`)**: skips the rule when a specified sibling property has its default value _or_ has already produced a validation error, preventing cascading errors when an upstream field is missing. +- 🔗 **Sibling-property guard (`DependsOnClause`)**: skips the rule when a specified sibling property has its default value _or_ has already produced a validation error; an optional predicate can further condition execution on that property's value. - ⚡ **Async-native**: both clause types are fully async; `WhenClause` accepts a `PredicateAsync` delegate so external lookups can be awaited inside a clause without blocking. ## Key types @@ -21,7 +21,7 @@ Clauses are never instantiated directly by consumers. They are created and attac | [IPropertyClause](./IPropertyClauseT.cs) | Non-generic-property clause contract; defines `CheckAsync(IPropertyContext, CancellationToken)`; used when clause implementation does not need the resolved property type. | | [IPropertyClause](./IPropertyClauseT2.cs) | Fully typed clause contract; defines `CheckAsync(PropertyContext, CancellationToken)`; the standard implementation target for built-in clauses. | | **[`WhenClause`](./WhenClause.cs)** | Evaluates a `PredicateAsync` delegate; returns `false` (skip rule) when the delegate returns `false`. Wired via `When()`, `WhenValue()`, `WhenHasValue()`, `WhenEntity()`. | -| **[`DependsOnClause`](./DependsOnClause.cs)** | Returns `false` (skip rule) when the depends-on sibling property is at its default value or already carries a validation error. Wired via `DependsOn(expression)`. | +| **[`DependsOnClause`](./DependsOnClause.cs)** | Returns `false` (skip rule) when the depends-on sibling property is at its default value or already carries a validation error; with a predicate, also skips when the predicate returns `false`. Wired via `DependsOn(expression)` and `DependsOn(expression, predicate)`. | ## Related namespaces diff --git a/src/CoreEx.Validation/README.md b/src/CoreEx.Validation/README.md index f268a8c53..f5a141627 100644 --- a/src/CoreEx.Validation/README.md +++ b/src/CoreEx.Validation/README.md @@ -52,7 +52,7 @@ Clauses are conditional guards that run _before_ the rule body. When a clause re | Clause Class | Description | Extensions | |---|---|---| -| **[`DependsOnClause`](./Clauses/DependsOnClause.cs)** | Skips the rule when a specified sibling property has its default value or has already reported a validation error. | `DependsOn(Expression>)` | +| **[`DependsOnClause`](./Clauses/DependsOnClause.cs)** | Skips the rule when a specified sibling property has its default value or has already reported a validation error; an optional predicate can further condition execution on that property's value. | `DependsOn(expression)`, `DependsOn(expression, predicate)` | | **[`WhenClause`](./Clauses/WhenClause.cs)** | Skips the rule when a boolean condition or async predicate evaluates to `false`. | `When(bool)`, `When(Func)`, `When(Func)`, `When(PredicateAsync)`, `WhenValue(Predicate)`, `WhenHasValue()`, `WhenEntity(Predicate)` | ## Rules diff --git a/src/CoreEx.Validation/ValidationExtensions.DependsOnClause.cs b/src/CoreEx.Validation/ValidationExtensions.DependsOnClause.cs index 994e6ae8b..a14659bca 100644 --- a/src/CoreEx.Validation/ValidationExtensions.DependsOnClause.cs +++ b/src/CoreEx.Validation/ValidationExtensions.DependsOnClause.cs @@ -14,4 +14,18 @@ public static partial class ValidationExtensions /// Represents a depends on clause; in that specified property () of the entity must have a non-default value, and not have a validation error, to continue. public static IPropertyRule DependsOn(this IPropertyRule rule, Expression> dependsOnPropertyExpression) where TEntity : class => AddClause(rule, new DependsOnClause(dependsOnPropertyExpression.ThrowIfNull())); -} \ No newline at end of file + + /// + /// Adds a depends on expression with an additional predicate () clause to the existing . + /// + /// The entity . + /// The property . + /// The depends on property . + /// The being extended. + /// The depends on property expression. + /// The predicate evaluated when the dependent property has a non-default value and no validation error; returning skips the rule. + /// The to support fluent-style method-chaining. + /// The specified property must have a non-default value and no validation error, and must return , for the rule to continue. + public static IPropertyRule DependsOn(this IPropertyRule rule, Expression> dependsOnPropertyExpression, Func andAlso) where TEntity : class + => AddClause(rule, new DependsOnClause(dependsOnPropertyExpression.ThrowIfNull()) { AndAlso = andAlso.ThrowIfNull() }); +} diff --git a/tests/CoreEx.Validation.Test.Unit/Clauses/DependsOnClauseTests.cs b/tests/CoreEx.Validation.Test.Unit/Clauses/DependsOnClauseTests.cs index 373014b9d..3f6d22027 100644 --- a/tests/CoreEx.Validation.Test.Unit/Clauses/DependsOnClauseTests.cs +++ b/tests/CoreEx.Validation.Test.Unit/Clauses/DependsOnClauseTests.cs @@ -15,9 +15,90 @@ public void DependsOn() v.ValidateAsError(new Person { FirstName = "Joh", LastName = "Doe" }, "firstName", "First name must be exactly 4 character(s) in length."); } + [Test] + public void DependsOn_WhenPredicateIsTrue_ValidatesTheProperty() + { + var v = Validator.Create() + .HasProperty(p => p.FareClass, c => c.Mandatory().DependsOn(p => p.SegmentType, s => s!.IsFlight)); + + v.ValidateAsError(new Person { SegmentType = new SegmentType { IsFlight = true } }, "fareClass", "Fare class is required."); + v.ValidateAsSuccess(new Person { SegmentType = new SegmentType { IsFlight = true }, FareClass = "Y" }); + } + + [Test] + public void DependsOn_WhenPredicateIsFalse_SkipsThePropertyRules() + { + var v = Validator.Create() + .HasProperty(p => p.FareClass, c => c.Mandatory().DependsOn(p => p.SegmentType, s => s!.IsFlight)); + + v.ValidateAsSuccess(new Person { SegmentType = new SegmentType { IsFlight = false } }); + } + + [Test] + public void DependsOn_AndWhenEntityClausesAreCombined() + { + var v = Validator.Create() + .HasProperty(p => p.FareClass, c => c.Mandatory() + .WhenEntity(p => p.IsEnabled) + .DependsOn(p => p.SegmentType, s => s!.IsFlight)); + + v.ValidateAsError(new Person { IsEnabled = true, SegmentType = new SegmentType { IsFlight = true } }, "fareClass", "Fare class is required."); + v.ValidateAsSuccess(new Person { IsEnabled = false, SegmentType = new SegmentType { IsFlight = true } }); + v.ValidateAsSuccess(new Person { IsEnabled = true, SegmentType = new SegmentType { IsFlight = false } }); + } + + [Test] + public void DependsOn_WhenDependencyIsMissing_SkipsPredicateAndPropertyRules() + { + var predicateInvoked = false; + var v = Validator.Create() + .HasProperty(p => p.FareClass, c => c.Mandatory().DependsOn(p => p.SegmentType, s => + { + predicateInvoked = true; + return s!.IsFlight; + })); + + v.ValidateAsSuccess(new Person()); + predicateInvoked.Should().BeFalse(); + } + + [Test] + public void DependsOn_WhenDependencyHasValidationError_SkipsPredicateAndPropertyRules() + { + var predicateInvoked = false; + var v = Validator.Create() + .HasProperty(p => p.SegmentType, c => c.Error("Segment type is invalid.").WhenValue(value => value != null)) + .HasProperty(p => p.FareClass, c => c.Mandatory().DependsOn(p => p.SegmentType, s => + { + predicateInvoked = true; + return s!.IsFlight; + })); + + v.ValidateAsError(new Person { SegmentType = new SegmentType { IsFlight = true } }, "segmentType", "Segment type is invalid."); + predicateInvoked.Should().BeFalse(); + } + + [Test] + public void DependsOn_WhenPredicateIsNull_Throws() + { + var v = Validator.Create(); + var ex = Assert.Throws(() => v.HasProperty(p => p.FareClass, + c => c.Mandatory().DependsOn(p => p.SegmentType, (Func)null!))); + + ex!.ParamName.Should().Be("andAlso"); + } + private class Person { public string? FirstName { get; set; } public string? LastName { get; set; } + public string? FareClass { get; set; } + public SegmentType? SegmentType { get; set; } + public bool IsEnabled { get; set; } + } + + private class SegmentType + { + public bool IsFlight { get; set; } } -} \ No newline at end of file +} From f38f89b49e33d2888ead8fadfcb4f96b0be1615d Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Wed, 21 Oct 2026 15:29:28 -0700 Subject: [PATCH 21/25] Stabilize outbox relay tests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .github/instructions/coreex-tests.instructions.md | 10 ++++++---- .github/skills/coreex-test-relay/SKILL.md | 10 ++++++---- .../tests/Contoso.Products.Test.Relay/RelayTests.cs | 11 ++++++----- .../tests/Contoso.Shopping.Test.Relay/RelayTests.cs | 11 ++++++----- .../tests/solution-name.Test.Relay/RelayTests.cs | 12 ++++++------ 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/.github/instructions/coreex-tests.instructions.md b/.github/instructions/coreex-tests.instructions.md index b2c674626..621313af3 100644 --- a/.github/instructions/coreex-tests.instructions.md +++ b/.github/instructions/coreex-tests.instructions.md @@ -885,11 +885,13 @@ public class RelayTests : WithApiTester pub.Add("contoso", [ce1, ce2]); await pub.PublishAsync(); - for (int i = 0; i < 5; i++) + var list = new List(); + for (int i = 0; i < 30 && list.Count < 2; i++) + { await Task.Delay(TimeSpan.FromSeconds(1)); - - var list = await Test.GetAndClearAzureServiceBusAsync( - ServiceBusSessionReceiverOptions.CreateForTopicSubscription("contoso", "products")); + list.AddRange(await Test.GetAndClearAzureServiceBusAsync( + ServiceBusSessionReceiverOptions.CreateForTopicSubscription("contoso", "products"))); + } list.Should().HaveCount(2); }).AssertSuccess(); diff --git a/.github/skills/coreex-test-relay/SKILL.md b/.github/skills/coreex-test-relay/SKILL.md index b517ea6a1..d8730ca3f 100644 --- a/.github/skills/coreex-test-relay/SKILL.md +++ b/.github/skills/coreex-test-relay/SKILL.md @@ -67,11 +67,13 @@ public class RelayTests : WithApiTester pub.Add("{solution}", [ce1, ce2]); await pub.PublishAsync(); - for (int i = 0; i < 5; i++) + var list = new List(); + for (int i = 0; i < 30 && list.Count < 2; i++) + { await Task.Delay(TimeSpan.FromSeconds(1)); - - var list = await Test.GetAndClearAzureServiceBusAsync( - ServiceBusSessionReceiverOptions.CreateForTopicSubscription("{solution}", "{domain}")); + list.AddRange(await Test.GetAndClearAzureServiceBusAsync( + ServiceBusSessionReceiverOptions.CreateForTopicSubscription("{solution}", "{domain}"))); + } list.Should().HaveCount(2); }).AssertSuccess(); diff --git a/samples/tests/Contoso.Products.Test.Relay/RelayTests.cs b/samples/tests/Contoso.Products.Test.Relay/RelayTests.cs index 1c0853a8b..7a3d4a313 100644 --- a/samples/tests/Contoso.Products.Test.Relay/RelayTests.cs +++ b/samples/tests/Contoso.Products.Test.Relay/RelayTests.cs @@ -28,12 +28,13 @@ public void Outbox_Relay() pub.Add("contoso", [ce1, ce2]); await pub.PublishAsync(); - // Hosted-service(s) are currently running and should relay to Azure Service Bus; we just need to give it a few seconds to do so. - for (int i = 0; i < 5; i++) + // Poll for delivery because relay polling can vary under load. + var list = new List(); + for (int i = 0; i < 30 && list.Count < 2; i++) + { await Task.Delay(TimeSpan.FromSeconds(1)); - - // Receive the events from Azure Service Bus and assert. - var list = await Test.GetAndClearAzureServiceBusAsync(ServiceBusSessionReceiverOptions.CreateForTopicSubscription("contoso", "products")); + list.AddRange(await Test.GetAndClearAzureServiceBusAsync(ServiceBusSessionReceiverOptions.CreateForTopicSubscription("contoso", "products"))); + } list.Should().NotBeNull().And.HaveCount(2); var ce1Msg = list.Should().ContainSingle(x => x.MessageId == ce1.Id).Subject; diff --git a/samples/tests/Contoso.Shopping.Test.Relay/RelayTests.cs b/samples/tests/Contoso.Shopping.Test.Relay/RelayTests.cs index 3685b352e..62004d0cb 100644 --- a/samples/tests/Contoso.Shopping.Test.Relay/RelayTests.cs +++ b/samples/tests/Contoso.Shopping.Test.Relay/RelayTests.cs @@ -28,12 +28,13 @@ public void Outbox_Relay() pub.Add("contoso", [ce1, ce2]); await pub.PublishAsync(); - // Hosted-service(s) are currently running and should relay to Azure Service Bus; we just need to give it a few seconds to do so. - for (int i = 0; i < 5; i++) + // Poll for delivery because relay polling and transient SQL lock retries can exceed a fixed sleep. + var list = new List(); + for (int i = 0; i < 30 && list.Count < 2; i++) + { await Task.Delay(TimeSpan.FromSeconds(1)); - - // Receive the events from Azure Service Bus and assert. - var list = await Test.GetAndClearAzureServiceBusAsync(ServiceBusSessionReceiverOptions.CreateForTopicSubscription("contoso", "shopping")); + list.AddRange(await Test.GetAndClearAzureServiceBusAsync(ServiceBusSessionReceiverOptions.CreateForTopicSubscription("contoso", "shopping"))); + } list.Should().NotBeNull().And.HaveCount(2); var ce1Msg = list.Should().ContainSingle(x => x.MessageId == ce1.Id).Subject; diff --git a/src/CoreEx.Template/content/CoreEx.Relay/tests/solution-name.Test.Relay/RelayTests.cs b/src/CoreEx.Template/content/CoreEx.Relay/tests/solution-name.Test.Relay/RelayTests.cs index 362a1c5e0..d53b33cd9 100644 --- a/src/CoreEx.Template/content/CoreEx.Relay/tests/solution-name.Test.Relay/RelayTests.cs +++ b/src/CoreEx.Template/content/CoreEx.Relay/tests/solution-name.Test.Relay/RelayTests.cs @@ -44,14 +44,14 @@ public void Outbox_Relay() pub.Add("domain-parent-lower", [ce1, ce2]); await pub.PublishAsync(); - // Hosted-service(s) are currently running and should relay to Azure Service Bus; we just need to give it a few seconds to do so. - for (int i = 0; i < 5; i++) + // Poll for delivery because relay polling and transient database retries can exceed a fixed sleep. + var list = new List(); + for (int i = 0; i < 30 && list.Count < 2; i++) + { await Task.Delay(TimeSpan.FromSeconds(1)); + list.AddRange(await Test.GetAndClearAzureServiceBusAsync(ServiceBusSessionReceiverOptions.CreateForTopicSubscription("domain-parent-lower", "domain-name-lower"))); + } - // Receive the events from Azure Service Bus and assert (order is not guaranteed as different sessions). - var list = await Test.GetAndClearAzureServiceBusAsync(ServiceBusSessionReceiverOptions.CreateForTopicSubscription("domain-parent-lower", "domain-name-lower")); - - list.Should().NotBeNull().And.HaveCount(2); list.Should().NotBeNull().And.HaveCount(2); var ce1Msg = list.Should().ContainSingle(x => x.MessageId == ce1.Id).Subject; var ce2Msg = list.Should().ContainSingle(x => x.MessageId == ce2.Id).Subject; From 9a598df4bba8831f0acba515a6d6ec96175b7be5 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Thu, 22 Oct 2026 15:10:10 -0700 Subject: [PATCH 22/25] Improve validation metadata and preserve mutation outcomes Add expression-based validation factories and cached metadata fallbacks, distinguish request validation from domain business errors, and preserve DataResult mutation flags through mapping and event publication. Align samples, regression coverage, and AI guidance. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .../coreex-api-controllers.instructions.md | 2 + ...oreex-application-services.instructions.md | 55 ++++- .../coreex-domain.instructions.md | 32 ++- .../coreex-repositories.instructions.md | 12 +- .../instructions/coreex-tests.instructions.md | 6 +- .../coreex-validators.instructions.md | 10 + .github/skills/coreex-adapter/SKILL.md | 1 + .../coreex-adapter/references/workflow.md | 2 +- .github/skills/coreex-aggregate/SKILL.md | 3 + .../coreex-aggregate/references/workflow.md | 28 +-- .github/skills/coreex-app-service/SKILL.md | 4 +- .../coreex-app-service/references/workflow.md | 34 +-- .github/skills/coreex-policy/SKILL.md | 10 +- .../coreex-policy/references/workflow.md | 53 +++-- .github/skills/coreex-repository/SKILL.md | 2 +- .../coreex-repository/references/workflow.md | 27 ++- .github/skills/coreex-test-api/SKILL.md | 1 + .../coreex-test-api/references/workflow.md | 1 + .github/skills/coreex-validator/SKILL.md | 8 + CHANGELOG.md | 10 + docs/capabilities.md | 4 +- samples/docs/application-layer.md | 27 ++- samples/docs/domain-layer.md | 38 ++- samples/docs/infrastructure-layer.md | 2 + .../BasketService.cs | 32 ++- .../Repositories/IBasketRepository.cs | 6 +- samples/src/Contoso.Shopping.Domain/Basket.cs | 5 +- .../ValueObjects/ItemPricing.cs | 21 +- .../Repositories/BasketRepository.cs | 8 +- .../Domains/BasketTests.cs | 33 +++ .../Domains/ItemPricingTests.cs | 53 +++++ .../Contoso.Shopping.Test.Unit/GlobalUsing.cs | 20 +- .../Services/BasketMutationTests.cs | 152 ++++++++++++ src/CoreEx.Data/AGENTS.md | 9 +- src/CoreEx.Data/DataResultT.cs | 9 + src/CoreEx.Data/README.md | 14 +- src/CoreEx.DomainDriven/AGENTS.md | 12 +- src/CoreEx.DomainDriven/README.md | 10 +- src/CoreEx.Validation/AGENTS.md | 2 + .../Abstractions/InlineValidator.cs | 9 +- src/CoreEx.Validation/GlobalUsing.cs | 3 +- src/CoreEx.Validation/README.md | 6 + src/CoreEx.Validation/Rules/DictionaryRule.cs | 14 +- .../ValidatingInlineValidator.cs | 6 +- src/CoreEx/AGENTS.md | 16 +- src/CoreEx/Entities/MessageItem.Create.cs | 26 +- src/CoreEx/Metadata/README.md | 12 +- .../Metadata/RuntimeMetadata.GetJsonPath.cs | 116 +++++++++ src/CoreEx/Metadata/RuntimeMetadata.cs | 88 ++++--- src/CoreEx/README.md | 21 ++ src/CoreEx/Results/README.md | 16 +- src/CoreEx/Results/Result.Error.cs | 13 +- src/CoreEx/Results/ResultT.Error.cs | 13 +- src/CoreEx/Validation/Validation.cs | 22 +- src/CoreEx/ValidationException.cs | 50 +++- .../CoreEx.Data.Test.Unit/DataResultTests.cs | 80 +++++++ tests/CoreEx.RefData.Test.Unit/GlobalUsing.cs | 6 + .../ReferenceDataMetadataValidationTests.cs | 58 +++++ .../Entities/MessageItemExpressionTests.cs | 58 +++++ tests/CoreEx.Test.Unit/GlobalUsing.cs | 5 + .../Results/ValidationErrorExpressionTests.cs | 94 ++++++++ .../Runtime/MetadataSourceTests.cs | 135 +++++++++++ .../ValidationExceptionCreateTests.cs | 225 ++++++++++++++++++ .../JsonPathParityTests.cs | 109 +++++++++ 64 files changed, 1767 insertions(+), 192 deletions(-) create mode 100644 samples/tests/Contoso.Shopping.Test.Unit/Domains/ItemPricingTests.cs create mode 100644 samples/tests/Contoso.Shopping.Test.Unit/Services/BasketMutationTests.cs create mode 100644 src/CoreEx/Metadata/RuntimeMetadata.GetJsonPath.cs create mode 100644 tests/CoreEx.Data.Test.Unit/DataResultTests.cs create mode 100644 tests/CoreEx.RefData.Test.Unit/GlobalUsing.cs create mode 100644 tests/CoreEx.RefData.Test.Unit/ReferenceDataMetadataValidationTests.cs create mode 100644 tests/CoreEx.Test.Unit/Entities/MessageItemExpressionTests.cs create mode 100644 tests/CoreEx.Test.Unit/Results/ValidationErrorExpressionTests.cs create mode 100644 tests/CoreEx.Test.Unit/Runtime/MetadataSourceTests.cs create mode 100644 tests/CoreEx.Test.Unit/ValidationExceptionCreateTests.cs create mode 100644 tests/CoreEx.Validation.Test.Unit/JsonPathParityTests.cs diff --git a/.github/instructions/coreex-api-controllers.instructions.md b/.github/instructions/coreex-api-controllers.instructions.md index 6398b0bf3..f68f9b5bc 100644 --- a/.github/instructions/coreex-api-controllers.instructions.md +++ b/.github/instructions/coreex-api-controllers.instructions.md @@ -120,6 +120,8 @@ public Task GetAsync(string id, CancellationToken cancellationTok Do not use `.ThrowIfNull()` / `.ThrowIfNullOrEmpty()` on route parameters — those throw `ArgumentNullException`, which results in a 500 rather than a 400. +Classify lookup failures by the identifier's role, not URI presence: a missing target resource or explicitly addressed child is 404; a missing request-body reference is a field-level 400 validation error. Unmatched collection filters normally return empty results, not 404. Application services/policies interpret expected lookup outcomes; controllers preserve that classification through `WebApi`. Retain deliberate idempotent-delete and security-concealment semantics. See [Not Found Handling](/.github/instructions/coreex-application-services.instructions.md#not-found-handling). + ### POST — Create with Location Header Use `ro.WithLocationUri(...)` to set the `Location` response header: diff --git a/.github/instructions/coreex-application-services.instructions.md b/.github/instructions/coreex-application-services.instructions.md index dab0f9fb2..0ff39eef7 100644 --- a/.github/instructions/coreex-application-services.instructions.md +++ b/.github/instructions/coreex-application-services.instructions.md @@ -170,15 +170,37 @@ if (result.IsFailure) return result.AsResult(); ## Not Found Handling -After loading an entity, throw immediately if it does not exist: +Choose the error according to the identifier's role in the operation, not merely because a repository or downstream API returned "not found": + +- **Missing target resource:** use `NotFoundException` / `Result.NotFoundError` (HTTP 404) for the primary resource or an explicitly addressed child, such as `/bookings/{id}` or `/bookings/{id}/travellers/{travellerId}`. +- **Invalid reference:** use a field-level `ValidationException` / `Result.ValidationError` (HTTP 400 under CoreEx conventions) when a request-body property refers to a nonexistent entity. An unknown `travellerId` in `POST /bookings/{id}/travellers` is an invalid reference; a missing target booking is a 404. +- **Query inputs:** URI presence alone does not imply a target resource. An unmatched collection filter normally returns an empty result; invalid query inputs follow the endpoint's validation contract. +- **Downstream failures:** translate only an expected entity-not-found outcome for an invalid reference; preserve other failures unchanged. A downstream HTTP 404 can also indicate an incorrect route or deployment/configuration problem. Do not automatically turn every downstream 404 into caller validation. +- **State/process restrictions:** use business errors where appropriate, with specific conflict/concurrency or authorization semantics taking precedence. Do not use business errors or choose HTTP 404 merely to retain a machine-readable error code. +- **Endpoint semantics:** preserve deliberate behavior, including idempotent deletion and security-driven concealment of resource existence. + +For a missing target resource: ```csharp var current = await _repository.GetAsync(id, cancellationToken).ConfigureAwait(false); NotFoundException.ThrowIfDefault(current); ``` +For an expected missing reference returned as `Result`, translate only that failure: + +```csharp +var traveller = await _travellerAdapter.GetAsync(request.TravellerId, cancellationToken).ConfigureAwait(false); +return traveller.IsNotFoundError + ? Result.ValidationError(x => x.TravellerId, "Traveller was not found.") + : traveller; +``` + +The expression type is the request contract containing the invalid reference; the result value type is the entity returned on success. For a nullable lookup, check null and use the same field-error factory instead. Use `ValidationException.Create(...)` in exception-style services, `MessageItem.CreateErrorMessage(...)` for aggregation, and `context.AddError(...)` inside validators to preserve contextual paths. Reusable policies without a known request contract may accept an already-resolved JSON path; do not introduce a DTO dependency solely for an expression. `nameof(param)` alone does not resolve JSON names. + ## Business Rule Exceptions +Request/input validity (including invalid references) uses validation errors with request-field context where known. Aggregate/entity/value-object business invariants use business errors with domain messages/codes, not request paths; Domain must not reference request DTOs or infer their JSON paths. Both normally map to 400, but are distinct contracts. Keep domain enforcement after request validation. Application may selectively translate a known domain error to a field error only when the mapping is unambiguous; never blanket-convert business failures. See [domain invariant conventions](/.github/instructions/coreex-domain.instructions.md#invariant-failures-versus-request-validation). + Use `BusinessException` for domain rule violations that are the caller's fault but are not validation errors: ```csharp @@ -275,6 +297,7 @@ return await _unitOfWork.TransactionAsync(async ct => ``` - `WhereMutated(action)` — executes `action` only when the data result records a mutation; add the event inside this callback. **Mind the overload:** `DataResult` (from `Create`/`Update`) carries the value, so use `WhereMutated(v => ...)`; `DataResult` (from `Delete`) has **no value**, so use the parameterless `WhereMutated(() => ...)` — not `WhereMutated(_ => ...)`. +- **Success is not evidence of mutation.** Preserve `DataResult` through repository/ROP/domain mapping until mutation-dependent decisions are complete. A successful `WasMutated == false` returns success without a mutation-announcement event; use `WhereMutated` inside the same transaction as the write, then unwrap for the public response. An earlier successful read or aggregate `HasChanges`/`PersistenceState` check does not replace the persistence outcome. The flag is the provider's mutation signal, potentially before deferred writes/commit; it is not an exactly-once or committed-row guarantee. Commands and deliberate attempt notifications have separate semantics, so do not mechanically gate them as mutation events. - `EventData.CreateEventWith(value, action)` — a typed event **carrying the entity value** (Create/Update only — pass the **real** mutated value `v`). For a **no-value** event (Delete), use `EventData.CreateEvent(action).WithKey(id)` — the type + action + key, **no value**. **Never fabricate a value to feed `CreateEventWith` on a delete** (e.g. `CreateEventWith(new Employee { Id = id }, …)` or `CreateEventWith(default, …)`): a delete event must have **no body** — a synthetic entity wrongly serialises a near-empty value, adds a version suffix that delete events must not have, and buries the id in the body instead of the metadata key. The id belongs in `.WithKey(id)`. - `EventAction.Created`, `EventAction.Updated`, `EventAction.Deleted` — use the standard constants. @@ -312,16 +335,36 @@ public Task> CreateAsync(string customerId, CancellationToken can return _unitOfWork.TransactionAsync(async ct => { var br = await _repository.CreateAsync(aggregate, ct).ConfigureAwait(false); - return br.ThenAs(b => + return br.ThenAs(dr => { - var contract = BasketMapper.Map(b); - _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.Created)); + var contract = BasketMapper.Map(dr.Value); + dr.WhereMutated(_ => _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.Created))); return contract; }); }, cancellationToken); } ``` +The repository above returns `Result>` from Create/Update, preserving its provider's flag when mapping persistence models: `dr.Map(value => BasketMapper.Map(value))`. `Map` transforms even a no-op's value without changing `WasMutated`; the lambda retains the CoreEx mapper's conditional non-null guarantee. Reads remain `Result`; delete returns `Result`. For a repository with an idempotent delete: + +```csharp +public Task DeleteAsync(string id, CancellationToken cancellationToken = default) +{ + id.ThrowIfNullOrEmpty(); + return _unitOfWork.TransactionAsync(async ct => + { + var result = await _repository.DeleteAsync(id, ct).ConfigureAwait(false); + return result.ThenAs(dr => + { + dr.WhereMutated(() => _unitOfWork.Events.Add(EventData.CreateEvent(EventAction.Deleted).WithKey(id))); + return Result.Success; + }); + }, cancellationToken); +} +``` + +If deletion needs a prior aggregate read to enforce business rules, keep it, but still gate the event on the successful delete result. Test the successful no-op outcome even after that read found the target. A sequential repeat-delete test does not cover the read/delete gap; depending on tracking/provider behavior, a concurrent deletion can instead produce a concurrency failure, which must propagate unchanged. + For multi-step orchestration with early exit on the first failure: ```csharp @@ -453,7 +496,7 @@ public class ProductPolicy(IProductAdapter productAdapter) public Task> EnsureExistsAsync(string productId, CancellationToken cancellationToken = default) => Result .GoAsync(() => _productAdapter.GetAsync(productId, cancellationToken)) .OnFailure(r => r.IsNotFoundError - ? Result.ValidationError(MessageItem.CreateErrorMessage(nameof(productId), "Product was not found.")) + ? Result.ValidationError(MessageItem.CreateErrorMessage("productId", "Product was not found.")) : r); } @@ -461,6 +504,8 @@ public class ProductPolicy(IProductAdapter productAdapter) var result = await new ProductPolicy(_productAdapter).EnsureExistsAsync(productId, cancellationToken); ``` +This policy checks a request-body reference whose agreed JSON path is `"productId"`; it does not define missing-target behavior. For a policy reused across fields, accept a resolved path from the caller. Apply [Not Found Handling](#not-found-handling) before choosing the failure type. + ## Application-Level Mapping Pass code strings directly to concrete typed reference-data parameters/properties; map DTO destinations through their annotated `XxxCode` properties. Do not add conversion-only `RefData` helpers or redundant null checks. See [Reference-Data Code Conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion); conversion is not validation. diff --git a/.github/instructions/coreex-domain.instructions.md b/.github/instructions/coreex-domain.instructions.md index c50400040..0ad5c66ba 100644 --- a/.github/instructions/coreex-domain.instructions.md +++ b/.github/instructions/coreex-domain.instructions.md @@ -27,7 +27,15 @@ The Domain layer is **optional**. It is introduced only when a domain contains a | Package | Key types provided | |---|---| | `CoreEx.DomainDriven` | `Aggregate`, `Entity`, `PersistenceState`, `.AsNew()`, `.AsNotModified()`, `.SetPersistenceState()` | -| `CoreEx` | `Result`, `Result`, `Result.GoAsync()`, `.ThenAs()`, `.ThenAsAsync()`, `Result.BusinessError()`, `Result.NotFoundError()`, `Result.ValidationError()`, `Runtime.NewId()`, `.ThrowIfNull()`, `.ThrowIfNullOrEmpty()`, `.ThrowIfInactive()`, `.ThrowIfLessThanZero()`, `ValidationException` | +| `CoreEx` | `Result`, `Result`, `Result.GoAsync()`, `.ThenAs()`, `.ThenAsAsync()`, `Result.BusinessError()`, `Result.NotFoundError()`, `BusinessException`, `Runtime.NewId()`, `.ThrowIfNull()`, `.ThrowIfNullOrEmpty()` | + +## Invariant Failures versus Request Validation + +Request/input validity belongs to Application validators/policies: use `ValidationException` / `Result.ValidationError` with request-field context where known. Aggregates, entities, and value objects protect business invariants using `BusinessException` / `Result.BusinessError`, with meaningful domain messages and stable error codes where useful, not request paths. Both normally map to HTTP 400, but their classification and diagnostic structure are different. Keep domain enforcement even after request validation: other callers, persisted state, and concurrent changes still require it. Specific not-found, conflict/concurrency, or authorization semantics remain distinct. + +Never reference request contracts or infer request JSON paths inside Domain, even if Contracts is already a project dependency. Application may deliberately translate a known domain error into a request-field error where the mapping is unambiguous; never blanket-convert business failures to validation. Exception versus `Result` is a separate choice: constructors/`init` setters can throw business exceptions; result-returning operations generally return business failures. `OnCheckCanMutate` failures still throw through `Modify`/`Remove` internally. + +Retain argument/invalid-operation guards for programming misuse or broken internal assumptions (for example, a required internal dependency/value is null). Do not automatically classify these as business failures. Conversely, do not use argument guards for expected business rejections: `ThrowIfLessThanZero` throws `ArgumentOutOfRangeException`; reference-data `ThrowIfInvalid`/`ThrowIfInactive` throw `InvalidOperationException`. These helpers are not business-error factories. Use explicit domain checks when negative amounts, invalid/inactive selections, or similar values are business invariants; do not change the shared helpers' exception semantics. ## Aggregates @@ -158,7 +166,7 @@ public sealed class BasketItem : Entity Keep mutation methods on child entities `internal` so they can only be invoked by the owning aggregate — never directly from the Application layer. -Guard methods (`.ThrowIfNull()`, `.ThrowIfNullOrEmpty()`, `.ThrowIfInactive()`, `.ThrowIfLessThanZero()`) **return the guarded value** when the check passes, making them natural for inline use in property setters, `init` expressions, and method calls — as shown throughout the examples above. They also chain: `value.ThrowIfNull().ThrowIfInactive()` checks both conditions and returns the value if both pass. +Guard methods return the guarded value when the check passes, allowing inline checks for programming assumptions. The `Status` guard above protects internally assigned/rehydrated status, and required identifier/object guards protect the internal construction contract. They are not substitutes for business-rule rejection; see [Invariant Failures versus Request Validation](#invariant-failures-versus-request-validation). ## PersistenceState @@ -185,13 +193,24 @@ Value objects represent concepts with no independent identity — defined entire ```csharp public sealed record class ItemPricing { - public required Contracts.UnitOfMeasure UnitOfMeasure { get; init => field = value.ThrowIfInactive(); } - public decimal UnitPrice { get; init => field = value.ThrowIfLessThanZero(); } - public decimal Quantity { get; init => field = value.ThrowIfLessThanZero(); } + public required Contracts.UnitOfMeasure UnitOfMeasure + { + get; + init + { + value.ThrowIfNull(); + if (!value.IsValid || !value.IsActive) + throw new BusinessException("Pricing requires a valid, active unit of measure.").WithErrorCode("invalid-unit-of-measure"); + + field = value; + } + } + public decimal UnitPrice { get; init => field = value < 0 ? throw new BusinessException("Unit price cannot be negative.").WithErrorCode("negative-unit-price") : value; } + public decimal Quantity { get; init => field = value < 0 ? throw new BusinessException("Quantity cannot be negative.").WithErrorCode("negative-quantity") : value; } public decimal Total => UnitPrice * Quantity; public ItemPricing EnsureIsValid() => DecimalRuleHelper.CheckScale(Quantity, UnitOfMeasure.Scale) ? this - : throw new ValidationException($"Quantity decimal places exceed the unit-of-measure scale of {UnitOfMeasure.Scale}."); + : throw new BusinessException($"Quantity decimal places exceed the unit-of-measure scale of {UnitOfMeasure.Scale}.").WithErrorCode("quantity-scale-exceeded"); } ``` @@ -249,6 +268,7 @@ For CRUD-oriented domains, skip the Domain layer entirely and let the Applicatio ## Do Not - Do not perform async I/O (repository calls, HTTP requests) inside domain classes — async work belongs in Application services or Policies. +- Do not use request-validation errors for domain business invariants, reference request DTOs, or derive request-field paths in Domain. Use domain business errors; translate selectively in Application only where request context is known. - Do not expose child entity mutation methods as `public` — use `internal` so only the owning aggregate can drive mutations. - Prefer returning `Result.BusinessError(...)` or `Result.NotFoundError()` over throwing exceptions for expected business failures in domain methods — this keeps failures explicit and composable. Throwing `BusinessException` is acceptable where it feels more natural, but the `Result` style is recommended for consistency. - Do not reference Infrastructure, Application, or host assemblies from the Domain layer — it depends only on Contracts and CoreEx. diff --git a/.github/instructions/coreex-repositories.instructions.md b/.github/instructions/coreex-repositories.instructions.md index e3843eda0..a4026aa15 100644 --- a/.github/instructions/coreex-repositories.instructions.md +++ b/.github/instructions/coreex-repositories.instructions.md @@ -66,11 +66,13 @@ Cosmos-backed adapters inject the typed `*CosmosDb` accessor instead of `*EfDb` | Operation | Return type | Notes | |---|---|---| | Single entity lookup | `Task` | Returns `null` when not found; service checks | -| Create / Update | `Task>` | Includes mutation flag for event decisions | -| Delete | `Task` | Carries mutation flag only | +| Create / Update | `Task>` or `Task>>` | Includes mutation flag for event decisions | +| Delete | `Task` or `Task>` | Carries mutation flag only | | Collection query | `Task>` | Items + optional total count | | Result pipeline (optional) | `Task>` | Developer choice — can be used on any repository method; enables explicit failure propagation without exceptions | +**Preserve mutation metadata across repository boundaries, including ROP and Domain mapping.** Use `dr.Map(value => SomeMapper.Map(value))` to transform the value while retaining `WasMutated`; never unwrap/cast away `DataResult` before Application makes mutation-dependent decisions. `Map` always transforms the value, including successful no-ops. Prefer the lambda for CoreEx mappers: their nullable-return method groups can infer a nullable destination despite `NotNullIfNotNull`. Implicit conversion to `T`, `.AsResult()` on a successful mutation result, or constructing `new DataResult(mappedValue)` with the default `true` loses that information. For aggregates, report the whole parent/child operation, not an unrelated single-row flag; retain provider semantics and transaction commit handling. Reads remain unchanged. + ## CosmosDb For Cosmos domains there is **no EF Core `DbContext` or `EfDb`**. Instead, a typed `*CosmosDb` class extends `CosmosDb`, declares any shared container options, and exposes per-container accessors via `CosmosDbContainer` / `CosmosDbMappedContainer`: @@ -360,7 +362,7 @@ public Task> GetAsync(string id, CancellationToken cancell .GoAsync(() => _ef.Baskets.GetWithResultAsync(id, cancellationToken)) .ThenAs(model => BasketMapper.Map(model)); -public Task> CreateAsync(Domain.Basket basket, CancellationToken cancellationToken = default) => Result +public Task>> CreateAsync(Domain.Basket basket, CancellationToken cancellationToken = default) => Result .Go(() => { var model = new Persistence.Basket(); @@ -368,7 +370,7 @@ public Task> CreateAsync(Domain.Basket basket, Cancellatio return model; }) .ThenAsAsync(model => _ef.Baskets.CreateWithResultAsync(model, cancellationToken)) - .ThenAs(b => BasketMapper.Map(b)); + .ThenAs(dr => dr.Map(value => BasketMapper.Map(value))); ``` ## Mapping @@ -530,6 +532,8 @@ public async Task> GetAsync(string id, CancellationToken ct = d Validate only fields the consumer needs; do not impose create/update rules or identifier matching by default. Do not capture request state/scoped dependencies in a shared validator. Matched status codes from `.WithStatusResult(...)` short-circuit execution before deserialization and validation. + +Map HTTP 404 to entity-not-found only when the downstream endpoint contract defines that expected lookup outcome; a route/deployment/configuration 404 is not an invalid caller reference. Repositories/adapters report lookup outcomes; application services/policies distinguish missing target resources (404) from missing request-body references (field-level 400), preserving unrelated failures. See [Not Found Handling](/.github/instructions/coreex-application-services.instructions.md#not-found-handling). The wrapper validates successful non-null values and returns the validation result's `Value`. Reported errors become `HttpRequestException` with inner validation diagnostics: failed `ToResult*`, throwing `GetValue*`, standard internal 500 rather than caller 400. Required methods reject null; diff --git a/.github/instructions/coreex-tests.instructions.md b/.github/instructions/coreex-tests.instructions.md index 621313af3..0db05159f 100644 --- a/.github/instructions/coreex-tests.instructions.md +++ b/.github/instructions/coreex-tests.instructions.md @@ -57,9 +57,11 @@ The two test projects have **distinct, non-overlapping jobs**. Decide where a be - **Private inline response validators** are covered through `*.Test.Unit/Clients/{ExternalDomain}/{External}HttpClientTests.cs`, using mocked HTTP responses. Assert missing/invalid consumed fields produce `HttpRequestException` with exact inner validation messages, and valid minimal responses succeed; do not expose private rules or create standalone validator tests. For cached adapters, prove an invalid 2xx payload is not cached by replacing it with a valid one and retrying. - **Validator rules** are proven **exhaustively in unit tests** (every mandatory/range/format/cross-field rule). In the API tests, do **not** re-enumerate them — assert **one** representative `AssertBadRequest()` + `AssertErrors(...)` case to confirm the validator is *wired into the pipeline*, then move on. **But `AssertErrors` is an exact match — not a subset.** It fails unless the listed errors are **exactly** the set the input produces (none missing, none extra). So "one representative case" means **craft the input to produce exactly the errors you assert** — e.g. send a value valid in every respect *except* the one rule under test — rather than posting an empty object (which fires *all* mandatory errors) and listing only some. (See the validators guidance — "Unit Tests".) -- **Service/repository behaviour** is proven in the **API tests** over the real database. Do **not** re-create it in unit tests with a mocked repository/UoW — that would assert the mock's configured behaviour, not the real persistence/mapping/eventing. +- **Service/repository behaviour** is proven in the **API tests** over the real database. Do **not** re-create it in unit tests with a mocked repository/UoW — that would assert the mock's configured behaviour, not the real persistence/mapping/eventing. The narrow mutation-publication boundary exception below does not replace that coverage. - When a behaviour *could* sit in either, prefer the layer that exercises it **without mocking the thing under test**: validator → unit; anything touching the DB, cache, outbox, or HTTP pipeline → API. +**Mutation-publication boundary exception:** When real persistence cannot reliably force a successful no-op after a successful read, a narrowly scoped unit test may supply `DataResult` with `WasMutated == false`, true, and a failed Result to the real service. Verify success/no mutation event, success/expected event, and unchanged failure/no event, including when the aggregate had changes. Assert the write and event decision run inside the transaction delegate. This proves the service interprets the supplied outcome, not real persistence, rollback, deferred writes, or concurrency. Retain sequential delete-idempotency coverage, but do not treat it as covering the read/delete gap. + The goal is a single source of truth per behaviour: unit tests guard the rules and pure logic; API tests guard the wired-up, persisted, evented system. --- @@ -668,7 +670,7 @@ _mockHttpReserveRequest.Verify(); ## Unit Tests — Validators -Unit tests are for logic with **no external dependencies**; any injected services are **mocked**. **Validators are the primary unit-test target** — they encode the most conditional logic. Application service orchestration is exercised by the host integration tests (`*.Test.Api` / `*.Test.Subscribe`), **not** here. +Unit tests are for logic with **no external dependencies**; any injected services are **mocked**. **Validators are the primary unit-test target** — they encode the most conditional logic. Application service orchestration is exercised by the host integration tests (`*.Test.Api` / `*.Test.Subscribe`), except the narrow mutation-publication boundary regression described above. Maintain **one test class per validator**, under `*.Test.Unit/Validators/`, named `{Validator}Tests` and extending `WithGenericTester` — `EntryPoint` (in the template) configures the DI/host services the validator needs. Each `[Test]` runs inside `Test.Scoped(test => { ... })`. Invoke the validator exactly as the application does: diff --git a/.github/instructions/coreex-validators.instructions.md b/.github/instructions/coreex-validators.instructions.md index 504de6947..dcf0a074a 100644 --- a/.github/instructions/coreex-validators.instructions.md +++ b/.github/instructions/coreex-validators.instructions.md @@ -28,6 +28,8 @@ tags: ["validators", "validation", "fluent-api", "rules", "error-handling", "app ## Placement +Request validators provide input/field validity; aggregates, entities, and value objects independently enforce business invariants using `BusinessException` / `Result.BusinessError`, not request-validation errors. Do not move request validators or request JSON paths into Domain, or remove domain guards because input was validated. Known domain failures may be selectively translated in Application only when the request-field mapping is unambiguous. See [domain invariant conventions](/.github/instructions/coreex-domain.instructions.md#invariant-failures-versus-request-validation). + Application business/request validators live in `Application/Validators/` and may inject Application-layer dependencies (e.g., `IProductRepository`), never Infrastructure dependencies. External-response validation is integration-owned: prefer a private static readonly `Validator` @@ -285,6 +287,14 @@ protected async override Task OnValidateAsync(ValidationContext When you do add an error directly, identify the property using the **member-access expression** overload of `context.AddError` — `context.AddError(x => x.Property, ...)`. Never pass a property-name string such as `nameof(...)`; the expression resolves the label, JSON name, and metadata automatically. +Outside a validator context, use `ValidationException.Create(x => x.Address!.Street, "Street is invalid.")` for a manual property error. Direct/nested property chains resolve each segment's JSON name using runtime metadata, respecting explicit JSON names and the default serializer naming policy. An overload accepts `(expression, options, format, ...values)` for custom serializer options. Expressions are not executed: `!` is safe for nullable intermediate properties. Array/list indexes and string dictionary keys may be literals or captured locals: `x => x.Addresses[0].Street` yields `addresses[0].street`, while `x => x.AddressesByType["home"].Street` yields `addressesByType.home.street`. Collection access is never executed. String keys follow the effective serializer's `DictionaryKeyPolicy` (null preserves keys), matching serialized JSON and validator key/value error paths. CLR paths and `GetDictionaryKey()` retain the original key. No escaping is added; punctuation in keys can be ambiguous because these are error paths, not full JSONPath expressions. Entity fields, arbitrary getters/calls for indexes/keys, index calculations, and intermediate casts are unsupported. This factory always uses JSON names and preserves the existing string-property factory's `LText`/formatting behavior (no automatic label/value arguments). It does not carry validator-specific naming/path overrides; keep using `context.AddError` within validators. + +For Result-style manual property errors outside a validator, use `Result.ValidationError(x => x.Name, "Name is invalid.")` or `Result.ValidationError(...)`. The shorthand accepts an expression, an `LText` message (including preformatted arguments), and an optional exception configuration action, using the default JSON naming/path resolver. For explicit serializer options or separate formatting arguments, use `ValidationException.Create` directly; the exception implicitly converts to `Result` or `Result`. + +The expression factories also allow an omitted/null message: `ValidationException.Create(x => x.Name)` or `Result.ValidationError(x => x.Name)` uses `Validation.InvalidFormat` with the leaf property's friendly metadata text, respecting localization/display labels rather than deriving a label from the JSON path. A terminal collection index or dictionary key retains its containing property's label. With explicit serializer options, pass null as the message to infer it. Explicit messages are unchanged; this default does not apply to existing string-property factories. + +For multiple manually collected errors outside a validator, use `MessageItem.CreateErrorMessage(x => x.Property, text)` and wrap the collection once. It shares inferred/default messages and paths with the exception/result factories; `(expression, options, text)` supports explicit serializer options. Keep using `context.AddError` inside validators. + **Choose the right guard — `HasErrors` vs `HasError(x => x.Prop)`:** - `context.HasErrors` (global) — bail early only when the *following logic needs the whole entity in a valid state*, e.g. before async I/O / database lookups (see [Async Validation](#async-validation-database-checks)). - `context.HasError(x => x.Prop)` (per-property) — when you are simply layering an additional rule onto a single property, gate on **that property only**. This still runs your check when unrelated properties have failed, and skips it only when the property itself is already in error (avoiding a misleading second message). diff --git a/.github/skills/coreex-adapter/SKILL.md b/.github/skills/coreex-adapter/SKILL.md index 0d33cf4b5..16d4ef2c0 100644 --- a/.github/skills/coreex-adapter/SKILL.md +++ b/.github/skills/coreex-adapter/SKILL.md @@ -50,6 +50,7 @@ Guides you through adding or modifying an adapter — the boundary that isolates - Terminal methods on `HttpRequestBuilder` (`ToResultAsync`, `GetValueAsync`, etc.) **automatically dispose** the response. When inspecting raw responses (e.g. `builder.GetResponseAsync()`), dispose the response with `using var`. Reported validation errors become `HttpRequestException` failures with inner diagnostics, not caller validation errors. Test inline rules through the client, including invalid 2xx responses and success; cached adapters must not cache invalid responses. - `response.ToResultAsync()` maps HTTP status to `Result` (2xx → `Success`, error + ProblemDetails → `BusinessException`/`ProblemDetailsException`, plain error → `HttpRequestException`) - `response.ToResultAsync()` is for responses that **must** carry a value — a null/empty body (including 204) is an `HttpRequestException` failure, so the result is `Result` with a non-null `T` and callers need no null handling. Use `ToResultOrDefaultAsync()` (→ `Result`) only where a missing body is genuinely valid; `GetValueAsync()`/`GetValueOrDefaultAsync()` are the throwing equivalents. Map specific status codes fluently using `.WithStatusResult(HttpStatusCode.NotFound, () => Result.NotFoundError())` (which short-circuits validation and deserialization) when "not found" is an expected outcome +- Map 404 to entity-not-found only when that is the downstream endpoint's expected contract; route/deployment/configuration failures are not invalid caller references. The application service/policy decides missing target (404) versus missing body reference (field-level 400) and preserves other failures; see [Not Found Handling](/.github/instructions/coreex-application-services.instructions.md#not-found-handling). - **Real-time read adapters should cache successes** with `IHybridCache.GetOrCreateWithResultAsync(key, factory, options, ct)` — the adapter method becomes one line; only successful results are cached (failures such as not-found are never cached). Use short expiry via `HybridCacheEntryOptions.CreateFor(...)` (overridable through `CoreEx:Caching:{Type}:*`). Concurrent cache misses are collapsed to one factory call per key per node with `FusionHybridCache` (cross-node needs an opt-in FusionCache distributed locker; non-FusionCache implementations such as `MemoryOnlyHybridCache` don't collapse them) — keep the factory an idempotent read. Tests must then clear the cache per test and register a hybrid cache in the unit-test host (see `references/workflow.md` → "Caching a real-time adapter") - Two distinct adapter roles in the samples — **synchronous** (real-time HTTP + events) and **replication** (event-driven local-store write; EF in relational domains, `CosmosDb` containers in Cosmos domains); separate interfaces, separate implementations - **Generate `*.Test.Unit/Clients/{ExternalDomain}/{External}HttpClientTests.cs`** for a typed HTTP client used by a synchronous or replication adapter — cover success (2xx), server error (5xx), and business error (422/ProblemDetails) per endpoint; skip for store-only replication adapters (no HTTP) and for combined local-store+HTTP+events orchestration adapters (mock the client via `Test.ReplaceHttpClientFactory` in integration tests instead) diff --git a/.github/skills/coreex-adapter/references/workflow.md b/.github/skills/coreex-adapter/references/workflow.md index 7590a5310..598a45ebc 100644 --- a/.github/skills/coreex-adapter/references/workflow.md +++ b/.github/skills/coreex-adapter/references/workflow.md @@ -132,7 +132,7 @@ return await response.WithValidator(_validator) .ToResultAsync(ct).ConfigureAwait(false); ``` -`ToResultAsync()` returns a non-null `T`: a successful response with a null/empty body (including 204) is an `HttpRequestException` failure, so no `?? throw` is needed. Use `ToResultOrDefaultAsync()` (→ `Result`) only where an absent body is a valid outcome. Map 404 yourself when "not found" is an expected business outcome (surface it as `Result.NotFoundError()`; the Application-layer policy decides how to present it). +`ToResultAsync()` returns a non-null `T`: a successful response with a null/empty body (including 204) is an `HttpRequestException` failure, so no `?? throw` is needed. Use `ToResultOrDefaultAsync()` (→ `Result`) only where an absent body is a valid outcome. Map 404 to `Result.NotFoundError()` only when entity-not-found is an expected outcome of that downstream endpoint; a wrong route or deployment/configuration failure is not an invalid caller reference. The application service/policy interprets expected lookup outcomes as missing targets (404) or missing body references (field-level 400), preserving other failures; see [Not Found Handling](/.github/instructions/coreex-application-services.instructions.md#not-found-handling). ### Response-contract validation diff --git a/.github/skills/coreex-aggregate/SKILL.md b/.github/skills/coreex-aggregate/SKILL.md index 2281fbda7..74e137f11 100644 --- a/.github/skills/coreex-aggregate/SKILL.md +++ b/.github/skills/coreex-aggregate/SKILL.md @@ -59,8 +59,11 @@ then follows the corresponding pattern from `CoreEx.DomainDriven`. - Public mutation methods on the aggregate should return `Result`/`Result` for consistency with Application-layer `Result` pipelines — even though the internal guard throws - Child entity mutation methods are `internal` — only the owning aggregate may invoke them - No async I/O in domain classes — ever. Async belongs in Application services or Policies. +- Domain business invariants use `BusinessException` / `Result.BusinessError` with domain messages/codes, not `ValidationException` / `ValidationError` or request-field paths. Request validity belongs to Application validators/policies; preserve both boundaries' checks. Only Application may deliberately map a known domain failure to an unambiguous request field. +- Keep argument/invalid-operation guards for programming misuse; they are not business-error factories (`ThrowIfLessThanZero` throws `ArgumentOutOfRangeException`, ref-data `ThrowIfInvalid`/`ThrowIfInactive` throw `InvalidOperationException`). Use explicit business checks for expected invariant rejections; do not change shared guard semantics. See [invariant failure conventions](/.github/instructions/coreex-domain.instructions.md#invariant-failures-versus-request-validation). - Keep concrete reference-data factory parameters/value-object properties typed; callers pass code strings directly via [generated implicit conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion). Do not add conversion-only wrappers; requiredness, validity and activity remain separate guards. - No native domain-event dispatch (MediatR-style) — only integration events via `Aggregate.Events`, forwarded through `IUnitOfWork.Events` in the Application layer +- Application gates mutation-announcement events (including forwarding accumulated aggregate events) on successful persistence `DataResult.WasMutated`, inside the write transaction. Retain this metadata through repository/Domain mapping; aggregate changes and successful execution alone do not prove mutation. - Aggregates are the best unit-test target in the codebase (no injected dependencies) — cover every mutation method's happy path and rejection path with `WithGenericTester` + `Test.Scoped(...)`, calling the aggregate directly (no repository, no Application service); assert `OnCheckCanMutate()` guard failures as thrown exceptions, and failed-`Result` returns as `Result` assertions - A value object backed by a JSON database column (e.g. `Address` stored as `basket.ShippingAddressJson`) is authored **exactly like any other value object** — a plain `sealed record`/`record class` with invariant-enforcing `init` setters. It sits as the middle type in a three-type chain (`Contracts.Xxx` ↔ `Domain.ValueObjects.Xxx` ↔ `Persistence.Xxx`) bridged by two separate mappers (Application: Domain↔Contract; Infrastructure: Domain↔Persistence) — see [`coreex-domain.instructions.md#value-objects-backed-by-a-json-column`](/.github/instructions/coreex-domain.instructions.md#value-objects-backed-by-a-json-column) and the [`coreex-db-migration`](/.github/skills/coreex-db-migration/SKILL.md#json-columns) skill for the column/mapper setup diff --git a/.github/skills/coreex-aggregate/references/workflow.md b/.github/skills/coreex-aggregate/references/workflow.md index 2c1d1b1b8..0178e13e2 100644 --- a/.github/skills/coreex-aggregate/references/workflow.md +++ b/.github/skills/coreex-aggregate/references/workflow.md @@ -194,7 +194,7 @@ namespace {Domain}.Domain.ValueObjects; public sealed record class {ValueObject} { - public required {PropertyType} {Property} { get; init => field = value.ThrowIfLessThanZero(); } + public required decimal {Property} { get; init => field = value < 0 ? throw new BusinessException("{Property} cannot be negative.").WithErrorCode("negative-{property}") : value; } public {OtherType} {OtherProperty} { get; init => field = value.ThrowIfNull(); } // Derived read-only members are welcome — they are not persisted, only computed. @@ -202,16 +202,14 @@ public sealed record class {ValueObject} /// Validates cross-property invariants not expressible via a single property initialiser. public {ValueObject} EnsureIsValid() => {condition} ? this - : throw new ValidationException("{Description of the violated invariant}."); + : throw new BusinessException("{Description of the violated invariant}.").WithErrorCode("{invariant-error-code}"); } ``` **Rules:** - `sealed record`, never a mutable class — structural equality and `with`-expression semantics are intrinsic to a value object's definition. -- Enforce single-property invariants directly in the `init` accessor using the guard helpers - (`.ThrowIfNull()`, `.ThrowIfLessThanZero()`, `.ThrowIfInactive()`, etc.) — they return the guarded - value, making them composable inline. +- Enforce business invariants directly in `init` accessors using explicit domain checks that throw `BusinessException`, not request-validation errors. Use domain messages/codes, never request contracts or JSON paths. Required internal object/argument checks may retain argument guards; a numeric business restriction is not programming misuse merely because a guard helper exists. - Cross-property invariants that cannot be expressed in a single `init` accessor go in an `EnsureIsValid()` method, called by the owning entity/aggregate after construction. - Place value objects in a `ValueObjects/` sub-folder within the Domain project. @@ -347,14 +345,14 @@ public class {Aggregate}Tests : WithGenericTester These extension methods return the guarded value on success (enabling inline chaining) and throw on failure — used throughout property setters, `init` accessors, and factory method bodies: -| Helper | Throws when | -|---|---| -| `.ThrowIfNull()` | value is `null` | -| `.ThrowIfNullOrEmpty()` | value is `null` or an empty string | -| `.ThrowIfInactive()` | a reference-data value's `IsActive` is `false` | -| `.ThrowIfLessThanZero()` | a numeric value is less than zero | +| Helper | Throws when | Exception | +|---|---|---| +| `.ThrowIfNull()` | value is `null` | `ArgumentNullException` | +| `.ThrowIfNullOrEmpty()` | value is `null` or an empty string | `ArgumentNullException` / `ArgumentException` | +| `.ThrowIfInactive()` | reference data is invalid or inactive | `InvalidOperationException` | +| `.ThrowIfLessThanZero()` | a numeric value is less than zero | `ArgumentOutOfRangeException` | -Chain them for multiple checks on one value: `value.ThrowIfNull().ThrowIfInactive()`. +These helpers return the guarded value and can be chained for programming assumptions, such as checking an internally assigned status. They do not represent expected business failures. For a business rule rejecting a negative quantity or invalid/inactive selection, use an explicit check and `BusinessException` / `Result.BusinessError` instead. Keep shared helper behavior unchanged. --- @@ -388,10 +386,7 @@ Application layer. CoreEx deliberately does not provide in-process domain-event side-effects; coarse integration events communicated through `IUnitOfWork.Events` are explicit, auditable, and transactional. - Add events from within a mutation method using the inherited `AddEvent(...)`. -- The Application service is responsible for forwarding `aggregate.Events` into `_unitOfWork.Events` - inside the same `TransactionAsync(...)` scope that persists the aggregate — see - [`/.github/instructions/coreex-application-services.instructions.md`](/.github/instructions/coreex-application-services.instructions.md) for the Application-layer side - of this handoff. +- The Application service forwards mutation-announcement `aggregate.Events` into `_unitOfWork.Events` using `WhereMutated` on the successful persistence `DataResult`, inside the transaction that persists the aggregate. Preserve `WasMutated` through Domain mapping; an earlier read, aggregate state, or success alone is not a substitute. Commands/attempt notifications have separate semantics; the flag is not a commit/exactly-once guarantee. See [Application guidance](/.github/instructions/coreex-application-services.instructions.md) for this handoff. - If a genuine in-process domain-event use case exists, that is an opt-in extension a consumer adds themselves (e.g., raising events dispatched via MediatR after commit) — not a CoreEx default. @@ -399,6 +394,7 @@ Application layer. CoreEx deliberately does not provide in-process domain-event ## Guardrails +- **Business invariants are not request validation** — use `BusinessException` / `Result.BusinessError` in aggregates, entities, and value objects. Keep request validators and domain enforcement; selectively translate known errors in Application only when the request-field mapping is unambiguous. Never derive request JSON paths or reference request DTOs in Domain. - **No async I/O in domain classes** — repository calls, HTTP requests, adapter calls all belong in Application services or Policies, never in an aggregate, entity, or value object - **Constructor is always `private`** — `CreateNew(...)` and `CreateFrom(...)` are the only public diff --git a/.github/skills/coreex-app-service/SKILL.md b/.github/skills/coreex-app-service/SKILL.md index b625f5a06..c7468ae85 100644 --- a/.github/skills/coreex-app-service/SKILL.md +++ b/.github/skills/coreex-app-service/SKILL.md @@ -45,15 +45,17 @@ Guides you through creating or modifying a CoreEx Application-layer service in ` 6. Read-only queries or collection results needed? (→ CQRS read service, Path C) **Key rules at a glance:** +- Request validity uses validation errors; domain business invariants use business errors without request-field paths. Keep domain enforcement after request validation. Only selectively translate known domain failures where the request-field mapping is unambiguous; do not blanket-convert business errors. See [domain invariant conventions](/.github/instructions/coreex-domain.instructions.md#invariant-failures-versus-request-validation). - `[ScopedService]` on every service — auto-registers via `AddDynamicServicesUsing()` - **Only inject**: repository, unit of work, adapter interfaces, logger — **never** validators, mappers, policies - `ValidateAndThrowAsync` (exception style) / `ValidateWithResultAsync` (Result<T>) — **never bare `ValidateAsync`** - Service assigns `Id` after validation: `value.Id = Runtime.NewId()` (string key) / `Runtime.NewGuid()` (Guid) - All mutations in `_unitOfWork.TransactionAsync(...)` — event added inside, never outside - `WhereMutated(v => ...)` for Create/Update (`DataResult` carries value); `WhereMutated(() => ...)` for Delete +- Success is not mutation: retain `DataResult` through ROP/domain mapping until the service gates mutation events on `WasMutated`, inside the write transaction, then unwrap for its response. An earlier read or `HasChanges` check is not a substitute; the flag is a provider signal, not a commit/exactly-once guarantee. Commands/attempt notifications follow their own semantics. - Delete event: `EventData.CreateEvent(EventAction.Deleted).WithKey(id)` — **no value body** - Check `EventAction` before reaching for `CreateEventWith(v, "string")` — most business actions (`Confirmed`, `Cancelled`, `Started`, `Completed`, `Suspended`, `Closed`, `Expired`, etc.) already have an enum member; the raw-string overload is only for genuinely novel actions -- `NotFoundException.ThrowIfDefault(entity)` after any Get that must find the entity +- Missing target resource/explicit child → `NotFoundException` / `Result.NotFoundError`; missing body reference → field-level validation (400), preferably an expression over the request contract. Query filters normally return empty results. Interpret only expected entity-not-found outcomes; preserve unrelated failures and deliberate endpoint semantics. See [Not Found Handling](/.github/instructions/coreex-application-services.instructions.md#not-found-handling). - Validators / Mappers / Policies: **not DI-registered** — call or instantiate at point of use - `Validator`: call via singleton — `{Name}Validator.Default.ValidateAndThrowAsync(...)` - `Validator` (with injection): instantiate at call site — `new {Name}Validator(_dep).ValidateAndThrowAsync(...)` diff --git a/.github/skills/coreex-app-service/references/workflow.md b/.github/skills/coreex-app-service/references/workflow.md index c040296b8..6ada8749c 100644 --- a/.github/skills/coreex-app-service/references/workflow.md +++ b/.github/skills/coreex-app-service/references/workflow.md @@ -164,7 +164,7 @@ public async Task {Action}Async(string id, CancellationToken c Use when the project has elected the ROP style. Method signatures return `Result`. `TransactionAsync` returns `Task>` when its delegate returns `Result`. Compose with `Result.GoAsync` / `.ThenAs` / `.ThenAsAsync`. -> **Domain aggregate case:** When a Domain layer is present, repositories return `Result` directly (not `DataResult`). Map to contract via a `Mapper` in `Application/Mapping/` — call via `{Name}Mapper.Map(aggregate)`. See Shopping samples in Key References. +> **Repository mutation contract:** ROP Create/Update return `Result>`; Delete returns `Result`. This includes Domain aggregates (`Result>`), not just DTO CRUD. Reads keep their existing shapes. Map domain values to contracts via `{Name}Mapper.Map(dr.Value)`, retaining `WasMutated` until mutation-event decisions are complete; see the [complete ROP/domain mapping example](/.github/instructions/coreex-application-services.instructions.md). Non-ROP repositories may return plain `DataResult`/`DataResult`; adapt the examples to the declared contract, never implicitly unwrap either result. > > **JSON-backed value object case:** If the aggregate holds a value object persisted via a JSON column (e.g. `Basket.ShippingAddress`), that value object needs its own `BiDirectionMapper` in `Application/Mapping/` — not the root aggregate's uni-directional `Mapper`, because the value flows both out (to the read contract) and in (as update input, e.g. `UpdateShippingAddressAsync`). Compose it in both directions: `{ValueObject}Mapper.To.Map(...)` in the root mapper's `OnMap`, and `{ValueObject}Mapper.From.Map(...)` when accepting the value as a service parameter before calling the aggregate's `UpdateXxx(...)` method. See [`coreex-application-services.instructions.md#json-backed-value-object-mapping`](/.github/instructions/coreex-application-services.instructions.md#json-backed-value-object-mapping). @@ -200,8 +200,8 @@ public async Task> CreateAsync(Contracts.{Name} value, return await _unitOfWork.TransactionAsync(async ct => { var dr = await _repository.CreateAsync(value, ct).ConfigureAwait(false); - return Result.Ok( - dr.WhereMutated(v => _unitOfWork.Events.Add(EventData.CreateEventWith(v, EventAction.Created)))); + return dr.ThenAs(data => + data.WhereMutated(v => _unitOfWork.Events.Add(EventData.CreateEventWith(v, EventAction.Created)))); }, cancellationToken).ConfigureAwait(false); } ``` @@ -219,14 +219,14 @@ public async Task> UpdateAsync(Contracts.{Name} value, return vr.AsResult(); var current = await _repository.GetAsync(value.Id, cancellationToken).ConfigureAwait(false); - if (current is null) - return Result.NotFoundError(); + if (current.IsFailure) + return current.AsResult(); return await _unitOfWork.TransactionAsync(async ct => { var dr = await _repository.UpdateAsync(value, ct).ConfigureAwait(false); - return Result.Ok( - dr.WhereMutated(v => _unitOfWork.Events.Add(EventData.CreateEventWith(v, EventAction.Updated)))); + return dr.ThenAs(data => + data.WhereMutated(v => _unitOfWork.Events.Add(EventData.CreateEventWith(v, EventAction.Updated)))); }, cancellationToken).ConfigureAwait(false); } ``` @@ -241,18 +241,18 @@ private async Task> OrchestrateUpdateAsync( string id, Func mutate, EventAction action = EventAction.Updated, CancellationToken cancellationToken = default) { var current = await _repository.GetAsync(id, cancellationToken).ConfigureAwait(false); - if (current is null) - return Result.NotFoundError(); + if (current.IsFailure) + return current.AsResult(); - var mr = mutate(current); + var mr = mutate(current.Value); if (mr.IsFailure) return mr.AsResult(); return await _unitOfWork.TransactionAsync(async ct => { - var dr = await _repository.UpdateAsync(current, ct).ConfigureAwait(false); - return Result.Ok( - dr.WhereMutated(v => _unitOfWork.Events.Add(EventData.CreateEventWith(v, action)))); + var dr = await _repository.UpdateAsync(current.Value, ct).ConfigureAwait(false); + return dr.ThenAs(data => + data.WhereMutated(v => _unitOfWork.Events.Add(EventData.CreateEventWith(v, action)))); }, cancellationToken).ConfigureAwait(false); } @@ -364,6 +364,8 @@ public class {Name}Service(IUnitOfWork unitOfWork, I{Name}Repository repository, Create `Application/Policies/{Name}Policy.cs`. Policies are **not registered in DI** — instantiate at call site using dependencies already injected into the calling service: +The example checks a request-body reference with the agreed JSON path `"id"` and translates only expected entity-not-found failures. Missing target resources (including explicitly addressed children) retain 404 semantics; unmatched collection filters normally return empty results. Preserve other failures, deliberate idempotent deletion, and security concealment. Do not choose 404/business errors merely for an error code; see [Not Found Handling](/.github/instructions/coreex-application-services.instructions.md#not-found-handling). For a policy reused across fields, accept a resolved JSON path; when the request contract is already known, prefer `Result.ValidationError<{Request}>(x => x.{ReferenceId}, text)`. Do not add a request DTO dependency merely to use the shorthand. + ```csharp namespace {Solution}.Application.Policies; @@ -374,7 +376,7 @@ public class {Name}Policy(I{ExternalDomain}Adapter adapter) public Task> EnsureExistsAsync(string id, CancellationToken cancellationToken = default) => Result .GoAsync(() => _adapter.GetAsync(id, cancellationToken)) .OnFailure(r => r.IsNotFoundError - ? Result.ValidationError(MessageItem.CreateErrorMessage(nameof(id), "{Name} was not found.")) + ? Result.ValidationError(MessageItem.CreateErrorMessage("id", "{Name} was not found.")) : r); } ``` @@ -405,10 +407,11 @@ if (pr.IsFailure) 1. `dotnet build` — no errors or warnings. 2. Verify the interface in `Application/Interfaces/` declares all new methods. -3. Verify every mutating method wraps the repository call in `_unitOfWork.TransactionAsync(...)` and adds an event inside that scope. +3. Verify every mutating method wraps the repository call in `_unitOfWork.TransactionAsync(...)` and gates mutation-announcement events on the successful `DataResult.WasMutated` inside that scope. Preserve the flag through mapping/ROP before unwrapping the response. 4. Verify validators are called via `Default.ValidateAndThrowAsync` (exception style) or `Default.ValidateWithResultAsync` (ROP) — never bare `ValidateAsync`. 5. Check `Id` is assigned via `Runtime.NewId()` / `Runtime.NewGuid()` on Create — never left to the caller or the database. 6. **Offer to create or update the matching integration test** in `*.Test.Api/` or unit test in `*.Test.Unit/Services/` — go straight there. Do not write a throwaway smoke test (a scratch console/test snippet, ad-hoc scoped call, or manual `dotnet run` check) first "to see if it works"; that confirmation is exactly what the real test proves, so a preliminary pass is discarded effort. +7. Cover success/true (event), success/false (success without event), and failure (unchanged failure without event), including a successful earlier read and changed aggregate followed by a persistence no-op. Sequential delete idempotency alone does not cover that boundary; use a focused deterministic boundary test when necessary, without claiming it proves real concurrency/transaction behavior. --- @@ -417,6 +420,7 @@ if (pr.IsFailure) - **Never call bare `ValidateAsync`** — it returns results without throwing; use `ValidateAndThrowAsync` or `ValidateWithResultAsync`. - **Never inject validators, mappers, or policies** into service constructors — these are YAGNI injection points; call or instantiate at the point of use. - **Never publish events outside `TransactionAsync`** — the event and the database write must commit together. +- **Success is not mutation.** An existence read or aggregate `HasChanges`/`PersistenceState` check cannot replace the provider's `WasMutated`. Deferred writes may commit later; do not claim this flag proves a committed row or exactly-once delivery. Commands and deliberate attempt messages are not automatically mutation events. - **`WhereMutated(() => ...)` on Delete** — `DataResult` (delete) has no value; do not use `(v => ...)` or `(_ => ...)`. - **Delete event has no body** — use `EventData.CreateEvent(action).WithKey(id)`; never pass a fabricated entity instance to `CreateEventWith` on a delete. - **`Runtime.NewId()` / `Runtime.NewGuid()` — not `Guid.NewGuid()`** — always use `Runtime` so the id source is test-controllable. diff --git a/.github/skills/coreex-policy/SKILL.md b/.github/skills/coreex-policy/SKILL.md index c1e6be02e..9857abaca 100644 --- a/.github/skills/coreex-policy/SKILL.md +++ b/.github/skills/coreex-policy/SKILL.md @@ -45,12 +45,14 @@ Guides you through creating or modifying a CoreEx Application-layer policy class - **Not DI-registered** — instantiated at the call site: `new {Name}Policy(_adapter).EnsureExistsAsync(id)` - Constructor accepts adapters and/or repositories already injected into the calling service - Always returns `Result` or `Result` — never throws (unless adapter itself throws unexpectedly) -- **EnsureExists:** translate `r.IsNotFoundError` → `Result.ValidationError(...)` — do not let `NotFoundException` propagate as-is -- `MessageItem.CreateErrorMessage(nameof(param), "{Entity} was not found.")` produces a field-level validation error -- `LText` static fields hold **localizable entity names** used inside message text — e.g. `new LText("Product")` as the `{0}` substitution in `"{0} was not found."`. The **property name** (`nameof(param)`) is always a plain `string?` — never `LText` +- **EnsureExists:** classify by the identifier's role: a missing request-body reference becomes a field-level validation error (400); a missing target resource or explicitly addressed child remains not-found (404). Query filters normally return empty results, not 404. See [Not Found Handling](/.github/instructions/coreex-application-services.instructions.md#not-found-handling). +- Translate only expected entity-not-found outcomes; preserve unrelated failures. Do not interpret route/configuration 404s as missing references, or choose a failure type merely to retain an error code. Preserve deliberate delete/security semantics; use business errors for state/process rules unless more specific conflict/concurrency or authorization semantics apply. +- When the owning request contract is known, prefer `Result.ValidationError(x => x.ProductId, "Product was not found.")`; use `MessageItem.CreateErrorMessage(...)` when assembling multiple errors. Expressions must identify the caller's input field, not the fetched entity's identifier. +- Reusable policies receiving only an identifier may use `MessageItem.CreateErrorMessage(propertyPath, ...)` with an explicit caller-agreed JSON path. Do not couple a reusable policy to a request DTO merely to use an expression. `nameof(param)` is a CLR parameter name, not a JSON-name resolver. +- `LText` static fields hold **localizable entity names** used inside message text — e.g. `new LText("Product")` as the `{0}` substitution in `"{0} was not found."`. Explicit property paths remain plain strings; expression factories resolve JSON naming metadata. - Guard methods can return the loaded entity as `Result` so callers can use it without a second fetch - Always `.ConfigureAwait(false)` on every `await` -- **Always generate a matching `{Name}PolicyTests.cs`** in `*.Test.Unit/Policies/` — cover the success path (expect `IsSuccess`) for each guard method, plus its failure path: `IsValidationError` for `EnsureExists`-style not-found guards, `IsBusinessError` for state/condition guards (e.g. `EnsureActive`) +- **Always generate a matching `{Name}PolicyTests.cs`** in `*.Test.Unit/Policies/` — cover success, the intended failure classification (`IsValidationError` for invalid references, `IsNotFoundError` for missing targets, `IsBusinessError` for business-state rules), and unchanged propagation of unrelated failures. For full workflow and code examples see [`references/workflow.md`](references/workflow.md). diff --git a/.github/skills/coreex-policy/references/workflow.md b/.github/skills/coreex-policy/references/workflow.md index ee975df19..718e263db 100644 --- a/.github/skills/coreex-policy/references/workflow.md +++ b/.github/skills/coreex-policy/references/workflow.md @@ -18,6 +18,7 @@ Full workflow for creating or modifying a CoreEx Application-layer policy class |---|---|---| | What entity is guarded? | Ask | Names the policy and its methods | | Guard type? | EnsureExists | EnsureExists / business rule / state check | +| Identifier's role? | Resolve from operation | Target resource/explicit child → 404; body reference → field validation (400); query filter → normally empty results | | Returns entity or pass/fail? | Returns entity | `Result` lets callers reuse loaded data; `Result` for pure pass/fail | | Adapter or repository dependency? | Ask | Inject whatever is already available in the calling service | | Multiple guard methods in one class? | No | Group when they share the same dependency | @@ -52,28 +53,44 @@ public class {Name}Policy(I{Name}Repository repository) ## Step 2 — EnsureExists Guard (most common) -Fetch the entity via the adapter and handle the `NotFoundError`. **How you translate it is a developer choice** — two patterns, different trade-offs: +Fetch the entity via the adapter and interpret the expected lookup outcome according to the identifier's role in the operation. This is not an arbitrary status-code choice; apply [Not Found Handling](/.github/instructions/coreex-application-services.instructions.md#not-found-handling). | Pattern | HTTP status | When to use | |---|---|---| -| `Result.ValidationError(...)` | 400 | The `id` is a **property on the request body** — links the error to the field, consistent with other validation errors, allows aggregation | -| `new NotFoundException().WithErrorCode(...)` | 404 | The `id` is a **path/query parameter** (primary resource), or the caller needs a machine-readable error code to branch on | +| `Result.ValidationError(...)` | 400 | A request-body property references a nonexistent entity — attach the error to that input field | +| `Result.NotFoundError(...)` / `NotFoundException` | 404 | The operation's target resource or explicitly addressed child does not exist | -`CreateErrorMessage(string? property, LText text)` — first arg is the **JSON property name** (`nameof(param)` or a string literal, never `LText`); second arg is the **localizable error message text**. +For example, `/bookings/{id}/travellers/{travellerId}` explicitly targets a child; an unknown `travellerId` in the body of `POST /bookings/{id}/travellers` is instead an invalid reference. Query parameters often filter collections: no matches normally means an empty result, while invalid query inputs follow the endpoint's validation contract. Preserve deliberate idempotent-delete and security-concealment behavior. -### Validation error (request-body property reference) — default for most policies +Translate only an expected entity-not-found result; other failures propagate unchanged. A downstream HTTP 404 may indicate an incorrect route or deployment/configuration problem rather than a missing entity. Do not turn those failures into caller validation. Machine-readable codes enrich the selected error type; they never justify selecting 404 or a business error for a missing body reference. + +When the owning request contract is known, prefer `Result.ValidationError(x => x.ProductId, "Product was not found.")`. Expressions identify the caller's input field, not the fetched entity's `Id`. For aggregation, use `MessageItem.CreateErrorMessage(x => x.ProductId, text)`; omitted/null text infers the standard invalid message from the property's friendly metadata label. Explicit naming options are supported as `(expression, options, text)`, and preformatted `LText` carries message arguments. + +Reusable policies that receive only an identifier need not depend on a request DTO. `CreateErrorMessage(string? property, LText text)` accepts an already-resolved JSON path as its first argument, not a naming expression. `nameof(param)` supplies only a CLR parameter name; it does not honor JSON attributes or naming policies. The examples below assume the caller's JSON field is explicitly agreed to be `"id"`; use a caller-supplied resolved path when the policy is reused for different fields. + +### Validation error (request-body property reference) + +When the policy already knows the owning request contract, the result value and request-property owner can differ: + +```csharp +return lookup.IsNotFoundError + ? Result.ValidationError<{Request}>(x => x.{ReferenceId}, "{Name} was not found.") + : lookup; +``` + +Here `lookup` is `Result`, not a nullable entity: unrelated failures remain unchanged. The expression type is the request contract, not the returned entity. With a nullable lookup, check null instead. Inside validators use `context.AddError(...)` to preserve contextual paths. For a reusable policy with the agreed JSON path `"id"`: ```csharp public Task> EnsureExistsAsync(string id) => Result .GoAsync(() => _{dep}Adapter.GetAsync(id)) .OnFailure(r => r.IsNotFoundError - ? Result.ValidationError(MessageItem.CreateErrorMessage(nameof(id), "{0} was not found.", _entityText)) + ? Result.ValidationError(MessageItem.CreateErrorMessage("id", "{0} was not found.", _entityText)) : r); ``` `OnFailure` runs only when the result is a failure. The ternary re-maps `NotFoundError` to a validation error; any other failure kind propagates unchanged (the `: r` branch). -### NotFoundException with context (path parameter / primary resource) +### NotFoundException with context (target resource / explicitly addressed child) ```csharp public Task> EnsureExistsAsync(string id) => Result @@ -83,7 +100,7 @@ public Task> EnsureExistsAsync(string id) => Result : r); ``` -`WithErrorCode` gives the consumer a stable, machine-readable code; `WithKey` records the offending identifier for diagnostics. +An existing target-not-found result may propagate unchanged; enrichment is optional. `WithErrorCode` gives the consumer a stable, machine-readable code; `WithKey` records the offending identifier for diagnostics. Neither changes which failure classification is appropriate. --- @@ -115,7 +132,7 @@ public Task EnsureActiveAsync(string id) => Result : Result.Success); ``` -**`Result.BusinessError` vs `Result.ValidationError`:** use `BusinessError` for domain/state violations that the user understands as a process rule ("entity is no longer available"), and `ValidationError` for constraint violations tied to a specific input field ("Product was not found"). +**`Result.BusinessError` vs `Result.ValidationError`:** use `BusinessError` for domain/state violations that the user understands as a process rule ("entity is no longer available"), unless specific conflict/concurrency or authorization semantics apply. Use field-level `ValidationError` for invalid input references ("Product was not found"), not a business error merely to retain an error code. --- @@ -134,16 +151,16 @@ public class {Name}Policy(I{Dep}Adapter {dep}Adapter) public Task> EnsureExistsAsync(string id) => Result .GoAsync(() => _{dep}Adapter.GetAsync(id)) .OnFailure(r => r.IsNotFoundError - ? Result.ValidationError(MessageItem.CreateErrorMessage(nameof(id), "{0} was not found.", _entityText)) + ? Result.ValidationError(MessageItem.CreateErrorMessage("id", "{0} was not found.", _entityText)) : r); } ``` **Two-argument vs three-argument `CreateErrorMessage`:** -- `CreateErrorMessage(nameof(id), "{Name} was not found.")` — the whole message is one `LText`; the entity name is baked in. Simple, but the entity name cannot be independently localized. -- `CreateErrorMessage(nameof(id), "{0} was not found.", _entityText)` — the format string is an `LText` and the entity name is a separately localizable `LText` argument. Preferred when the entity name is a shared concept. +- `CreateErrorMessage("id", "{Name} was not found.")` — the whole message is one `LText`; the entity name is baked in. Simple, but the entity name cannot be independently localized. +- `CreateErrorMessage("id", "{0} was not found.", _entityText)` — the format string is an `LText` and the entity name is a separately localizable `LText` argument. Preferred when the entity name is a shared concept. -**The property name is always `string?`** — `nameof(param)` for a local parameter, or a string literal for a fixed JSON path. Never pass `LText` as the property argument. +**For explicit-path overloads, the property path is `string?`, never `LText`.** For expression overloads, runtime metadata resolves the JSON path automatically. Neither requires changing a reusable policy's dependencies. --- @@ -160,11 +177,11 @@ public class {Name}Policy(I{Dep}Adapter {dep}Adapter) private readonly I{Dep}Adapter _{dep}Adapter = {dep}Adapter.ThrowIfNull(); /// Ensures the {Name} exists; returns the loaded entity for reuse by the caller. - /// Uses ValidationError because {Name}Id is a request-body property; swap for NotFoundException if it is a path parameter. + /// Checks a request-body reference using the agreed JSON path "id"; missing target resources have different semantics. public Task> EnsureExistsAsync(string id) => Result .GoAsync(() => _{dep}Adapter.GetAsync(id)) .OnFailure(r => r.IsNotFoundError - ? Result.ValidationError(MessageItem.CreateErrorMessage(nameof(id), "{0} was not found.", _entityText)) + ? Result.ValidationError(MessageItem.CreateErrorMessage("id", "{0} was not found.", _entityText)) : r); /// Ensures the {Name} is in active state. @@ -235,10 +252,10 @@ if (pr.IsFailure) 1. `dotnet build` — no errors or warnings. 2. Verify the policy lives in `Application/Policies/` and is **not** added to DI. 3. Verify call sites use `new {Name}Policy(_{dep}).MethodAsync(...)` — never inject via constructor. -4. Verify `NotFoundError` is translated to `ValidationError` — not allowed to propagate as a 404. +4. Verify missing body references produce field-level validation errors, missing targets retain not-found semantics, and unrelated failures propagate unchanged. Cover the agreed JSON path and deliberately retained endpoint semantics. 5. **Generate a matching unit test class** — always create `*.Test.Unit/Policies/{Name}PolicyTests.cs`. Policies have no infrastructure dependencies at test time (all I/O is mocked via the adapter/repository interface), making them ideal unit-test targets. Use `WithGenericTester` as the base class and `Test.Scoped(async test => ...)` to wrap each test body, consistent with other unit tests in the project. -### EnsureExists guard test +### EnsureExists guard test (request-body reference) ```csharp public class {Name}PolicyTests : WithGenericTester @@ -321,7 +338,7 @@ public void {Name}Policy_EnsureActiveAsync_Active_ReturnsSuccess() => Test.Scope ## Guardrails - **Never register a policy in DI** — instantiate at call site only. If you find yourself registering `I{Name}Policy`, stop and reconsider. -- **Never let `NotFoundException` propagate** from an EnsureExists method — translate it to either `ValidationError` (HTTP 400, for request-body property references) or a richer `NotFoundException` with `WithErrorCode`/`WithKey` (HTTP 404, for path/query parameters) — see Step 2 for the decision. +- **Interpret missing entities by their role, not URI location or error-code needs** — translate expected missing body references to field-level validation; preserve missing-target not-found results and unrelated failures. Do not require enrichment just to propagate a valid 404; see Step 2. - **`Result.BusinessError` for process/state violations; `Result.ValidationError` for field-constraint violations** — do not use `BusinessError` to signal a missing entity. - **Return `Result` (entity)** when the caller needs the loaded value; return `Result` (pass/fail) only when the entity is not needed downstream — this avoids a duplicate fetch in the service. - **Always `.ConfigureAwait(false)` on every `await`** — policy methods run in service context where thread-pool continuations matter. diff --git a/.github/skills/coreex-repository/SKILL.md b/.github/skills/coreex-repository/SKILL.md index 0bc070365..2162526c6 100644 --- a/.github/skills/coreex-repository/SKILL.md +++ b/.github/skills/coreex-repository/SKILL.md @@ -46,7 +46,7 @@ Guides you through creating or modifying a CoreEx Infrastructure-layer repositor - `[ScopedService]` on every repository class — auto-registers in DI - Primary constructor: relational example `public class ProductRepository(ProductsEfDb ef) : IProductRepository`; Cosmos example `public class CustomerRepository(CustomersCosmosDb cosmos) : ICustomerRepository`; always guard the injected accessor with `ThrowIfNull()` - Use the provider's typed accessors: `EfDb` delegate shortcuts (`GetAsync`, `CreateAsync`, `UpdateAsync`, `DeleteAsync`) for relational domains; `CosmosDbMappedContainer` / `CosmosDbContainer` delegates for Cosmos domains. Never write raw `DbContext` CRUD or instantiate a `CosmosClient` in the repository. -- `DataResult` return for Create/Update; `DataResult` for Delete — includes mutation flag for event decisions +- Create/Update return `DataResult` or `Result>`; Delete returns `DataResult` or `Result`. Use `dr.Map(value => SomeMapper.Map(value))` to preserve `WasMutated` through ROP/Domain mapping until Application gates mutation events; prefer the lambda to retain CoreEx mapper nullability guarantees. Do not implicitly unwrap or default the flag to true. Aggregate outcomes cover the whole parent/child operation; reads remain unchanged. - `*WithResultAsync` variants for `Result` ROP pipelines (per-project style choice) - `BiDirectionMapper`: override **both** `OnMap` overloads; map `Id` explicitly; **never** map `ETag` or `ChangeLog` — base mapper owns them - `QueryArgsConfig`: create a dedicated `{Name}QueryArgsConfig : QueryArgsConfig<{Name}QueryArgsConfig>` class per entity in `Infrastructure/Repositories/`; access via `.Default`; call `.Parse(query).ThrowOnError()` before use — never instantiate per-request. Relational queries materialize from `IQueryable` extensions; Cosmos queries materialize from `CosmosDbQuery` instance methods (`Container.Query(...).ToMappedItemsResultAsync(...)`). diff --git a/.github/skills/coreex-repository/references/workflow.md b/.github/skills/coreex-repository/references/workflow.md index 1555b14be..1b78458d5 100644 --- a/.github/skills/coreex-repository/references/workflow.md +++ b/.github/skills/coreex-repository/references/workflow.md @@ -376,34 +376,37 @@ Use when the project has elected to use `Result` pipelines for explicit failu ```csharp // Get — Persistence → Domain via mapper -public Task> GetAsync(string id) => Result - .GoAsync(() => _ef.{Name}s.GetWithResultAsync(id)) +public Task> GetAsync(string id, CancellationToken cancellationToken = default) => Result + .GoAsync(() => _ef.{Name}s.GetWithResultAsync(id, cancellationToken)) .ThenAs(model => {Name}Mapper.Map(model)); // Create — Domain → Persistence → Domain via into-mapper + create -public Task> CreateAsync(Domain.{Name} value) => Result +public Task>> CreateAsync(Domain.{Name} value, CancellationToken cancellationToken = default) => Result .Go(() => { var model = new Persistence.{Name}(); {Name}IntoMapper.MapInto(value, model); return model; }) - .ThenAsAsync(model => _ef.{Name}s.CreateWithResultAsync(model)) - .ThenAs(m => {Name}Mapper.Map(m)); + .ThenAsAsync(model => _ef.{Name}s.CreateWithResultAsync(model, cancellationToken)) + .ThenAs(dr => dr.Map(value => {Name}Mapper.Map(value))); // Update — load model, apply into-mapper, save, map back -public Task> UpdateAsync(Domain.{Name} value) => Result - .GoAsync(() => _ef.{Name}s.GetWithResultAsync(value.Id)) +public Task>> UpdateAsync(Domain.{Name} value, CancellationToken cancellationToken = default) => Result + .GoAsync(() => _ef.{Name}s.GetWithResultAsync(value.Id, cancellationToken)) .Then(model => { {Name}IntoMapper.MapInto(value, model); return model; }) - .ThenAsAsync(model => _ef.{Name}s.UpdateWithResultAsync(model)) - .ThenAs(m => {Name}Mapper.Map(m)); + .ThenAsAsync(model => _ef.{Name}s.UpdateWithResultAsync(model, cancellationToken)) + .ThenAs(dr => dr.Map(value => {Name}Mapper.Map(value))); + +public Task> DeleteAsync(string id, CancellationToken cancellationToken = default) + => _ef.{Name}s.DeleteWithResultAsync(id, cancellationToken); ``` -**Return types differ by operation:** `GetWithResultAsync` returns `Result` directly. `CreateWithResultAsync` / `UpdateWithResultAsync` return `Result>`; `DeleteWithResultAsync` returns `Result`. The `DataResult` → `T` implicit operator means mapper calls like `.ThenAs(m => {Name}Mapper.Map(m))` work without explicit `.Value` unwrapping — but use `.ThenAs(dr => dr.Value)` when you need the persistence model value explicitly. +**Return types differ by operation:** Reads return `Result`, but Create/Update return `Result>`; Delete returns `Result`. Keep these shapes in the Application repository interface as well. An implicit `DataResult` → `T` conversion into a mapper discards `WasMutated`, and constructing the mapped result without its original bool silently defaults to true. Use `dr.Map(value => {Name}Mapper.Map(value))` to transform the value while preserving the flag, including on successful no-ops. Prefer the lambda over a CoreEx mapper method group so `NotNullIfNotNull` preserves non-null destination inference. Aggregate mutation outcomes must cover the whole parent/child persistence operation, not merely an unrelated row. The service gates mutation-announcement events with `WhereMutated` in the same transaction, then unwraps its public response; aggregate state is an optimization, not a substitute for this provider signal. **Domain ↔ Persistence mappers are one-way** (`Mapper`) — they map from persistence to domain only; use a separate `{Name}IntoMapper` (`IntoMapper`) for the write direction. @@ -413,7 +416,7 @@ public Task> UpdateAsync(Domain.{Name} value) => Result 1. `dotnet build` — no errors or warnings. 2. Verify the repository interface in `Application/Repositories/` declares all new methods. -3. Verify the service calling the repository handles `null` returns (Get) and checks `DataResult.HasMutations` before publishing events (Create/Update/Delete). +3. Verify the service handles `null`/not-found Result on reads and gates mutation events on successful `DataResult.WasMutated` using `WhereMutated`. Preserve the flag through all Domain/DTO mapping and ROP before Application unwraps the value. 4. **Offer to create or update the matching integration test** in `*.Test.Integration/Repositories/` or `*.Test.Api/`. Integration tests seed via `Data/*.yaml` and assert round-trip fidelity. 5. Check `GlobalUsing.cs` for required namespace imports (`CoreEx.Data.Querying`, `{Solution}.Infrastructure.Mapping`). @@ -427,7 +430,7 @@ public Task> UpdateAsync(Domain.{Name} value) => Result - **`BiDirectionMapper.OnMap` — two overloads, same name, different source type** — do not invent `OnMapToPrimary` / `OnMapToSecondary`. - **`Id` must be mapped explicitly** in `OnMap` — the base does not auto-map it. - **`ETag` and `ChangeLog` must NOT be mapped** in `OnMap` — the base mapper owns them; the model has no `RowVersion` member. -- **`DataResult` for Create/Update, `DataResult` for Delete** — do not return `T` directly; the mutation flag is needed for event decisions. +- **Retain `DataResult` for mutations, including ROP/DDD** — Create/Update return `DataResult` or `Result>`; Delete returns `DataResult` or `Result`. Preserve `WasMutated` until Application gates mutation events; successful no-ops must not announce a mutation. Cover success/false after a successful earlier read as well as success/true and failure; a sequential delete-idempotency test alone does not cover that boundary. - **`[ScopedService]` on every repository** — do not add `AddScoped<>()` calls in `Program.cs` for repositories. - **`QueryArgsConfig`** — create one `{Name}QueryArgsConfig : QueryArgsConfig<{Name}QueryArgsConfig>` class per entity; access via `.Default`; call `.Parse(query).ThrowOnError()` before use — never instantiate per-request. Collect all field names, types, and operators from the developer before writing any code; AI cannot infer them from the entity shape. - **Match the database package to the project**: PostgreSQL → `CoreEx.Database.Postgres`, `PostgresDatabase`, `UseNpgsql`; SQL Server → `CoreEx.Database.SqlServer`, `SqlServerDatabase`, `UseSqlServer`. Check the project's `Program.cs` — do not introduce the wrong provider. diff --git a/.github/skills/coreex-test-api/SKILL.md b/.github/skills/coreex-test-api/SKILL.md index d1a37f3a6..c59e4a846 100644 --- a/.github/skills/coreex-test-api/SKILL.md +++ b/.github/skills/coreex-test-api/SKILL.md @@ -52,6 +52,7 @@ setup shared with Subscribe host tests (`coreex-test-subscribe` links back here - **Expectation helpers auto-exclude from JSON compare**: `ExpectIdentifier()`, `ExpectETag()`, `ExpectChangeLogCreated()`/`Updated()` — omit those fields from `.res.json` - **412 vs 409 vs 428**: stale ETag → `AssertPreconditionFailed()` (412); duplicate/business conflict → `AssertConflict()` (409); no ETag supplied → `428` (auto on PUT/PATCH, or POST/DELETE opted in via `ro.WithIfMatchRequired()`) — assert with `.Assert(HttpStatusCode.PreconditionRequired)`, no dedicated helper exists - **Delete is idempotent**: always `AssertNoContent()` (204), never `AssertNotFound()` on DELETE — the 404 belongs to the follow-up GET; only the first delete emits an outbox event +- **Mutation success is not mutation evidence**: cover successful `WasMutated == false` with no mutation event, as well as true and failure outcomes, including after a successful read. Sequential repeat-delete coverage alone does not exercise the read/delete gap; a focused boundary test can force this signal but cannot prove real concurrency/transaction behavior. - **Verify persistence, not just the echo**: every Create/Update/Patch `_Success` test explicitly asserts the specific changed property/properties on the mutation response (e.g. `updated.Salary.Should().Be(60000.00m)`), then ends with a follow-up `GET` + `AssertValue(...)` — the mutation response alone only proves the handler echoed the value back, not that the intended value was actually written - **Outbox assertions are provider-specific** — `ExpectPostgresOutboxEvents`/`ExpectSqlServerOutboxEvents`/`ExpectCosmosDbOutboxEvents`; `.AssertWithValue(destination, subject)` for value-carrying events (Create/Update), `.AssertMetadata(destination, subject, key)` for no-value events (Delete) — `key` is optional (`AssertMetadata(destination, subject)` checks routing only). If the event's payload type differs from the API's returned type (e.g. a leaner projection), use `.AssertWithValue(() => expectedPayload, destination, subject)` instead of the plain form - **Every event-publishing call needs an explicit expectation** — `UseExpectedXxxOutboxPublisher` defaults to asserting **zero** events when none is declared; omitting it on a call that legitimately publishes fails the test. Prefer `AssertMetadata`/`AssertWithValue` (confirms *which* event) over a bare `ExpectXxxOutboxEvents()` (confirms only that *some* event was published) diff --git a/.github/skills/coreex-test-api/references/workflow.md b/.github/skills/coreex-test-api/references/workflow.md index cec668331..c9b745343 100644 --- a/.github/skills/coreex-test-api/references/workflow.md +++ b/.github/skills/coreex-test-api/references/workflow.md @@ -429,6 +429,7 @@ values, include them; don't exclude them. The only volatile fields are `id`/`eta - [ ] One seed row per destructive test, not per operation - [ ] Provider-correct outbox helpers used throughout (no Postgres/SQL Server/Cosmos mixing) - [ ] Delete tested as the idempotent 4-step flow; never `AssertNotFound()` on DELETE +- [ ] Successful persistence no-op (`WasMutated == false`) produces success with no mutation event, including after a successful read; true produces the expected event, failure propagates without an event. Use focused boundary coverage if a real-provider no-op is not deterministic; sequential repeat-delete tests are not concurrency/read-delete-gap proof. - [ ] Create/Update/Patch `_Success` tests assert the specific changed property/properties explicitly (`updated.Prop.Should().Be(expected)`), then re-GET and `AssertValue(...)` to verify the change persisted (not just that the response echoed it back) - [ ] ETag concurrency asserted as 412, not 409; 428 case understood for missing-ETag scenarios - [ ] `.res.json` omits fields covered by `Expect*` auto-exclude helpers diff --git a/.github/skills/coreex-validator/SKILL.md b/.github/skills/coreex-validator/SKILL.md index 5d1cb4308..a225869c2 100644 --- a/.github/skills/coreex-validator/SKILL.md +++ b/.github/skills/coreex-validator/SKILL.md @@ -50,11 +50,13 @@ Guides you through creating or modifying a CoreEx validator (`Application/Valida | FluentValidation-style preferred | `AbstractValidator` | `RuleFor(x => ...)` / `NotEmpty()` / `IsValid()` — still CoreEx; has `Default` | **Key rules at a glance:** +- Request/input validation uses field-level validation errors; aggregate/entity/value-object business invariants use business errors with domain messages/codes. Keep both checks; never derive request paths in Domain or blanket-convert business errors. See [domain invariant conventions](/.github/instructions/coreex-domain.instructions.md#invariant-failures-versus-request-validation). - Ref-data: `.IsValid()` on the **navigation property** (`Gender`), never on `*Code` string - `Mandatory()` on a non-nullable value type errors on `default` (`0`, `MinValue`) — use a range rule if `0` is valid - Runtime-computed thresholds: use delegate overloads (`.LessThanOrEqualTo(_ => ...)`) — prefer over `OnValidateAsync` imperative logic - `context.HasErrors` guard before async I/O; `context.HasError(x => x.Prop)` for per-property guards - `context.AddError(x => x.Prop, ...)` — member-access expression, never `nameof(...)` +- Outside a validator context, prefer `ValidationException.Create(x => x.Address!.Street, "Street is invalid.")` for manual property errors. Property names resolve through metadata; entity access is never executed, so `!` is safe on nullable intermediates. Array/list indexes and string dictionary keys accept literals/captured locals: `addresses[0].street` / `addressesByType.home.street`. String keys follow `DictionaryKeyPolicy` (null preserves keys), matching serialized JSON and validator key/value errors; CLR paths and `GetDictionaryKey()` keep original keys. No escaping/full JSONPath support. Use `(expression, options, format, ...values)` for custom naming. No arbitrary index/key getters, calls, calculations, or intermediate casts; unlike `context.AddError`, no label/value formatting arguments or contextual naming overrides are applied. - Conditional rules on a dependency: `.DependsOn(x => x.Other, value => predicate)` skips its attached rule when the dependency is default/invalid and otherwise runs it only when the predicate is true. Clauses guard the rule they are attached to; use another `Property(...)` call for a separate root rule chain. Use `.WhenEntity(...)` for conditions based on broader entity state. - Message text argument is a `{2}` suffix substitution — not a full sentence; use `.Error("...")` to override the whole message - **Always generate or update the matching `{Validator}Tests`** in `*.Test.Unit/Validators/` — cover conditional/business rules and cross-field logic thoroughly, plus a representative error/success pair for simple rules; skip dedicated boundary tests for framework-guaranteed constraints like length rules (`MaximumLength`/`MinimumLength`/`Length`/`String`) @@ -63,6 +65,12 @@ For full workflow, rule reference, and code examples see [`references/workflow.m ## Key References +For aggregating manual errors outside a validator, use `MessageItem.CreateErrorMessage(x => x.Property, text)` and wrap the collection once. Omitted/null text infers the invalid message; `(expression, options, text)` supports custom naming. Exception/result factories share this message construction. + +Expression-based exception/result factories allow an omitted/null message, defaulting to `Validation.InvalidFormat` and the leaf property's friendly metadata label: `Result.ValidationError(x => x.Name)` yields `Name is invalid.`. Terminal indexes/keys use the containing property's label. Explicit messages remain unchanged; string-property factories keep their existing semantics. + +For Result-style manual errors outside a validator, use `Result.ValidationError(x => x.Name, "Name is invalid.")` or `Result.ValidationError(...)`. These accept an expression, `LText` message (including preformatted arguments), and optional exception configuration action. For explicit serializer options or separate formatting arguments, use `ValidationException.Create` directly and its implicit conversion to the result type. + - [`/.github/instructions/coreex-validators.instructions.md`](/.github/instructions/coreex-validators.instructions.md) — full rule set, comparison operators, localization, DependsOn, DI registration - [`/.github/instructions/coreex-tests.instructions.md`](/.github/instructions/coreex-tests.instructions.md) — validator unit test conventions: `Test.Scoped`, `AssertErrors`, expected message text - Related skills: [`coreex-app-service`](../coreex-app-service/SKILL.md) (invokes the validator), [`coreex-policy`](../coreex-policy/SKILL.md) (async I/O guard sibling for checks a validator cannot do), [`coreex-contract`](../coreex-contract/SKILL.md) (the type being validated) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4f220b3d..738be1450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ Represents the **NuGet** versions. ## v4.2.0 +- *Enhancement:* Added expression-based `MessageItem.CreateErrorMessage` factories for aggregating field errors, with inferred friendly invalid messages and optional JSON naming options. Validation exception factories share this message construction; policy guidance now distinguishes known request-contract expressions from explicit paths in reusable policies. - *Enhancement:* Added single-use fluent HTTP request builders with `Get`/`Post`/`Put`/`Patch`/`Delete` shortcuts, `WithJson` for JSON bodies and `WithContent` for explicit HTTP content (also available on `HttpRequestMessage`). Exactly one body assignment is permitted; repeated execution, configuration after execution starts, and conflicting body assignments throw `InvalidOperationException`. PATCH does not imply JSON Merge Patch. - *Enhancement:* Added `WithStatusResult` / `WithErrorStatus` to HTTP request builders, including validated builders, to map selected response statuses to CoreEx results before normal deserialization and validation. `ValidatedHttpRequestBuilder` integrates response validation with fluent request execution and automatic response disposal. - *Enhancement:* Added `HttpResponseMessage.GetStringAsync()` and `ToStringResultAsync()` for raw response text, returning a null value for successful empty/204 responses; corresponding terminal methods are available on `HttpRequestBuilder`. @@ -10,12 +11,21 @@ Represents the **NuGet** versions. - *Enhancement:* Added `IEfDb.SqlQuery(SqlStatement, params object[])` for deferred, parameterized, composable EF queries over scalar or unmapped result types. Queries use the existing context and transaction, not the CoreEx Database command pipeline; CoreEx tenant and logical-delete filters are not automatically applied. - *Enhancement:* Added `ReferenceDataOrchestrator.PrefetchAsync(IEnumerable)` for typed reference-data prefetch, validating all requested types before loading, deduplicating them and honoring configured parallelism. Name-based prefetch now also checks cancellation immediately. - *Enhancement:* Added a predicate overload for validation `DependsOn`, allowing a rule to run only when its dependency is non-default, has no validation error, and satisfies an additional condition. +- *Enhancement:* Added expression-based `ValidationException.Create` factories and `RuntimeMetadata.GetJsonPath` for direct/nested property errors, including array/list indexes and string dictionary keys supplied as literals or captured locals. Resolves property JSON names from runtime metadata without executing entity access, using existing validator path notation; supports explicit serializer naming options and existing `LText` formatting. +- *Enhancement:* Added expression-based `Result.ValidationError` and `Result.ValidationError` shorthand factories accepting a property expression, `LText` message, and optional exception configuration, using the same default JSON paths as `ValidationException.Create`. +- *Enhancement:* Expression-based validation exception and result factories now infer the standard invalid message when text is omitted/null, using `Validation.InvalidFormat` and friendly/localized property text resolved alongside the JSON path by `RuntimeMetadata.GetJsonPathAndText`. - *Enhancement:* Updated the shipped AI guidance and reference-data documentation to explain implicit conversion from reference-data codes to concrete typed values, the annotated contract code-property pattern, and the separation of conversion from validation. - *Enhancement:* Added OData-style free-text `$search` support to the `QueryArgs` querying pipeline: `QueryArgs.Search`; `QueryArgsConfig.WithSearch(Action)` with `QuerySearchMode` (`Keyword` executed by `QueryExtensions.Where` as a provider-agnostic predicate where every term must match a configured keyword field; `FullText`/`Semantic`/`Hybrid` are relevance-ranked and executed by the repository via `QueryArgsParseResult.IsRankedSearch`/`SearchResult.Text`, with `$orderby` rejected); `QuerySearchParserException`; `WithSearch` on `HttpRequestMessage`/`HttpRequestBuilder`; `QueryAttribute.SupportsSearch` (NSwag `$search` parameter); and a GraphQL-lite `search` argument on list roots. `$search` is rejected where not configured. The Products sample (and its MCP `search_products` tool) now uses `$search` to demonstrate. +- *Enhancement:* Added `DataResult.Map` to transform mutation-result values while preserving `WasMutated`, including successful no-ops; simplified Shopping repository mapping and aligned usage guidance. - *Behavior change:* Added a string-specific `Required()` validation overload that rejects null, empty and whitespace values with a `ValidationException`. Existing string call sites now reject empty/whitespace strings rather than only null. +- *Behavior change:* Preserve Shopping aggregate mutation metadata through repository mapping and gate Create/Update/Checkout mutation events with `WhereMutated`; align repository/service guidance and add successful-no-op publication regressions. +- *Behavior change:* Aligned Domain guidance and Shopping pricing/discount examples on business errors for aggregate/entity/value-object invariants, distinct from request validation. Pricing rejections (negative values, invalid/inactive unit of measure, quantity scale) now throw coded `BusinessException`; invalid/inactive discount application returns a coded business failure. Request validators retain field errors; shared argument/reference-data guards keep their existing exception semantics. - *Maintenance:* Updated framework dependencies and corresponding `CoreEx.Template` dependencies, including Aspire 13.6.1, EF/.NET servicing packages and FusionCache 2.9.0. The SQL Server package now explicitly references `System.Security.Cryptography.Pkcs` 10.0.13; the Aspire template's MessagePack override is updated to 2.5.307. - *Maintenance:* Pinned local development, CI and package publishing to the stable .NET SDK in `global.json`; updated `nuget-publish.ps1` to build before packing and remove matching package versions from the local NuGet cache by default. +- *Documentation:* Aligned application/API instructions, policy/service/adapter skills, and package/sample guidance on missing target resources (404) versus invalid request-body references (field-level 400), including query filters, expected downstream lookup outcomes, and deliberate endpoint semantics. - *Fixed:* Named/typed HTTP client registration now binds standard resilience settings from the nested `Resilience` configuration section rather than the parent client section. +- *Fixed:* Dictionary validation JSON paths now apply the effective serializer's `DictionaryKeyPolicy` to string keys for both key and value errors, matching serialized JSON and expression-based exception factories. Original keys remain unchanged in CLR paths and `GetDictionaryKey()`; affected JSON error paths now use the serialized casing (for example, `HomeAddress` becomes `homeAddress` with the default policy). +- *Fixed:* Shared cached property metadata lookups now consistently prefer static `IRuntimeMetadata` providers, including explicit implementations, with reflection as the fallback for other types. JSON path/friendly-label resolution respects custom/generated metadata for both root and nested types regardless of cache-population order. Explicit expression lookups separately cache reflected metadata for properties omitted by the static provider (including generated reference-data navigation properties), preserving validation support without changing contract equality/copy/clean property collections. - *Fixed:* GraphQL-lite search parsing errors now preserve actionable per-property validation details in `extensions.messages["$search"]` while retaining the `SEARCH_PARSE_ERROR` code, including unsupported search, length/term limits and custom validation failures. - *Fixed:* `CoreEx.UnitTesting` and the sample/template MockHost now explicitly reference MessagePack 2.5.307 so their transitive Aspire dependency cannot resolve the vulnerable 2.5.302 version; a central version entry alone does not pin transitive dependencies. - *Fixed:* SQL Server and PostgreSQL unit-of-work root transactions now honor `DatabaseArgs.IsolationLevel`; nested invocations use savepoints and inherit the existing transaction's isolation level. diff --git a/docs/capabilities.md b/docs/capabilities.md index 1d66582a0..a65e1f92d 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -74,6 +74,8 @@ CoreEx exception types include: - `AuthenticationException` — Unauthorized (401). - `AuthorizationException` — Forbidden (403). +Request/input validity uses validation errors with request-field context where known. Aggregate/entity/value-object business invariants use business errors with domain messages/codes, never request DTOs or JSON paths. Both normally map to 400, but their semantics and diagnostics differ. Keep domain enforcement after request validation, retain argument/invalid-operation exceptions for programming misuse, and translate known domain failures to request fields only deliberately in Application. See [domain error conventions](../samples/docs/domain-layer.md#business-invariants-versus-request-validation). + **Why it matters:** Instead of throwing generic `Exception` or returning error codes, you throw domain-specific exceptions that middlewares automatically convert to RFC 9457 ProblemDetails responses. This keeps error handling logic centralized and consistent across APIs. **Example:** @@ -1013,7 +1015,7 @@ public class Basket : IIdentifier, IETag, IChangeLog public void Checkout() { if (!_items.Any()) - throw new ValidationException("Cannot checkout empty basket."); + throw new BusinessException("Cannot checkout empty basket.").WithErrorCode("empty-basket"); StatusCode = "CheckedOut"; } diff --git a/samples/docs/application-layer.md b/samples/docs/application-layer.md index 0e0a59f99..b37613706 100644 --- a/samples/docs/application-layer.md +++ b/samples/docs/application-layer.md @@ -55,6 +55,8 @@ Each service interface lives in `Application/Interfaces/` and exposes only what Repository interfaces (`Application/Repositories/`) define the **data contract** for the Application layer without coupling it to any persistence technology. They surface domain-idiomatic operations (`GetAsync`, `CreateAsync`, `QueryAsync`, etc.) and return CoreEx result types (`DataResult`, `ItemsResult`) rather than raw EF entities. +The Shopping DDD/ROP repository uses `Result>` for Create/Update; reads still return `Result`. The service maps `dr.Value` to a contract, gates mutation-announcement events with `dr.WhereMutated(...)` inside the write transaction, then returns the contract. Successful no-ops emit no mutation event even if an earlier read found the aggregate and `HasChanges` indicated intent to save. This applies to Create, shared Update, and Checkout; the reservation-confirmation command still follows its own successful-reservation semantics. For repositories exposing Delete, retain `Result`/`DataResult` and use parameterless `WhereMutated(() => ...)`. Sequential delete idempotency does not exercise the read/delete gap; the provider may report a no-op or a concurrency failure, and the latter must propagate unchanged. + ```csharp // samples/src/Contoso.Products.Application/Repositories/IProductRepository.cs public interface IProductRepository @@ -224,6 +226,27 @@ public class BasketMapper : Mapper; preserve unrelated failures and successful values. +return lookup.IsNotFoundError + ? Result.ValidationError(x => x.TravellerId, "Traveller was not found.") + : lookup; +``` + +For nullable lookups, check null instead. In exception-style services use `ValidationException.Create(...)`; for aggregation use `MessageItem.CreateErrorMessage(...)`; inside validators keep using `context.AddError(...)` for contextual paths. Reusable policies receiving only an identifier can keep explicit caller-agreed JSON paths (or accept a resolved path); do not add a request DTO dependency solely for the shorthand. `nameof(param)` does not resolve JSON naming. + +### Sample reference policies + +The following Product policy checks a body reference whose agreed JSON path is `"productId"`; the literal is already the resolved path, not a parameter-name-to-JSON conversion. + ```csharp // samples/src/Contoso.Shopping.Application/Policies/ProductPolicy.cs public class ProductPolicy(IProductAdapter productAdapter) @@ -231,9 +254,9 @@ public class ProductPolicy(IProductAdapter productAdapter) public Task> EnsureExistsAsync(string productId) => Result .GoAsync(() => _productAdapter.GetAsync(productId)) .OnFailure(r => r.IsNotFoundError - ? Result.ValidationError(MessageItem.CreateErrorMessage(nameof(productId), "Product was not found.")) + ? Result.ValidationError(MessageItem.CreateErrorMessage("productId", "Product was not found.")) : r); } ``` -`CustomerPolicy` (`EnsureExistsAsync`) follows the same shape for the basket's `customerId`, returning a `customerId` validation error ("Customer was not found.") when the Customers API responds 404. +`CustomerPolicy` (`EnsureExistsAsync`) follows the same shape for the basket's `customerId`, returning a `customerId` validation error ("Customer was not found.") when the adapter reports the expected customer-not-found outcome from the Customers lookup endpoint. diff --git a/samples/docs/domain-layer.md b/samples/docs/domain-layer.md index 551c8a1c3..2aa8b9a74 100644 --- a/samples/docs/domain-layer.md +++ b/samples/docs/domain-layer.md @@ -55,14 +55,36 @@ Value objects represent concepts that have no independent identity — they are // samples/src/Contoso.Shopping.Domain/ValueObjects/ItemPricing.cs public sealed record class ItemPricing { - public required Contracts.UnitOfMeasure UnitOfMeasure { get; init => field = value.ThrowIfInactive(); } - public decimal UnitPrice { get; init => field = value.ThrowIfLessThanZero(); } - public decimal Quantity { get; init => field = value.ThrowIfLessThanZero(); } + public required Contracts.UnitOfMeasure UnitOfMeasure + { + get; + init + { + value.ThrowIfNull(); + if (!value.IsValid || !value.IsActive) + throw new BusinessException("Pricing requires a valid, active unit of measure.").WithErrorCode("invalid-unit-of-measure"); + + field = value; + } + } + public decimal UnitPrice { get; init => field = value < 0 ? throw new BusinessException("Unit price cannot be negative.").WithErrorCode("negative-unit-price") : value; } + public decimal Quantity { get; init => field = value < 0 ? throw new BusinessException("Quantity cannot be negative.").WithErrorCode("negative-quantity") : value; } public decimal Total => UnitPrice * Quantity; + + public ItemPricing EnsureIsValid() => DecimalRuleHelper.CheckScale(Quantity, UnitOfMeasure.Scale) ? this + : throw new BusinessException($"Quantity decimal places exceed the unit-of-measure scale of {UnitOfMeasure.Scale}.").WithErrorCode("quantity-scale-exceeded"); } ``` -Value objects enforce their own invariants in property initialisers using CoreEx guard extensions (`ThrowIfInactive`, `ThrowIfLessThanZero`) so that invalid instances simply cannot exist. +Single-property business invariants are enforced in `init` setters; the owning entity invokes `EnsureIsValid` for cross-property invariants before accepting the value object. `with` expressions run setters, but do not automatically run `EnsureIsValid`; retain that ownership check. + +### Business invariants versus request validation + +Application validators/policies use `ValidationException` / `Result.ValidationError` for request/input validity, with request-field feedback where known. Aggregates, entities, and value objects enforce business invariants with `BusinessException` / `Result.BusinessError`, meaningful domain messages, and stable codes where useful, not request paths. Both normally map to HTTP 400; they remain different semantic and diagnostic contracts. Never reference request DTOs or derive request JSON paths inside Domain. Application may selectively translate a known domain error when a specific request-field mapping is unambiguous, but must not blanket-convert business errors into validation. + +Keep domain enforcement after request validation: other callers, persisted state, and concurrent changes still require invariant protection. Exception versus `Result` is separate: constructors/setters may throw business exceptions, result-returning operations generally return business failures, and `OnCheckCanMutate` failures throw internally through `Modify`/`Remove`. + +Argument/invalid-operation exceptions remain appropriate for programming misuse or broken internal assumptions, such as a required internal object being null or an internally assigned status being invalid. The general `ThrowIfLessThanZero` helper throws `ArgumentOutOfRangeException`; reference-data `ThrowIfInvalid`/`ThrowIfInactive` throw `InvalidOperationException`, not business exceptions. Do not use those guards as substitutes for expected business-rule rejection or change their shared semantics. Shopping pricing and discount application use explicit business checks, while internal identifier/object/status guards remain programming-assumption checks. > **See also**: [Value Object pattern](https://learn.microsoft.com/en-us/dotnet/architecture/microservices/microservice-ddd-cqrs-patterns/implement-value-objects) @@ -78,14 +100,16 @@ CoreEx deliberately does not provide a native domain-event mechanism (e.g. Media ```csharp // Events are added by the application SERVICE after the repository operation — not by the aggregate itself -return ur.ThenAs(basket => +return ur.ThenAs(dr => { - var contract = BasketMapper.Map(basket); - _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.CheckedOut)); + var contract = BasketMapper.Map(dr.Value); + dr.WhereMutated(_ => _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.CheckedOut))); return contract; }); ``` +Here `ur` is the repository's `Result>`, evaluated inside the write transaction. Domain mapping preserves `WasMutated` until Application decides whether to announce a mutation; success alone, an earlier successful read, or `HasChanges` is insufficient. Successful no-ops return a contract without a mutation event. Commands and deliberate attempt messages follow their own semantics; provider mutation flags do not guarantee committed rows or exactly-once delivery. + A developer can opt in to a domain-event mechanism if genuinely needed — for example, dispatching via MediatR after the transaction commits — but this is an explicit extension, not a framework default. > **See also**: [`IAggregateRoot`](../../src/CoreEx.DomainDriven/IAggregateRoot.cs) · [`CoreEx.DomainDriven` README](../../src/CoreEx.DomainDriven/README.md#domain-events--intentionally-not-supported) diff --git a/samples/docs/infrastructure-layer.md b/samples/docs/infrastructure-layer.md index 8c85398cb..bcd3b660b 100644 --- a/samples/docs/infrastructure-layer.md +++ b/samples/docs/infrastructure-layer.md @@ -35,6 +35,8 @@ public partial class Product : ModelBase, ILogicallyDeleted Repository implementations live in `Repositories/` and implement the `IXxxRepository` interfaces declared in the Application layer. They use CoreEx's `EfDb` abstraction, which provides strongly-typed, mapper-aware access patterns (`GetAsync`, `CreateAsync`, `UpdateAsync`, `DeleteAsync`, `Query`) built on top of EF Core, adding CoreEx's `DataResult` return semantics, change-tracking integration, and query filtering support. +These mutation outcomes survive Domain mapping and ROP: Shopping's BasketRepository returns `Result>` for Create/Update and maps with `.ThenAs(dr => dr.Map(value => BasketMapper.Map(value)))`, not an implicit conversion to the aggregate. `Map` transforms the value even on a no-op and preserves `WasMutated`; the lambda retains CoreEx's conditional mapper nullability guarantee. Delete repositories retain `DataResult`/`Result`. The flag must represent the complete parent/child operation; Shopping explicitly marks the parent modified when synchronizing items, preserving aggregate-wide mutation tracking. Application decides whether to announce the mutation before unwrapping the value. This is a provider mutation signal; transaction commit still determines the atomic outcome. + ```csharp // samples/src/Contoso.Products.Infrastructure/Repositories/ProductRepository.cs [ScopedService] diff --git a/samples/src/Contoso.Shopping.Application/BasketService.cs b/samples/src/Contoso.Shopping.Application/BasketService.cs index 6db82209b..3c0ed7a10 100644 --- a/samples/src/Contoso.Shopping.Application/BasketService.cs +++ b/samples/src/Contoso.Shopping.Application/BasketService.cs @@ -25,13 +25,13 @@ public async Task> CreateAsync(string customerId, CancellationTok { // Create the aggregate in the repository, which will return the created aggregate with any updates (e.g. Id). var br = await _repository.CreateAsync(aggregate, tct).ConfigureAwait(false); - return br.ThenAs(b => + return br.ThenAs(dr => { // Map the result to the contract representation. - var contract = BasketMapper.Map(b); + var contract = BasketMapper.Map(dr.Value); - // Add an event for the creation of the basket. - _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.Created)); + // Only announce a creation when persistence reports a mutation. + dr.WhereMutated(_ => _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.Created))); return contract; }); @@ -115,7 +115,7 @@ private async Task> OrchestrateUpdateAsync(string basketId, Func< } /// - /// Performs the "actual" update of the basket within a transaction, ensuring that any events are only published if the transaction is successful. + /// Persists the basket within a transaction, announcing an update only when persistence reports a mutation. /// private Task> UpdateAsync(Domain.Basket basket, EventAction action, CancellationToken ct) => _unitOfWork.TransactionAsync(async tct => { @@ -125,13 +125,13 @@ private Task> UpdateAsync(Domain.Basket basket, EventAction actio // Update the aggregate in the repository, which will return the updated aggregate with any updates (e.g. Id). var ur = await _repository.UpdateAsync(basket, tct).ConfigureAwait(false); - return ur.ThenAs(basket => + return ur.ThenAs(dr => { // Map the result to the contract representation. - var contract = BasketMapper.Map(basket); + var contract = BasketMapper.Map(dr.Value); - // Add an event for the update of the basket. - _unitOfWork.Events.Add(EventData.CreateEventWith(contract, action)); + // Aggregate state expresses intent; persistence decides whether to announce a mutation. + dr.WhereMutated(_ => _unitOfWork.Events.Add(EventData.CreateEventWith(contract, action))); return contract; }); @@ -148,18 +148,16 @@ private Task> UpdateAsync(Domain.Basket basket, EventAction actio { return await _unitOfWork.TransactionAsync(tct => Result .GoAsync(() => _repository.UpdateAsync(basket, tct)) // Update the basket aggregate to reflect the checkout (e.g. status change). - .ThenAsAsync(async basket => + .ThenAsAsync(async dr => { // Map the result to the contract representation. - var contract = BasketMapper.Map(basket); + var contract = BasketMapper.Map(dr.Value); - // Outbox the "checked-out" event for the basket aggregate. - _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.CheckedOut)); + // Only announce checkout when persistence reports a mutation. + dr.WhereMutated(_ => _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.CheckedOut))); - // Note: the reservation confirmation is outboxed as a command to the products domain, as it's an action that we want to occur asynchronously after the transaction has completed, but it doesn't - // necessarily need to be performed immediately after the checkout event (e.g. it could be performed after some delay, or after some other events have been processed). By using a command, we - // also have more flexibility in how we handle failures and retries for the reservation confirmation, without impacting the checkout event processing. - return (await _productAdapter.CreateConfirmReservationCommand(basket, tct).ConfigureAwait(false)) + // Outbox confirmation of the successful reservation even when persistence reports no mutation. + return (await _productAdapter.CreateConfirmReservationCommand(dr.Value, tct).ConfigureAwait(false)) .ThenAs(() => contract); }), ct).ConfigureAwait(false); } diff --git a/samples/src/Contoso.Shopping.Application/Repositories/IBasketRepository.cs b/samples/src/Contoso.Shopping.Application/Repositories/IBasketRepository.cs index e812f0578..1589368ca 100644 --- a/samples/src/Contoso.Shopping.Application/Repositories/IBasketRepository.cs +++ b/samples/src/Contoso.Shopping.Application/Repositories/IBasketRepository.cs @@ -4,9 +4,9 @@ public interface IBasketRepository { Task> GetAsync(string id, CancellationToken ct = default); - Task> CreateAsync(Domain.Basket basket, CancellationToken ct = default); + Task>> CreateAsync(Domain.Basket basket, CancellationToken ct = default); - Task> UpdateAsync(Domain.Basket basket, CancellationToken ct = default); + Task>> UpdateAsync(Domain.Basket basket, CancellationToken ct = default); /// /// Gets the schema. @@ -21,4 +21,4 @@ public interface IBasketRepository /// The . /// The resulting baskets. Task> QueryAsync(string customerId, QueryArgs? query, PagingArgs? paging, CancellationToken ct = default); -} \ No newline at end of file +} diff --git a/samples/src/Contoso.Shopping.Domain/Basket.cs b/samples/src/Contoso.Shopping.Domain/Basket.cs index 1ebae7692..9cf511d05 100644 --- a/samples/src/Contoso.Shopping.Domain/Basket.cs +++ b/samples/src/Contoso.Shopping.Domain/Basket.cs @@ -64,7 +64,10 @@ protected override void OnMutate() /// public Result ApplyDiscount(Contracts.DiscountCoupon discountCoupon) { - discountCoupon.ThrowIfNull().ThrowIfInactive(); + discountCoupon.ThrowIfNull(); + if (!discountCoupon.IsValid || !discountCoupon.IsActive) + return Result.BusinessError("A valid, active discount coupon is required.", c => c.WithKey(Id).WithErrorCode("invalid-discount-coupon")); + if (discountCoupon != DiscountCoupon) Modify(() => DiscountCoupon = discountCoupon); diff --git a/samples/src/Contoso.Shopping.Domain/ValueObjects/ItemPricing.cs b/samples/src/Contoso.Shopping.Domain/ValueObjects/ItemPricing.cs index 548216efe..d61d4248b 100644 --- a/samples/src/Contoso.Shopping.Domain/ValueObjects/ItemPricing.cs +++ b/samples/src/Contoso.Shopping.Domain/ValueObjects/ItemPricing.cs @@ -2,14 +2,25 @@ namespace Contoso.Shopping.Domain.ValueObjects; public sealed record class ItemPricing { - public required Contracts.UnitOfMeasure UnitOfMeasure { get; init => field = value.ThrowIfInactive(); } + public required Contracts.UnitOfMeasure UnitOfMeasure + { + get; + init + { + value.ThrowIfNull(); + if (!value.IsValid || !value.IsActive) + throw new BusinessException("Pricing requires a valid, active unit of measure.").WithErrorCode("invalid-unit-of-measure"); - public decimal UnitPrice { get; init => field = value.ThrowIfLessThanZero(); } + field = value; + } + } - public decimal Quantity { get; init => field = value.ThrowIfLessThanZero(); } + public decimal UnitPrice { get; init => field = value < 0 ? throw new BusinessException("Unit price cannot be negative.").WithErrorCode("negative-unit-price") : value; } + + public decimal Quantity { get; init => field = value < 0 ? throw new BusinessException("Quantity cannot be negative.").WithErrorCode("negative-quantity") : value; } public decimal Total => UnitPrice * Quantity; public ItemPricing EnsureIsValid() => DecimalRuleHelper.CheckScale(Quantity, UnitOfMeasure.Scale) ? this - : throw new ValidationException($"Quantity decimal places exceed the specified unit-of-measure ({UnitOfMeasure.Text}) configuration of {UnitOfMeasure.Scale}."); -} \ No newline at end of file + : throw new BusinessException($"Quantity decimal places exceed the specified unit-of-measure ({UnitOfMeasure.Text}) configuration of {UnitOfMeasure.Scale}.").WithErrorCode("quantity-scale-exceeded"); +} diff --git a/samples/src/Contoso.Shopping.Infrastructure/Repositories/BasketRepository.cs b/samples/src/Contoso.Shopping.Infrastructure/Repositories/BasketRepository.cs index 9d4f2e4f7..4dd362771 100644 --- a/samples/src/Contoso.Shopping.Infrastructure/Repositories/BasketRepository.cs +++ b/samples/src/Contoso.Shopping.Infrastructure/Repositories/BasketRepository.cs @@ -9,7 +9,7 @@ public class BasketRepository(ShoppingEfDb ef) : IBasketRepository .GoAsync(() => _ef.Baskets.GetWithResultAsync(id, ct)) .ThenAs(model => BasketMapper.Map(model)); - public Task> CreateAsync(Domain.Basket basket, CancellationToken ct = default) => Result + public Task>> CreateAsync(Domain.Basket basket, CancellationToken ct = default) => Result .Go(() => { var model = new Persistence.Basket(); @@ -17,9 +17,9 @@ public class BasketRepository(ShoppingEfDb ef) : IBasketRepository return SynchronizeItems(basket, model); }) .ThenAsAsync(model => _ef.Baskets.CreateWithResultAsync(model, ct)) - .ThenAs(b => BasketMapper.Map(b)); + .ThenAs(dr => dr.Map(value => BasketMapper.Map(value))); - public Task> UpdateAsync(Domain.Basket basket, CancellationToken ct = default) => Result + public Task>> UpdateAsync(Domain.Basket basket, CancellationToken ct = default) => Result .GoAsync(() => _ef.Baskets.GetWithResultAsync(basket.Id, ct)) .Then(model => { @@ -27,7 +27,7 @@ public class BasketRepository(ShoppingEfDb ef) : IBasketRepository return SynchronizeItems(basket, model); }) .ThenAsAsync(model => _ef.Baskets.UpdateWithResultAsync(model, ct)) - .ThenAs(basket => BasketMapper.Map(basket)); + .ThenAs(dr => dr.Map(value => BasketMapper.Map(value))); /// public Task QuerySchemaAsync(CancellationToken ct = default) => Task.FromResult(BasketQueryArgsConfig.Default.ToJsonSchema()); diff --git a/samples/tests/Contoso.Shopping.Test.Unit/Domains/BasketTests.cs b/samples/tests/Contoso.Shopping.Test.Unit/Domains/BasketTests.cs index 898d94646..caaf3a3a7 100644 --- a/samples/tests/Contoso.Shopping.Test.Unit/Domains/BasketTests.cs +++ b/samples/tests/Contoso.Shopping.Test.Unit/Domains/BasketTests.cs @@ -2,6 +2,39 @@ namespace Contoso.Shopping.Test.Unit.Domains; public class BasketTests : WithGenericTester { + [TestCase(-1, "negative-quantity")] + [TestCase(1.5, "quantity-scale-exceeded")] + public void Basket_ItemUpdate_InvalidPricing_ThrowsWithoutChangingState(decimal quantity, string code) => Test.Scoped(test => + { + var item = Domain.BasketItem.CreateFrom("item-id", "product-id", "sku", "text", new Domain.ValueObjects.ItemPricing { UnitOfMeasure = "EA", Quantity = 1, UnitPrice = 10 }, null); + var basket = Domain.Basket.CreateFrom("basket-id", "customer-id", BasketStatus.Active, null, null, [item], null, null); + + Action act = () => basket.ItemUpdate("item-id", quantity, "new-etag"); + + act.Should().Throw().Which.ErrorCode.Should().Be(code); + basket.Items.Single().Pricing.Quantity.Should().Be(1); + basket.Items.Single().ETag.Should().BeNull(); + basket.Total.Should().Be(10); + basket.HasChanges.Should().BeFalse(); + }); + + [TestCase(false)] + [TestCase(true)] + public void Basket_ApplyDiscount_InvalidCoupon_ReturnsBusinessError(bool inactive) => Test.Scoped(test => + { + var basket = Domain.Basket.CreateNew("customer-id"); + var coupon = new DiscountCoupon { Code = "INVALID", IsInactive = inactive }; + if (!inactive) + ((IReferenceData)coupon).SetInvalid(); + + var result = basket.ApplyDiscount(coupon); + + result.IsBusinessError.Should().BeTrue(); + result.Error.Should().BeOfType().Which.ErrorCode.Should().Be("invalid-discount-coupon"); + basket.DiscountCoupon.Should().BeNull(); + basket.Total.Should().Be(0); + }); + [Test] public void Basket_ApplyDiscount_Success() => Test.Scoped(test => { diff --git a/samples/tests/Contoso.Shopping.Test.Unit/Domains/ItemPricingTests.cs b/samples/tests/Contoso.Shopping.Test.Unit/Domains/ItemPricingTests.cs new file mode 100644 index 000000000..81cb1d640 --- /dev/null +++ b/samples/tests/Contoso.Shopping.Test.Unit/Domains/ItemPricingTests.cs @@ -0,0 +1,53 @@ +namespace Contoso.Shopping.Test.Unit.Domains; + +public class ItemPricingTests : WithGenericTester +{ + [Test] + public void ItemPricing_NullInternalValue_RemainsArgumentError() => Test.Scoped(test => + { + // Simulates programming misuse; the null-forgiving operator only bypasses compile-time checking. + Action act = () => _ = new Domain.ValueObjects.ItemPricing { UnitOfMeasure = null! }; + act.Should().Throw(); + }); + + [TestCase(true, "negative-unit-price", "Unit price cannot be negative.")] + [TestCase(false, "negative-quantity", "Quantity cannot be negative.")] + public void ItemPricing_NegativeValue_ThrowsBusinessException(bool price, string code, string message) => Test.Scoped(test => + { + Action act = () => _ = new Domain.ValueObjects.ItemPricing { UnitOfMeasure = "EA", UnitPrice = price ? -1 : 1, Quantity = price ? 1 : -1 }; + act.Should().Throw().WithMessage(message).Which.ErrorCode.Should().Be(code); + }); + + [TestCase(false)] + [TestCase(true)] + public void ItemPricing_InvalidUnitOfMeasure_ThrowsBusinessException(bool inactive) => Test.Scoped(test => + { + UnitOfMeasure unit = inactive ? new UnitOfMeasure { Code = "OLD", IsInactive = true } : "UNKNOWN"; + Action act = () => _ = new Domain.ValueObjects.ItemPricing { UnitOfMeasure = unit }; + act.Should().Throw().WithMessage("Pricing requires a valid, active unit of measure.").Which.ErrorCode.Should().Be("invalid-unit-of-measure"); + }); + + [Test] + public void ItemPricing_QuantityScale_ThrowsBusinessException() => Test.Scoped(test => + { + var pricing = new Domain.ValueObjects.ItemPricing { UnitOfMeasure = "EA", UnitPrice = 10, Quantity = 1.5m }; + Action act = () => pricing.EnsureIsValid(); + act.Should().Throw().WithMessage("Quantity decimal places exceed the specified unit-of-measure (Each) configuration of 0.").Which.ErrorCode.Should().Be("quantity-scale-exceeded"); + + Action create = () => Domain.BasketItem.CreateNew("product", "sku", "Product", pricing); + create.Should().Throw().Which.ErrorCode.Should().Be("quantity-scale-exceeded"); + }); + + [Test] + public void ItemPricing_ValidValues_PreserveTotalsAndCopyGuards() => Test.Scoped(test => + { + var pricing = new Domain.ValueObjects.ItemPricing { UnitOfMeasure = "HR", UnitPrice = 10, Quantity = 1.25m }.EnsureIsValid(); + pricing.Total.Should().Be(12.5m); + var zero = new Domain.ValueObjects.ItemPricing { UnitOfMeasure = "EA", UnitPrice = 0, Quantity = 0 }.EnsureIsValid(); + zero.Total.Should().Be(0); + + Action copy = () => _ = pricing with { Quantity = -1 }; + copy.Should().Throw().Which.ErrorCode.Should().Be("negative-quantity"); + pricing.Quantity.Should().Be(1.25m); + }); +} diff --git a/samples/tests/Contoso.Shopping.Test.Unit/GlobalUsing.cs b/samples/tests/Contoso.Shopping.Test.Unit/GlobalUsing.cs index 38717e64f..95be8b395 100644 --- a/samples/tests/Contoso.Shopping.Test.Unit/GlobalUsing.cs +++ b/samples/tests/Contoso.Shopping.Test.Unit/GlobalUsing.cs @@ -1,28 +1,32 @@ global using AwesomeAssertions; -global using Contoso.Shopping.Application; global using Contoso.Shopping.Application.Adapters.Customers; global using Contoso.Shopping.Application.Adapters.Products; global using Contoso.Shopping.Application.Policies; global using Contoso.Shopping.Application.Repositories; global using Contoso.Shopping.Application.Validators; +global using Contoso.Shopping.Application; global using Contoso.Shopping.Contracts; global using Contoso.Shopping.Infrastructure.Adapters.Customers; global using Contoso.Shopping.Infrastructure.Clients.Customers; global using Contoso.Shopping.Infrastructure.Clients.Products; global using Contoso.Shopping.Infrastructure.Clients.SendGrid; -global using CoreEx; -global using CoreEx.RefData; +global using CoreEx.Data.Json; +global using CoreEx.Data; +global using CoreEx.Events.Publishing; +global using CoreEx.Events; +global using CoreEx.Http; global using CoreEx.RefData.Abstractions; +global using CoreEx.RefData; global using CoreEx.Results; global using CoreEx.UnitTesting; -global using CoreEx.Data.Json; -global using CoreEx.Http; global using CoreEx.Validation; +global using CoreEx; +global using ExecutionContext = CoreEx.ExecutionContext; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging.Abstractions; global using Moq; -global using System.Net; global using System.Net.Http; -global using UnitTestEx; +global using System.Net; global using UnitTestEx.Mocking; -global using ExecutionContext = CoreEx.ExecutionContext; +global using UnitTestEx; diff --git a/samples/tests/Contoso.Shopping.Test.Unit/Services/BasketMutationTests.cs b/samples/tests/Contoso.Shopping.Test.Unit/Services/BasketMutationTests.cs new file mode 100644 index 000000000..4644dc90b --- /dev/null +++ b/samples/tests/Contoso.Shopping.Test.Unit/Services/BasketMutationTests.cs @@ -0,0 +1,152 @@ +namespace Contoso.Shopping.Test.Unit.Services; + +public class BasketMutationTests : WithGenericTester +{ + private static readonly Result _writeFailure = Result.ConflictError("Write failed."); + + [TestCase("mutated")] + [TestCase("noop")] + [TestCase("failed")] + public void Create_PublishesOnlySuccessfulMutations(string outcome) => Test.Scoped(async test => + { + var harness = new Harness(); + harness.Customers.Setup(x => x.GetAsync("customer-id", It.IsAny())) + .ReturnsAsync(Result.Ok(new Customer { Id = "customer-id" })); + harness.Repository.Setup(x => x.CreateAsync(It.IsAny(), It.IsAny())) + .Returns((Domain.Basket basket, CancellationToken _) => + { + harness.InTransaction.Should().BeTrue(); + return Task.FromResult(WriteResult(basket, outcome)); + }); + + var result = await harness.Service.CreateAsync("customer-id").ConfigureAwait(false); + + AssertOutcome(harness, result, outcome, EventAction.Created); + harness.Repository.Verify(x => x.CreateAsync(It.IsAny(), It.IsAny()), Times.Once); + }); + + [TestCase("mutated")] + [TestCase("noop")] + [TestCase("failed")] + public void Update_AfterSuccessfulRead_UsesPersistenceMutationOutcome(string outcome) => Test.Scoped(async test => + { + var harness = new Harness(); + var basket = CreateBasket(); + harness.Repository.Setup(x => x.GetAsync(basket.Id, It.IsAny())).ReturnsAsync(Result.Ok(basket)); + harness.Repository.Setup(x => x.UpdateAsync(basket, It.IsAny())) + .Returns((Domain.Basket value, CancellationToken _) => + { + harness.InTransaction.Should().BeTrue(); + value.HasChanges.Should().BeTrue(); + return Task.FromResult(WriteResult(value, outcome)); + }); + + var result = await harness.Service.UpdateShippingAddressAsync(basket.Id, + new Address { Street1 = "2 Main St", City = "Sydney", PostCode = "2000", State = "NSW" }).ConfigureAwait(false); + + AssertOutcome(harness, result, outcome, EventAction.Updated); + harness.Repository.Verify(x => x.GetAsync(basket.Id, It.IsAny()), Times.Once); + harness.Repository.Verify(x => x.UpdateAsync(basket, It.IsAny()), Times.Once); + }); + + [TestCase("mutated")] + [TestCase("noop")] + [TestCase("failed")] + public void Checkout_GatesMutationEventButPreservesReservationCommand(string outcome) => Test.Scoped(async test => + { + var harness = new Harness(); + var basket = CreateBasket(); + harness.Repository.Setup(x => x.GetAsync(basket.Id, It.IsAny())).ReturnsAsync(Result.Ok(basket)); + harness.Repository.Setup(x => x.UpdateAsync(basket, It.IsAny())) + .Returns((Domain.Basket value, CancellationToken _) => + { + harness.InTransaction.Should().BeTrue(); + return Task.FromResult(WriteResult(value, outcome)); + }); + harness.Products.Setup(x => x.ReserveInventoryAsync(basket, It.IsAny())).ReturnsAsync(Result.Success); + harness.Products.Setup(x => x.CreateConfirmReservationCommand(basket, It.IsAny())) + .Returns(() => + { + harness.InTransaction.Should().BeTrue(); + return Task.FromResult(Result.Success); + }); + + var result = await harness.Service.CheckoutAsync(basket.Id).ConfigureAwait(false); + + AssertOutcome(harness, result, outcome, EventAction.CheckedOut); + harness.Products.Verify(x => x.CreateConfirmReservationCommand(basket, It.IsAny()), outcome == "failed" ? Times.Never : Times.Once); + }); + + [Test] + public void Update_UnchangedAggregate_SkipsPersistenceAndEvent() => Test.Scoped(async test => + { + var harness = new Harness(); + var basket = CreateBasket(); + harness.Repository.Setup(x => x.GetAsync(basket.Id, It.IsAny())).ReturnsAsync(Result.Ok(basket)); + + var result = await harness.Service.UpdateShippingAddressAsync(basket.Id, + new Address { Street1 = "1 Main St", City = "Sydney", PostCode = "2000", State = "NSW" }).ConfigureAwait(false); + + AssertOutcome(harness, result, "noop", EventAction.Updated); + harness.Repository.Verify(x => x.UpdateAsync(It.IsAny(), It.IsAny()), Times.Never); + }); + + private static Result> WriteResult(Domain.Basket basket, string outcome) + => outcome == "failed" ? _writeFailure : Result.Ok(new DataResult(basket, outcome == "mutated")); + + private static void AssertOutcome(Harness harness, Result result, string outcome, EventAction action) + { + result.IsSuccess.Should().Be(outcome != "failed"); + if (outcome == "failed") + result.Error.Should().BeSameAs(_writeFailure.Error); + else + result.Value.CustomerId.Should().Be("customer-id"); + + harness.Events.Should().HaveCount(outcome == "mutated" ? 1 : 0); + if (outcome == "mutated") + harness.Events[0].Action.Should().Be(action.ToString()); + + harness.InTransaction.Should().BeFalse(); + } + + private static Domain.Basket CreateBasket() => Domain.Basket.CreateFrom("basket-id", "customer-id", BasketStatus.Active, null, + new Domain.ValueObjects.Address { Street1 = "1 Main St", City = "Sydney", PostCode = "2000", State = "NSW" }, + [Domain.BasketItem.CreateFrom("item-id", "product-id", "sku", "Product", new Domain.ValueObjects.ItemPricing { UnitOfMeasure = "EA", Quantity = 1, UnitPrice = 10 }, null)], null, null); + + private sealed class Harness + { + public Mock Repository { get; } = new(MockBehavior.Strict); + public Mock Products { get; } = new(MockBehavior.Strict); + public Mock Customers { get; } = new(MockBehavior.Strict); + public List Events { get; } = []; + public bool InTransaction { get; private set; } + public BasketService Service { get; } + + public Harness() + { + // Exercise only the mutation-publication boundary; persistence/transactions remain covered by API tests. + var unitOfWork = new Mock(MockBehavior.Strict); + var queue = new Mock(MockBehavior.Strict); + queue.Setup(x => x.Add(It.IsAny>())).Callback>(events => + { + InTransaction.Should().BeTrue(); + Events.AddRange(events); + }); + unitOfWork.SetupGet(x => x.Events).Returns(queue.Object); + unitOfWork.Setup(x => x.TransactionAsync(It.IsAny>>>(), It.IsAny())) + .Returns(async (Func>> work, CancellationToken cancellationToken) => + { + InTransaction = true; + try + { + return await work(cancellationToken).ConfigureAwait(false); + } + finally + { + InTransaction = false; + } + }); + Service = new BasketService(unitOfWork.Object, Repository.Object, Products.Object, Customers.Object, NullLogger.Instance); + } + } +} diff --git a/src/CoreEx.Data/AGENTS.md b/src/CoreEx.Data/AGENTS.md index 9e76fcf7c..04057893c 100644 --- a/src/CoreEx.Data/AGENTS.md +++ b/src/CoreEx.Data/AGENTS.md @@ -16,12 +16,9 @@ public class OrderService(IUnitOfWork uow, IOrderRepository repo) return await uow.TransactionAsync(async ct => { - var created = await repo.CreateAsync(order, ct).ConfigureAwait(false); + var dr = await repo.CreateAsync(order, ct).ConfigureAwait(false); - // enqueue event — published atomically with the DB commit via the outbox - uow.Events.Add(EventData.CreateEventWith(created, EventAction.Created)); - - return created; + return dr.WhereMutated(created => uow.Events.Add(EventData.CreateEventWith(created, EventAction.Created))); }, ct).ConfigureAwait(false); } } @@ -51,7 +48,7 @@ public async Task> GetAllAsync(QueryArgs args, PagingArgs? pa ## DataResult -Return `DataResult` (no value) or `DataResult` from mutation operations to distinguish "mutated" from "not found" without throwing. +Return `DataResult` for Create/Update and `DataResult` for Delete; ROP wraps these in `Result<...>`, including when mapping persistence models to Domain aggregates. Use `dr.Map(value => SomeMapper.Map(value))` to transform the value and preserve `WasMutated` instead of relying on implicit conversion to the value. `Map` invokes the delegate once regardless of the flag, passes null values through, rejects a null delegate, and propagates mapper exceptions unchanged. Prefer lambdas for CoreEx mappers so conditional nullability is retained. The Application layer gates mutation-announcement events with `WhereMutated` in the write transaction before unwrapping its public response. Successful no-ops still succeed but emit no mutation event, even if an earlier read found the target or the aggregate had changes. Commands/attempt messages have separate semantics. The flag reports the provider's mutation outcome, potentially before deferred writes/commit; it does not guarantee committed rows or exactly-once delivery. ```csharp public async Task DeleteAsync(Guid id, CancellationToken ct = default) diff --git a/src/CoreEx.Data/DataResultT.cs b/src/CoreEx.Data/DataResultT.cs index 5d00baf6d..f600f3985 100644 --- a/src/CoreEx.Data/DataResultT.cs +++ b/src/CoreEx.Data/DataResultT.cs @@ -31,6 +31,15 @@ public DataResult(T value, bool wasMutated = true) /// public bool WasMutated { get; } + /// + /// Maps the while preserving . + /// + /// The mapped value type. + /// The function to map the value, invoked regardless of . + /// The mapped value with the original mutation flag. + /// The is . + public DataResult Map(Func mapper) => new(mapper.ThrowIfNull()(Value), WasMutated); + /// /// Invokes the specified where the result is returning the . /// diff --git a/src/CoreEx.Data/README.md b/src/CoreEx.Data/README.md index d7bedc912..21fa8b1bb 100644 --- a/src/CoreEx.Data/README.md +++ b/src/CoreEx.Data/README.md @@ -22,6 +22,18 @@ - 🔗 **IQueryable integration**: `QueryExtensions.Where` and `OrderBy` applies the filter and order-by to any `IQueryable`. - 🧩 **Multi-set query contract**: `IMultiSetArgsCore` is the minimal, provider-agnostic base for a single result-set within a multi-set query (`MinimumRows`, `MaximumRows`, `StopOnNull`, `InvokeResult()`) - `CoreEx.Database.Extended.IMultiSetArgs` (positional/ordered relational result sets) and `CoreEx.Cosmos.Extended.IMultiSetArgs` (discriminator-keyed Cosmos DB document sets) each extend it with their own provider-specific matching mechanism, so a future NoSQL provider (e.g. Mongo) can reuse the same base contract and row-count semantics. +## Preserving Mutation Outcomes + +Create/Update return `DataResult` (or `Result>`); Delete returns `DataResult` (or `Result`). Use `DataResult.Map` to transform the value while preserving `WasMutated`, including successful no-ops: + +```csharp +.ThenAs(dr => dr.Map(value => BasketMapper.Map(value))) +``` + +`Map` invokes the delegate exactly once regardless of `WasMutated`, passes null source/destination values through, rejects a null delegate, and propagates mapper exceptions unchanged. Prefer the lambda over a CoreEx mapper method group: nullable-return method groups can infer a nullable destination despite `NotNullIfNotNull`, whereas the lambda retains conditional flow analysis. + +Only unwrap after the Application layer gates mutation-announcement events with `WhereMutated`, inside the write transaction. Success with `WasMutated == false` is a successful no-op, not an event to announce; an earlier existence read or aggregate state check cannot substitute for this signal. For aggregates, the flag must represent the whole parent/child persistence operation. Deferred writes may commit later, so the flag does not prove committed rows or exactly-once delivery. Commands and intentional attempt messages follow their own semantics. + ## Key types | Type | Description | @@ -57,4 +69,4 @@ ## AI Usage Guide -An [`AGENTS.md`](./AGENTS.md) file is included with this package. AI coding assistants (GitHub Copilot, Claude, Cursor, etc.) that support workspace-injected package documentation will automatically surface concise usage guidance, code examples, and `Do Not` rules for this package without requiring a local CoreEx checkout. \ No newline at end of file +An [`AGENTS.md`](./AGENTS.md) file is included with this package. AI coding assistants (GitHub Copilot, Claude, Cursor, etc.) that support workspace-injected package documentation will automatically surface concise usage guidance, code examples, and `Do Not` rules for this package without requiring a local CoreEx checkout. diff --git a/src/CoreEx.DomainDriven/AGENTS.md b/src/CoreEx.DomainDriven/AGENTS.md index b4508dd7e..cc8f3c6d6 100644 --- a/src/CoreEx.DomainDriven/AGENTS.md +++ b/src/CoreEx.DomainDriven/AGENTS.md @@ -10,6 +10,10 @@ Provides DDD building blocks: aggregate roots (with integration events), typed e ## Aggregate Root +Domain business invariants (aggregate, child entity, and value object) use `BusinessException` / `Result.BusinessError`, not request-validation errors. Keep meaningful domain messages/codes without request DTO dependencies or JSON paths. Application validators/policies own input validation and may selectively map a known domain failure to a request field; never blanket-convert business errors. Both normally map to HTTP 400, but semantics and diagnostics differ. Domain enforcement is still needed after request validation. + +Exception versus `Result` is independent: constructors/setters may throw business exceptions; result-returning methods generally return business failures. `OnCheckCanMutate` rejection throws through `Modify`/`Remove` internally. Keep argument/invalid-operation exceptions for programming misuse; general numeric/ref-data guard helpers throw those types, not business errors, so use explicit checks for expected business invariant rejections without changing shared helper behavior. + `Aggregate` is an `Entity` with an `Events` collection for accumulating integration events. The aggregate owns its invariants; the application service is responsible for forwarding any accumulated events to `IUnitOfWork.Events` after a successful repository operation. ```csharp @@ -39,10 +43,10 @@ In the application service, events are added to `IUnitOfWork.Events` after the r return await _unitOfWork.TransactionAsync(async () => { var ur = await _repository.UpdateAsync(basket).ConfigureAwait(false); - return ur.ThenAs(b => + return ur.ThenAs(dr => { - var contract = BasketMapper.Map(b); - _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.CheckedOut)); + var contract = BasketMapper.Map(dr.Value); + dr.WhereMutated(_ => _unitOfWork.Events.Add(EventData.CreateEventWith(contract, EventAction.CheckedOut))); return contract; }); }).ConfigureAwait(false); @@ -50,6 +54,8 @@ return await _unitOfWork.TransactionAsync(async () => > `Aggregate` also exposes `AddEvent` / `ClearEvents` for cases where the aggregate itself needs to accumulate events internally, but this is not the primary pattern used in the sample domains. +Domain repositories retain persistence metadata: Create/Update return `Result>` (or non-ROP `DataResult`), Delete returns `Result`/`DataResult`, and reads keep their existing shapes. Map persistence values with the original `WasMutated` flag; only unwrap after Application gates mutation-announcement events. A successful no-op emits no mutation event even when the aggregate had changes or an earlier read found it. The flag covers the aggregate operation but is not a commit/exactly-once guarantee; commands have separate semantics. + ## Entity Base `Entity` provides identity-based equality, `PersistenceState` tracking, and mutation guards. All state changes go through `Modify()` or `Remove()` so the framework can track whether an entity is new, modified, or deleted. diff --git a/src/CoreEx.DomainDriven/README.md b/src/CoreEx.DomainDriven/README.md index 0f145e248..6fe0fa297 100644 --- a/src/CoreEx.DomainDriven/README.md +++ b/src/CoreEx.DomainDriven/README.md @@ -31,6 +31,12 @@ The package intentionally stays minimal: it does not dictate a persistence strat | **[`PersistenceState`](./PersistenceState.cs)** | Enum: `Unknown`, `New`, `NotModified`, `Modified`, `Removed`; governs the entity lifecycle from creation through persistence to deletion. | | **[`DomainDrivenExtensions`](./DomainDrivenExtensions.cs)** | Extension methods on `PersistenceState`: `IsNew`, `IsNotModified`, `IsModified`, `IsRemoved`, `IsNotRemoved`, `IsNewOrModified`. | +## Business Invariants versus Request Validation + +Aggregates, entities, and value objects enforce business invariants using `BusinessException` / `Result.BusinessError`, with domain messages/codes rather than request-field paths. Request/input validity belongs to Application validators/policies (`ValidationException` / `Result.ValidationError`). Both normally map to HTTP 400, but their classification and diagnostics differ. Never reference request DTOs or infer request JSON paths inside Domain; Application may selectively translate a known domain error where the request-field mapping is unambiguous. + +Keep domain enforcement even after request validation. Constructors/`init` setters may throw business exceptions; result-returning operations generally return business failures. `OnCheckCanMutate` returns a result but `Modify`/`Remove` call `ThrowOnError` internally, so guard rejection throws. Programming misuse/broken assumptions retain argument/invalid-operation exceptions. General numeric/reference-data guard helpers are not business-error factories; use explicit domain checks for expected invariant rejections without changing shared helper semantics. See the [Shopping domain walkthrough](../../samples/docs/domain-layer.md#business-invariants-versus-request-validation). + ## Domain Events — Intentionally Not Supported CoreEx deliberately does not provide a native domain-event mechanism (e.g. MediatR `INotification` dispatch or an in-process event bus). This is a conscious architectural decision: @@ -45,8 +51,8 @@ A developer can extend CoreEx with a domain-event mechanism if a genuine use cas - **[`CoreEx`](../CoreEx/README.md)** - Provides `IIdentifier`, `IChangeLog`, `IETag`, `CompositeKey`, `EventData`, and `Result` consumed by the DDD types. - **[`CoreEx.EntityFrameworkCore`](../CoreEx.EntityFrameworkCore/README.md)** - Persists `Entity` and `Aggregate` via `EfDbModel`; uses `PersistenceState` to determine insert/update/delete operations. -- **[`CoreEx.Validation`](../CoreEx.Validation/README.md)** - Validates entity state; `OnCheckCanMutate` can delegate to a CoreEx validator for pre-mutation rule checks. +- **[`CoreEx.Validation`](../CoreEx.Validation/README.md)** - Validates request/input contracts in Application; Domain protects its own business invariants without request-field context. ## AI Usage Guide -An [`AGENTS.md`](./AGENTS.md) file is included with this package. AI coding assistants (GitHub Copilot, Claude, Cursor, etc.) that support workspace-injected package documentation will automatically surface concise usage guidance, code examples, and `Do Not` rules for this package without requiring a local CoreEx checkout. \ No newline at end of file +An [`AGENTS.md`](./AGENTS.md) file is included with this package. AI coding assistants (GitHub Copilot, Claude, Cursor, etc.) that support workspace-injected package documentation will automatically surface concise usage guidance, code examples, and `Do Not` rules for this package without requiring a local CoreEx checkout. diff --git a/src/CoreEx.Validation/AGENTS.md b/src/CoreEx.Validation/AGENTS.md index 0d86e3af0..6bf9e81e5 100644 --- a/src/CoreEx.Validation/AGENTS.md +++ b/src/CoreEx.Validation/AGENTS.md @@ -87,6 +87,8 @@ Property(p => p.EndsOn).Mandatory().CompareProperty(CompareOperator.GreaterThanO ## Reusable Validators +Dictionary JSON error paths use the effective serializer's `DictionaryKeyPolicy` for string keys: `HomeAddress` becomes `homeAddress` with CoreEx defaults. A null key policy preserves keys. This applies to key and value errors and matches `ValidationException.Create`; CLR paths and `GetDictionaryKey()` retain original keys. Dictionary object members use `.key`, arrays/lists use `[index]`. No escaping is added for punctuation within names. + Use `CommonValidator` for logic reused across multiple entity validators. ```csharp diff --git a/src/CoreEx.Validation/Abstractions/InlineValidator.cs b/src/CoreEx.Validation/Abstractions/InlineValidator.cs index a605e035b..60240ebfd 100644 --- a/src/CoreEx.Validation/Abstractions/InlineValidator.cs +++ b/src/CoreEx.Validation/Abstractions/InlineValidator.cs @@ -25,6 +25,11 @@ internal InlineValidator(Action? configure) /// This will apply to all instances in which the is used; therefore, caution is required when using. This is intended for advanced usage only. protected string? OverrideName { get; set; } + /// + /// Gets or sets the JSON name override independently of the CLR name. + /// + protected string? OverrideJsonName { get; set; } + /// /// Gets or sets the property text override (where not ). /// @@ -40,7 +45,7 @@ internal InlineValidator(Action? configure) internal async Task ValidateAsync(PropertyContext context, CancellationToken cancellationToken) where TEntity : class { var root = new RootPropertyRule, TValue>( - new PropertyRuntimeMetadata, TValue>(OverrideName ?? context.Name, _ => context.Metadata.GetValue(context.Entity), text: () => OverrideText ?? context.Metadata.Text, jsonName: OverrideName ?? context.Metadata.JsonName), + new PropertyRuntimeMetadata, TValue>(OverrideName ?? context.Name, _ => context.Metadata.GetValue(context.Entity), text: () => OverrideText ?? context.Metadata.Text, jsonName: OverrideJsonName ?? OverrideName ?? context.Metadata.JsonName), context.IsValueNullable ? _ => context.GetNullableValueOrDefault() : null, context.IsValueNullable ? _ => context.IsNullableValueDefault() : null); ValidationExtensions.Chain(root, _validator); @@ -79,4 +84,4 @@ internal Validator() { } /// protected override Task OnValidateAsync(PropertyContext, TValue> context, CancellationToken cancellationToken) => Task.CompletedTask; } -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/GlobalUsing.cs b/src/CoreEx.Validation/GlobalUsing.cs index 9826293a3..2e8e9b1b4 100644 --- a/src/CoreEx.Validation/GlobalUsing.cs +++ b/src/CoreEx.Validation/GlobalUsing.cs @@ -2,6 +2,7 @@ global using CoreEx.Data; global using CoreEx.Entities; global using CoreEx.Entities.Abstractions; +global using CoreEx.Json; global using CoreEx.Localization; global using CoreEx.Metadata; global using CoreEx.RefData; @@ -20,4 +21,4 @@ global using System.Runtime.CompilerServices; global using System.Text.Json; global using System.Text.Json.Serialization; -global using System.Text.RegularExpressions; \ No newline at end of file +global using System.Text.RegularExpressions; diff --git a/src/CoreEx.Validation/README.md b/src/CoreEx.Validation/README.md index f5a141627..ed61117d8 100644 --- a/src/CoreEx.Validation/README.md +++ b/src/CoreEx.Validation/README.md @@ -85,6 +85,12 @@ Rules apply the actual validation logic to a property value. Each rule class is | **[`StringRule`](./Rules/StringRule.cs)** | Validates string length (min/max), regex pattern matching, or an exact length. | `String(maxLength)`, `String(min, max, regex)`, `Matches(regex)`, `Length(exact)`, `MinimumLength()`, `MaximumLength()` | | **[`WildcardRule`](./Rules/WildcardRule.cs)** | Validates that a `string` conforms to a CoreEx `Wildcard` pattern. | `Wildcard()`, `Wildcard(wildcard)` | +## Dictionary Error Paths + +JSON error paths follow the effective `JsonSerializerOptions.DictionaryKeyPolicy` for string keys, for both key and value errors. With CoreEx defaults, key `HomeAddress` produces `addresses.homeAddress.street`; a null key policy preserves `HomeAddress`. Property segments independently use their metadata/`PropertyNamingPolicy`. Explicit validation options take precedence over `JsonDefaults.SerializerOptions`. Non-string keys keep their existing string representation. + +CLR paths (`UseJsonNames = false`) and `GetDictionaryKey()` retain the original key. The expression-based `ValidationException.Create` factories follow the same JSON naming rules. Dictionary entries use `.key` and collection elements use `[index]`; these are validation error paths, not full JSONPath expressions, and no escaping is added. + ## Namespaces | Namespace | Description | Documentation | diff --git a/src/CoreEx.Validation/Rules/DictionaryRule.cs b/src/CoreEx.Validation/Rules/DictionaryRule.cs index 4a0b813db..d95d0a935 100644 --- a/src/CoreEx.Validation/Rules/DictionaryRule.cs +++ b/src/CoreEx.Validation/Rules/DictionaryRule.cs @@ -164,7 +164,7 @@ internal async Task ValidateAsync(PropertyContext context, i // Where the key is a string, then set the name on the validator to support better error messages. var kv = _getKeyValidator.Invoke(args); if (kvp.Key is string s && !string.IsNullOrEmpty(s) && kv is ValidatingInlineValidator vilv) - vilv.WithName(s); + vilv.WithName(s, (args.JsonSerializerOptions ?? JsonDefaults.SerializerOptions).DictionaryKeyPolicy?.ConvertName(s) ?? s); // Validate the key and merge the result. var r = await kv.ValidateAsync(kvp.Key, args, cancellationToken).ConfigureAwait(false); @@ -179,7 +179,7 @@ internal async Task ValidateAsync(PropertyContext context, i hasNullValue = true; else if (_getValueValidator is not null) { - var args = CreateValidationArgs(context, kvp.Key?.ToString()); + var args = CreateValidationArgs(context, kvp.Key); var last = context.GetDictionaryKeySafe(); context.SetDictionaryKey(kvp.Key); @@ -223,14 +223,16 @@ internal async Task ValidateAsync(PropertyContext context, i /// The . /// The dictionary key. /// The . - private static ValidationArgs CreateValidationArgs(PropertyContext context, string? key) + private static ValidationArgs CreateValidationArgs(PropertyContext context, TKey? key) { var args = context.CreateValidationArgs(); - args.FullyQualifiedEntityName += $"[{(key ?? "null")}]"; + var name = key?.ToString() ?? "null"; + args.FullyQualifiedEntityName += $"[{name}]"; // Note: an indexer for a dictionary from a JSON perspective is simply a property name; i.e. no square brackets. - args.FullyQualifiedJsonEntityName += $".{key ?? "null"}"; + var jsonName = key is string s ? (args.JsonSerializerOptions ?? JsonDefaults.SerializerOptions).DictionaryKeyPolicy?.ConvertName(s) ?? s : name; + args.FullyQualifiedJsonEntityName += $".{jsonName}"; return args; } } -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/ValidatingInlineValidator.cs b/src/CoreEx.Validation/ValidatingInlineValidator.cs index d421eecbf..0258db1b3 100644 --- a/src/CoreEx.Validation/ValidatingInlineValidator.cs +++ b/src/CoreEx.Validation/ValidatingInlineValidator.cs @@ -11,11 +11,13 @@ internal sealed class ValidatingInlineValidator(Action /// The property name. + /// The JSON name where different from the property name. /// The to support fluent-style method-chaining. /// This will apply to all instances in which the is used; therefore, caution is required when using. This is intended for advanced usage only. - public ValidatingInlineValidator WithName(string name) + public ValidatingInlineValidator WithName(string name, string? jsonName = null) { OverrideName = name.ThrowIfNullOrEmpty(); + OverrideJsonName = jsonName; return this; } @@ -126,4 +128,4 @@ internal readonly struct ValueValidationContext(IValidationContext public Result ToResult() => HasErrors ? Result.ValidationError(Messages!) : Result.Success; } -} \ No newline at end of file +} diff --git a/src/CoreEx/AGENTS.md b/src/CoreEx/AGENTS.md index 74b61329d..64b1fa39a 100644 --- a/src/CoreEx/AGENTS.md +++ b/src/CoreEx/AGENTS.md @@ -4,11 +4,13 @@ ## Semantic Exceptions +When collecting multiple field errors, use `MessageItem.CreateErrorMessage(x => x.Property, text)` and wrap the collection once with `Result.ValidationError(messages)` or `new ValidationException(messages)`. Text may be omitted/null for the inferred invalid message; `(expression, options, text)` supports custom naming. Exception factories delegate to this message factory. For reusable policies without a known request type, retain explicit caller-agreed JSON paths rather than inventing a contract dependency. + Throw the most specific exception; the framework maps each to its HTTP status automatically. ```csharp // 400 — field-level validation errors -throw new ValidationException(new MessageItemCollection { new("sku", "Sku is required.") }); +throw ValidationException.Create(x => x.Sku, "Sku is required."); // 404 — resource not found throw new NotFoundException(); @@ -23,6 +25,12 @@ throw new ConcurrencyException(); throw new DuplicateException(); ``` +For manual property errors, prefer `ValidationException.Create(x => x.Property, text)` over hard-coded names. Nested paths such as `x => x.Address!.Street` resolve every segment through runtime metadata to JSON names (e.g. `address.street`), respecting `[JsonPropertyName]` and the default serializer naming policy. Use `(expression, options, format, ...values)` for explicit serializer options. Entity access is never executed, so `!` is safe for nullable intermediate properties. Array/list indexes and string dictionary keys support literals and captured locals: `x => x.Addresses[0].Street` yields `addresses[0].street`; `x => x.AddressesByType["home"].Street` yields `addressesByType.home.street`. String keys follow the effective `DictionaryKeyPolicy` (null preserves keys), matching validators and serialized JSON. No escaping is added; punctuation can be ambiguous because these are error paths, not full JSONPath expressions. Entity fields, arbitrary getters/calls for indexes/keys, index calculations, and intermediate casts are rejected. Formatting matches the existing string-property factory; no label/value arguments are prepended. Within a validator, continue using `context.AddError(...)` to preserve contextual paths, naming overrides, and validator formatting. + +For lookup failures, classify by identifier role: missing target/explicit child → `NotFoundException` / `Result.NotFoundError` (404); missing body reference → field-level validation (400), using an expression over the request contract where known. Query filters normally return empty results. Translate only expected entity-not-found outcomes, not route/configuration failures; preserve unrelated failures and deliberate delete/security semantics. Business errors describe process/state rules, with more specific conflict/concurrency or authorization semantics taking precedence; error-code needs never determine classification. See [application-layer guidance](../../samples/docs/application-layer.md#missing-targets-versus-invalid-references). + +Request/input validity uses validation errors; aggregate/entity/value-object business invariants use `BusinessException` / `Result.BusinessError` with domain messages/codes and no request DTO dependencies or JSON paths. Both normally map to 400 but are different semantic contracts. Keep domain guards after request validation; Application may selectively map known domain failures to fields, never blanket-convert them. Programming misuse retains argument/invalid-operation errors; shared guard helpers keep their semantics. See [domain guidance](../../samples/docs/domain-layer.md#business-invariants-versus-request-validation). + ## ExecutionContext `ExecutionContext` is `AsyncLocal`-scoped and carries user identity, tenant ID, timestamp, and operation type for the lifetime of a request. Access it via DI injection, or use the static accessors when outside the DI graph. @@ -38,6 +46,10 @@ var ctx = ExecutionContext.TryGetCurrent(); // returns null if none set ## Result / Result\ +Expression-based `ValidationException.Create` and `Result.ValidationError`/`Result.ValidationError` accept an omitted/null message, defaulting to `Validation.InvalidFormat` with the leaf property's friendly metadata `Text` (including localization/display annotations). `RuntimeMetadata.GetJsonPathAndText` resolves the path and label together. A terminal index/key uses its containing property's label. Explicit messages and existing string-property factories are unchanged. + +For manual property validation failures, use `Result.ValidationError(x => x.Name, "Name is invalid.")` or `Result.ValidationError(x => x.Name, "Name is invalid.")`. The shorthand accepts an expression, an `LText` message (including preformatted arguments), and an optional exception configuration action. It uses the default JSON naming, nesting, and indexing from `ValidationException.Create`. For explicit serializer options or separate formatting arguments, use that exception factory directly; exceptions implicitly convert to `Result` and `Result`. + Use `Result` for expected business-error paths instead of throwing exceptions. Compose pipelines with `Then`, `ThenAs`, `GoAsync`, `ThenAsAsync`. ```csharp @@ -134,6 +146,8 @@ See [usage, inline examples, and semantics](./README.md#validating-http-response ## Entity Contracts +Cached runtime metadata lookups prefer the type's static `IRuntimeMetadata` provider and use reflection only for types without that interface. This applies to root/nested `GetJsonPath` and `GetJsonPathAndText` resolution, so custom/generated JSON names, friendly labels, and excluded properties are respected consistently regardless of which cache entry point runs first. Entity instances are never constructed for metadata discovery. + Use the standard interfaces on your contracts so the framework's ETag, paging, and change-log handling works automatically. For entities authored in a project that references `CoreEx.CodeGen`, prefer the Roslyn source generator — declare a `partial` class with `[Contract]` and the generator emits the boilerplate (constructors, `CopyFrom`, `Clone`, equality, `CleanUp`, etc.). diff --git a/src/CoreEx/Entities/MessageItem.Create.cs b/src/CoreEx/Entities/MessageItem.Create.cs index cfd030883..1e694b6b9 100644 --- a/src/CoreEx/Entities/MessageItem.Create.cs +++ b/src/CoreEx/Entities/MessageItem.Create.cs @@ -2,6 +2,30 @@ namespace CoreEx.Entities; public partial record class MessageItem { + /// + /// Creates an error message using a property's JSON path and optional message text. + /// + /// The root entity type. + /// The direct, nested, or indexed property expression. + /// The message; null uses the standard invalid format and friendly metadata label. + /// The error message. + public static MessageItem CreateErrorMessage(Expression> propertyExpression, LText? text = null) where TEntity : class + => CreateErrorMessage(propertyExpression, null, text); + + /// + /// Creates an error message using a property's JSON path and explicit serializer naming options. + /// + /// The root entity type. + /// The direct, nested, or indexed property expression. + /// The naming options, or null to use serializer defaults. + /// The message; null uses the standard invalid format and friendly metadata label. + /// The error message. + public static MessageItem CreateErrorMessage(Expression> propertyExpression, JsonSerializerOptions? options, LText? text) where TEntity : class + { + var metadata = RuntimeMetadata.GetJsonPathAndText(propertyExpression, options); + return text.HasValue ? CreateErrorMessage(metadata.Path, text.Value) : CreateErrorMessage(metadata.Path, Validation.Validation.InvalidFormat, metadata.Text); + } + /// /// Creates a new with a specified and text. /// @@ -92,4 +116,4 @@ public static MessageItem CreateWarningMessage(string? property, LText format, p /// A . public static MessageItem CreateInfoMessage(string? property, LText format, params IEnumerable values) => new() { Property = property, Type = MessageType.Info, Text = format.EnsureNoArgs().WithArgs(values) }; -} \ No newline at end of file +} diff --git a/src/CoreEx/Metadata/README.md b/src/CoreEx/Metadata/README.md index b6693405b..f446a4cfe 100644 --- a/src/CoreEx/Metadata/README.md +++ b/src/CoreEx/Metadata/README.md @@ -27,7 +27,17 @@ Consumers rarely interact with this namespace directly — its API surface is pr | [`IRuntimeMetadata`](./IRuntimeMetadata.cs) | Interface implemented by `IContract` types providing `GetStaticPropertyRuntimeMetadata()` for source-generated fast-path access. | | [`IRuntimeMetadataCore`](./IRuntimeMetadataCore.cs) | Minimal interface subset of `IRuntimeMetadata` used by non-`IContract` infrastructure that still needs property-level introspection. | +## JSON Property Paths + +All `GetCachedProperties` entry points share one source-selection path: on a cache miss, types implementing `IRuntimeMetadata` use their static metadata provider (including explicit implementations); other types fall back to reflected property metadata. For explicitly requested properties, `GetForExpression` and JSON path helpers prefer that metadata and use a separately cached reflection fallback when the property is absent. This supports generated reference-data navigation properties (such as `Channel` for `ChannelCode`), whose JSON-name and label attributes are reflected without adding them to the contract's equality/copy/clean property collection. No entity instance is constructed, instance-only `IRuntimeMetadataCore` providers are not invoked, and static provider failures propagate rather than silently falling back. Metadata filtering does not modify the shared cache. The uncached `GetPropertyRuntimeMetadata(Type, ...)` method remains an explicit reflection-based API. + +`RuntimeMetadata.GetJsonPathAndText(expression, options)` resolves both the JSON path and the last property's friendly `LText` in the same traversal. The label comes from runtime metadata, respecting localization/display annotations and otherwise sentence-casing the CLR property name. A terminal collection index or dictionary key retains the containing property's label. `ValidationException.Create` uses this metadata to infer its default invalid message. + +`RuntimeMetadata.GetJsonPath(x => x.Address!.Street)` returns a dot-separated JSON path such as `address.street`. Each segment uses its owning type's cached runtime metadata and `GetJsonName`: explicit JSON names take precedence over the supplied `JsonSerializerOptions.PropertyNamingPolicy`, or `JsonDefaults.SerializerOptions` when options are omitted. + +The expression is inspected without execution, so nullable intermediate properties can safely use `!`. Supports direct/nested public instance properties, outermost boxing (including nullable value types), one-dimensional array/list indexes (`addresses[0].street`), and string dictionary keys (`addressesByType.home.street`). Indexes/keys must be literals or captured locals; indexes must be non-negative. No collection access, bounds check, or key-existence check occurs. String dictionary keys use the effective serializer's `DictionaryKeyPolicy`, exactly as validators do; a null policy preserves the key. No escaping is added. These are validator error paths, not full JSONPath expressions; punctuation in keys can make paths ambiguous. Fields in entity paths, static members, arbitrary getters/calls used for keys, index calculations, and intermediate casts are rejected with `ArgumentException`. The existing single-property `GetForExpression` behavior is unchanged. `ValidationException.Create(...)` uses this resolver for manual property errors. + ## Related Namespaces - **[`CoreEx.Entities`](../Entities/README.md)** - `IContract` extends `IRuntimeMetadata`; `Cleaner` delegates to `RuntimeMetadata.Clean`; `IdentifierGenerator` and change-log population rely on property metadata. -- **[`CoreEx.Generator`](../../../gen/CoreEx.Generator/README.md)** - The Roslyn source generator emits `GetStaticPropertyRuntimeMetadata()` implementations consumed by `RuntimeMetadata`. \ No newline at end of file +- **[`CoreEx.Generator`](../../../gen/CoreEx.Generator/README.md)** - The Roslyn source generator emits `GetStaticPropertyRuntimeMetadata()` implementations consumed by `RuntimeMetadata`. diff --git a/src/CoreEx/Metadata/RuntimeMetadata.GetJsonPath.cs b/src/CoreEx/Metadata/RuntimeMetadata.GetJsonPath.cs new file mode 100644 index 000000000..286253936 --- /dev/null +++ b/src/CoreEx/Metadata/RuntimeMetadata.GetJsonPath.cs @@ -0,0 +1,116 @@ +namespace CoreEx.Metadata; + +public static partial class RuntimeMetadata +{ + /// + /// Gets the dot-separated JSON path for a direct or nested property expression. + /// + /// The root entity type. + /// A property chain rooted at the entity parameter. + /// The serializer options used for naming; defaults to . + /// The JSON path with each segment resolved using its owning type's runtime metadata. + /// Property and collection access is not executed. Supports outermost boxing, one-dimensional array/list indexes, and string dictionary keys. + /// Indexes and keys must be constants or captured locals. String keys use the effective serializer's dictionary key naming policy. + /// The expression is not a supported property chain. + public static string GetJsonPath(Expression> propertyExpression, JsonSerializerOptions? options = null) where TEntity : class + => GetJsonPathAndText(propertyExpression, options).Path; + + /// + /// Gets the JSON path and friendly text of the last property in an expression. + /// + /// The root entity type. + /// The direct, nested, or indexed property expression. + /// The JSON naming options, or null to use the defaults. + /// The JSON path and localized property text. For a terminal index/key, uses the containing property's text. + /// Uses the same expression restrictions as without executing entity access. + public static (string Path, LText Text) GetJsonPathAndText(Expression> propertyExpression, JsonSerializerOptions? options = null) where TEntity : class + { + Expression? expression = propertyExpression.ThrowIfNull().Body; + if (expression is UnaryExpression { NodeType: ExpressionType.Convert, Type: var type, Method: null } conversion && type == typeof(object)) + expression = conversion.Operand; + + if (expression == propertyExpression.Parameters[0]) + throw InvalidJsonPath(); + + var path = new StringBuilder(); + var text = Validation.Validation.ValueText; + AppendJsonPath(expression, propertyExpression.Parameters[0], options, path, ref text); + return (path.ToString(), text); + } + + /// + /// Appends property names and collection locations without executing entity access. + /// + private static void AppendJsonPath(Expression? expression, ParameterExpression root, JsonSerializerOptions? options, StringBuilder path, ref LText text) + { + if (expression == root) + return; + + if (expression is MemberExpression { Member: PropertyInfo property, Expression: not null } member + && property.GetMethod is { IsPublic: true, IsStatic: false } && property.GetIndexParameters().Length == 0) + { + AppendJsonPath(member.Expression, root, options, path, ref text); + var propertyMetadata = GetForProperty(member.Expression.Type, property); + + if (path.Length > 0) + path.Append('.'); + + path.Append(propertyMetadata.GetJsonName(options)); + text = propertyMetadata.Text; + return; + } + + if (expression is BinaryExpression { NodeType: ExpressionType.ArrayIndex } array && array.Left.Type.GetArrayRank() == 1) + { + AppendJsonPath(array.Left, root, options, path, ref text); + AppendJsonIndex(array.Left.Type, array.Right, options, path); + return; + } + + if (expression is MethodCallExpression { Object: not null, Arguments.Count: 1 } call + && call.Object.Type.GetProperties().Any(p => p.GetMethod == call.Method && p.GetIndexParameters().Length == 1)) + { + AppendJsonPath(call.Object, root, options, path, ref text); + AppendJsonIndex(call.Object.Type, call.Arguments[0], options, path); + return; + } + + throw InvalidJsonPath(); + } + + /// + /// Appends an index or dictionary key using the same notation as validation contexts. + /// + private static void AppendJsonIndex(Type collectionType, Expression argument, JsonSerializerOptions? options, StringBuilder path) + { + var value = argument switch + { + ConstantExpression constant => constant.Value, + MemberExpression { Member: FieldInfo field, Expression: ConstantExpression closure } when field.DeclaringType?.IsDefined(typeof(CompilerGeneratedAttribute), false) == true => field.GetValue(closure.Value), + _ => throw InvalidJsonPath() + }; + + var types = collectionType.GetInterfaces().Append(collectionType); + if (types.Any(t => t.IsGenericType && (t.GetGenericTypeDefinition() == typeof(IDictionary<,>) || t.GetGenericTypeDefinition() == typeof(IReadOnlyDictionary<,>)) + && t.GetGenericArguments()[0] == typeof(string)) && value is string key) + { + path.Append('.').Append((options ?? JsonDefaults.SerializerOptions).DictionaryKeyPolicy?.ConvertName(key) ?? key); + return; + } + + if ((collectionType.IsArray || types.Any(t => t.IsGenericType && (t.GetGenericTypeDefinition() == typeof(IList<>) || t.GetGenericTypeDefinition() == typeof(IReadOnlyList<>)))) + && value is int index && index >= 0) + { + path.Append('[').Append(index).Append(']'); + return; + } + + throw InvalidJsonPath(); + } + + /// + /// Creates the error for unsupported property paths or index arguments. + /// + private static ArgumentException InvalidJsonPath() + => new("Only public instance property paths with array/list indexes or string dictionary keys (constants or captured locals) are supported.", "propertyExpression"); +} diff --git a/src/CoreEx/Metadata/RuntimeMetadata.cs b/src/CoreEx/Metadata/RuntimeMetadata.cs index 14db3fa63..f28a07f4c 100644 --- a/src/CoreEx/Metadata/RuntimeMetadata.cs +++ b/src/CoreEx/Metadata/RuntimeMetadata.cs @@ -15,7 +15,7 @@ public static partial class RuntimeMetadata public static TimeSpan SlidingExpirationTimespan => Internal.GetConfigurationValue("CoreEx:Runtime:Metadata:SlidingExpirationTimespan", TimeSpan.FromMinutes(30)); /// - /// Gets the for the specified using . + /// Gets the for the specified , preferring static metadata with a cached reflection fallback. /// /// The entity . /// The property . @@ -27,14 +27,14 @@ public static IPropertyRuntimeMetadata GetForExpression(Expr throw new ArgumentException("Only member access expressions are supported.", nameof(propertyExpression)); var me = (MemberExpression)propertyExpression.Body; - if (GetCachedProperties().TryGetValue(me.Member.Name, out var m)) - return m; + if (me.Member is PropertyInfo property) + return GetForProperty(typeof(TEntity), property); throw new InvalidOperationException("The underlying property metadata cannot be retrieved for the property expression."); } /// - /// Gets the for the specified using reflection on a cache miss. + /// Gets the for the specified , preferring static metadata with a cached reflection fallback. /// /// The entity . /// The property . @@ -49,49 +49,55 @@ public static IPropertyRuntimeMetadata GetForExpression(Expr throw new ArgumentException("Only member access expressions are supported.", nameof(propertyExpression)); var me = (MemberExpression)propertyExpression.Body; - if (GetCachedProperties().TryGetValue(me.Member.Name, out var m)) - return m; + if (me.Member is PropertyInfo property) + return GetForProperty(typeof(TEntity), property); - throw new InvalidOperationException("The underlying property metadata can not be retrieved for the property expression."); + throw new InvalidOperationException("The underlying property metadata cannot be retrieved for the property expression."); } /// - /// Gets the cached dictionary for the entity using . + /// Resolves explicitly requested properties omitted by static metadata without changing the contract's property collection. /// - /// The . - /// The cached dictionary. - public static IDictionary GetCachedProperties() where T : IContract + private static IPropertyRuntimeMetadata GetForProperty(Type type, PropertyInfo property) { - var cacheKey = $"RuntimeMetadata_{typeof(T).FullName}"; + if (GetCachedProperties(type).TryGetValue(property.Name, out var metadata)) + return metadata; - return Internal.MemoryCache.GetOrCreate>(cacheKey, entry => - { - var dict = new Dictionary(); - foreach (var p in GetPropertyRuntimeMetadata()) - dict[p.Name] = p; + if (property.GetMethod is not { IsPublic: true, IsStatic: false } || property.GetIndexParameters().Length != 0) + throw new InvalidOperationException("The underlying property metadata cannot be retrieved for the property expression."); + return Internal.MemoryCache.GetOrCreate(("RuntimeMetadata_Property", type, property), entry => + { entry.SlidingExpiration = SlidingExpirationTimespan; - return dict; + return CreateReflectedPropertyMetadata(type, property); })!; } /// - /// Gets the cached dictionary for the entity using reflection on a cache miss. + /// Gets the cached dictionary for the entity using . + /// + /// The . + /// The cached dictionary. + public static IDictionary GetCachedProperties() where T : IContract => GetCachedProperties(typeof(T)); + + /// + /// Gets the cached property metadata, preferring static metadata with reflection as the fallback. /// /// The entity . /// The list of properties to ignore. - /// This implementation leverages System.Reflection; as such, consider implementing type as and use where performance is a premium. + /// Metadata source selection and creation occur only on a cache miss. No entity instance is constructed. public static IDictionary GetCachedProperties(params IEnumerable ignoreProperties) => GetCachedProperties(typeof(T), ignoreProperties); /// - /// Gets the cached dictionary for the entity using reflection on a cache miss. + /// Gets the cached property metadata, preferring static metadata with reflection as the fallback. /// /// The . /// The list of properties to ignore. - /// This implementation leverages System.Reflection; as such, consider implementing type as and use where performance is a premium. + /// Metadata source selection and creation occur only on a cache miss. No entity instance is constructed. + /// Static metadata errors propagate; reflection is used only when the type does not implement . public static IDictionary GetCachedProperties(Type type, params IEnumerable ignoreProperties) { - var cacheKey = $"RuntimeMetadata_{type.FullName}"; + var cacheKey = $"RuntimeMetadata_{type.ThrowIfNull().FullName}"; if (type.IsValueType) return new Dictionary(); @@ -99,7 +105,8 @@ public static IDictionary GetCachedProperties( var dict = Internal.MemoryCache.GetOrCreate>(cacheKey, entry => { var dict = new Dictionary(); - foreach (var p in GetPropertyRuntimeMetadata(type)) + var properties = typeof(IRuntimeMetadata).IsAssignableFrom(type) ? GetStaticMetadataForType(type) : GetPropertyRuntimeMetadata(type); + foreach (var p in properties) dict[p.Name] = p; entry.SlidingExpiration = SlidingExpirationTimespan; @@ -109,6 +116,23 @@ public static IDictionary GetCachedProperties( return ignoreProperties.Any() ? dict.Where(kvp => !ignoreProperties.Contains(kvp.Key)).ToDictionary(kvp => kvp.Key, kvp => kvp.Value) : dict; } + /// + /// Dispatches to the static interface metadata provider for a runtime type. + /// + private static IEnumerable GetStaticMetadataForType(Type type) + { + var method = typeof(RuntimeMetadata).GetMethod(nameof(GetStaticMetadata), BindingFlags.NonPublic | BindingFlags.Static) + ?? throw new InvalidOperationException("The static metadata dispatcher cannot be found."); + + var provider = method.MakeGenericMethod(type).CreateDelegate>>(); + return provider(); + } + + /// + /// Invokes the static metadata interface, including explicit implementations. + /// + private static IEnumerable GetStaticMetadata() where T : IRuntimeMetadata => T.GetStaticPropertyRuntimeMetadata(); + /// /// Gets the for each property of the entity using . /// @@ -146,9 +170,17 @@ public static IEnumerable GetPropertyRuntimeMetadata(T if (ignoreProperties.Contains(pi.Name)) continue; - var method = typeof(PropertyRuntimeMetadataReflector).GetMethod(nameof(PropertyRuntimeMetadataReflector.CreatePropertyRuntimeMetadata), BindingFlags.Static | BindingFlags.Public)!; - var genericMethod = method.MakeGenericMethod(type, pi.PropertyType); - yield return (IPropertyRuntimeMetadata)genericMethod.Invoke(null, [pi])!; + yield return CreateReflectedPropertyMetadata(type, pi); } } -} \ No newline at end of file + + /// + /// Creates typed reflected metadata for a single property without invoking its getter. + /// + private static IPropertyRuntimeMetadata CreateReflectedPropertyMetadata(Type type, PropertyInfo property) + { + var method = typeof(PropertyRuntimeMetadataReflector).GetMethod(nameof(PropertyRuntimeMetadataReflector.CreatePropertyRuntimeMetadata), BindingFlags.Static | BindingFlags.Public)!; + var genericMethod = method.MakeGenericMethod(type, property.PropertyType); + return (IPropertyRuntimeMetadata)genericMethod.Invoke(null, [property])!; + } +} diff --git a/src/CoreEx/README.md b/src/CoreEx/README.md index 0fb362f62..238cf11f1 100644 --- a/src/CoreEx/README.md +++ b/src/CoreEx/README.md @@ -143,6 +143,27 @@ Semantic (error-oriented) exception types with automatic HTTP status mapping: All inherit from [`ExtendedException`](./Abstractions/ExtendedExceptionT.cs) implementing [`IExtendedException`](./Abstractions/IExtendedException.cs). +Choose the error by the identifier's role: a missing operation target or explicitly addressed child is `NotFoundException` / `Result.NotFoundError` (404); a missing entity referenced by a request-body field is field-level validation (400 under CoreEx conventions). An unmatched collection filter normally returns an empty result. Services/policies interpret expected lookup outcomes, not every downstream 404 (which might indicate a route/configuration failure), and preserve unrelated failures. Error-code needs do not determine classification; retain deliberate idempotent-delete and security-concealment semantics. See [application-layer guidance](../../samples/docs/application-layer.md#missing-targets-versus-invalid-references). + +For a manually raised property validation error, prefer an expression over a hard-coded property name. + +Use validation errors for request/input validity; aggregate, entity, and value-object business invariants use `BusinessException` / `Result.BusinessError` with domain messages/codes and no request-field paths. Keep both request validation and domain enforcement. Argument/invalid-operation exceptions remain appropriate for programming misuse; shared guard helpers are not business-error factories. See [domain failure conventions](../../samples/docs/domain-layer.md#business-invariants-versus-request-validation). + +To aggregate errors without creating an exception per item, use `MessageItem.CreateErrorMessage(x => x.Name)` or supply an explicit/preformatted `LText`. The overload `(expression, options, text)` accepts custom serializer options; null text infers the standard invalid message. `ValidationException.Create` delegates to this message factory, sharing metadata paths and friendly labels. + +Omit the message, or pass null, to use `Validation.InvalidFormat` with the leaf property's friendly metadata label: `ValidationException.Create(x => x.Name)` produces `Name is invalid.`. Localization/display annotations are respected independently of JSON naming; terminal collection indexes or dictionary keys use the containing property's label. With explicit serializer options, pass null as the message to request inference. Explicit messages remain unchanged, and existing string-property factories retain their behavior. + +```csharp +throw ValidationException.Create(x => x.Name, "Name is already registered."); +throw ValidationException.Create(x => x.Address!.Street, "Street is spooky."); +// JSON paths: "name" and "address.street" with default camel-case naming. +throw ValidationException.Create(x => x.Addresses[0].Street, "Street is spooky."); +throw ValidationException.Create(x => x.AddressesByType["home"].Street, "Street is spooky."); +// Validator-compatible paths: "addresses[0].street" and "addressesByType.home.street". +``` + +Each path segment uses runtime metadata: explicit `[JsonPropertyName]` names take precedence over the naming policy in `JsonDefaults.SerializerOptions`. For custom naming, use `ValidationException.Create(x => x.Address!.Street, options, "Street is spooky.")`. The expression is inspected, never executed; `!` is safe for nullable intermediate properties here. Direct/nested public instance properties, outermost boxing, one-dimensional array/list indexes, and string dictionary keys are supported. Indexes/keys may be literals or captured locals; arbitrary getters, method calls, index calculations, and intermediate casts are rejected. Collections are never accessed, so indexes and keys do not need to exist (indexes must be non-negative). String dictionary keys use the effective serializer's `DictionaryKeyPolicy`, matching validators and serialized JSON (unchanged when that policy is null). No escaping is added. These are error paths, not full `$`-rooted JSONPath expressions; dots/brackets in keys can be ambiguous. The `LText` and formatting behavior matches the string-property factory: no property label or current value is automatically prepended. Unlike a validator context, this factory always uses JSON names and has no contextual name/path overrides. + Additionally, these support `With*`-style methods to add additional context that is added to the resulting `ProblemDetails`: ``` csharp diff --git a/src/CoreEx/Results/README.md b/src/CoreEx/Results/README.md index a0092b4e1..697506d32 100644 --- a/src/CoreEx/Results/README.md +++ b/src/CoreEx/Results/README.md @@ -33,6 +33,20 @@ The core principle is a clear distinction between *expected* errors and *unexpec | [`IToResult`](./Abstractions/IToResult.cs) | Interface for types that can project themselves to a `Result`. | | [`IToResult`](./Abstractions/IToResultT.cs) | Interface for types that can project themselves to a `Result`. | +## Property Validation Failures + +Use expression-based shorthand factories for manual property errors without throwing: + +The message is optional: `Result.ValidationError(x => x.Name)` and `Result.ValidationError(x => x.Name, null)` use `Validation.InvalidFormat` and the leaf property's friendly metadata label. `ValidationException.Create` delegates message construction/defaulting to `MessageItem.CreateErrorMessage`, with path/label inference shared in runtime metadata. + +```csharp +return Result.ValidationError(x => x.Name, "Name is already registered."); +return Result.ValidationError(x => x.Children[0].Name, "Name is invalid."); +return ValidationException.Create(x => x.Contacts["home"].Name, options, "Name {0} is taken.", name); +``` + +The shorthand accepts a property expression, an `LText` message, and an optional `configure` action for the created exception. It delegates to `ValidationException.Create` for default JSON naming, nesting, array/list indexes, and string dictionary keys. `TEntity` identifies the property owner independently of the `Result` value type. Supply a preformatted `LText` for message arguments. For explicit serializer options or composite formatting arguments, fall back to `ValidationException.Create` and its implicit conversion to `Result` or `Result`, as in the last example. Invalid expressions throw rather than being converted into failed results. + ## Related Namespaces - **[`CoreEx`](../README.md)** - Defines the semantic exception types (`ValidationException`, `NotFoundException`, etc.) that are used as the `Error` payload within a failed `Result`. @@ -43,4 +57,4 @@ The core principle is a clear distinction between *expected* errors and *unexpec - [Railway Oriented Programming — Scott Wlaschin](https://fsharpforfunandprofit.com/posts/recipe-part2/) - The foundational article describing the pattern that `CoreEx.Results` implements. - [Monad (functional programming) — Wikipedia](https://en.wikipedia.org/wiki/Monad_(functional_programming)) - Background on the monad pattern underlying `Result` types. -- [Error management — Microsoft F# guide](https://learn.microsoft.com/en-us/dotnet/fsharp/style-guide/conventions#error-management) - Microsoft's guidance on when to use result types versus exceptions. \ No newline at end of file +- [Error management — Microsoft F# guide](https://learn.microsoft.com/en-us/dotnet/fsharp/style-guide/conventions#error-management) - Microsoft's guidance on when to use result types versus exceptions. diff --git a/src/CoreEx/Results/Result.Error.cs b/src/CoreEx/Results/Result.Error.cs index d10377261..69d3720db 100644 --- a/src/CoreEx/Results/Result.Error.cs +++ b/src/CoreEx/Results/Result.Error.cs @@ -90,4 +90,15 @@ public readonly partial struct Result /// The optional configuration action. /// The that has a state of . public static Result ValidationError(MessageItem message, Action? configure = null) => (new ValidationException(message)).Adjust(configure); -} \ No newline at end of file + + /// + /// Creates a validation failure using the JSON path of a property expression. + /// + /// The entity type used to identify the property. + /// The direct, nested, or indexed property expression. + /// The message text; null uses the standard invalid format and the property's friendly metadata text. + /// The optional configuration action. + /// The that has a state of . + public static Result ValidationError(Expression> propertyExpression, LText? text = null, Action? configure = null) where TEntity : class + => ValidationException.Create(propertyExpression, text).Adjust(configure); +} diff --git a/src/CoreEx/Results/ResultT.Error.cs b/src/CoreEx/Results/ResultT.Error.cs index 547f2aad1..8045ef531 100644 --- a/src/CoreEx/Results/ResultT.Error.cs +++ b/src/CoreEx/Results/ResultT.Error.cs @@ -90,4 +90,15 @@ public readonly partial struct Result /// The optional configuration action. /// The that has a state of . public static Result ValidationError(MessageItem message, Action? configure = null) => (new ValidationException(message)).Adjust(configure); -} \ No newline at end of file + + /// + /// Creates a validation failure using the JSON path of a property expression. + /// + /// The entity type used to identify the property, independent of the result value type. + /// The direct, nested, or indexed property expression. + /// The message text; null uses the standard invalid format and the property's friendly metadata text. + /// The optional configuration action. + /// The that has a state of . + public static Result ValidationError(Expression> propertyExpression, LText? text = null, Action? configure = null) where TEntity : class + => ValidationException.Create(propertyExpression, text).Adjust(configure); +} diff --git a/src/CoreEx/Validation/Validation.cs b/src/CoreEx/Validation/Validation.cs index 83a01278f..e0a0c27cc 100644 --- a/src/CoreEx/Validation/Validation.cs +++ b/src/CoreEx/Validation/Validation.cs @@ -11,6 +11,12 @@ public static class Validation /// Defaults to: '{0} is required.'. public static LText MandatoryFormat { get; set; } = new("CoreEx.Validation.MandatoryFormat", "{0} is required."); + /// + /// Gets or sets the format string for the invalid error message. + /// + /// Defaults to: '{0} is invalid'. + public static LText InvalidFormat { get; set; } = new("CoreEx.Validation.InvalidFormat", "{0} is invalid."); + /// /// Gets or sets the format string for an invalid value error message. /// @@ -54,6 +60,20 @@ public static class Validation return (TResult)new TResult().ToFailure(ex); } + /// + /// Creates an invalid . + /// + /// The value name. + /// The friendly text name used in validation messages (defaults to as sentence case where not specified). + /// An optional action to configure the . + /// The resulting . + public static TResult CreateInvalidResult(string? name = null, LText? text = null, Action? configure = null) where TResult : IResult, new() + { + var ex = new ValidationException(MessageItem.CreateErrorMessage(name ?? ValueName, InvalidFormat, text ?? name?.ToSentenceCase() ?? ValueText)); + configure?.Invoke(ex); + return (TResult)new TResult().ToFailure(ex); + } + /// /// Creates an invalid value . /// @@ -68,4 +88,4 @@ public static class Validation configure?.Invoke(ex); return (TResult)new TResult().ToFailure(ex); } -} \ No newline at end of file +} diff --git a/src/CoreEx/ValidationException.cs b/src/CoreEx/ValidationException.cs index 6c344335b..980882df3 100644 --- a/src/CoreEx/ValidationException.cs +++ b/src/CoreEx/ValidationException.cs @@ -28,6 +28,54 @@ public class ValidationException(LText? message, Exception? innerException) /// The . public static ValidationException Create(string? property, LText format, params IEnumerable values) => new(MessageItem.CreateErrorMessage(property, format, values)); + /// + /// Creates a validation exception for a direct or nested property using its JSON path. + /// + /// The root entity type. + /// The property chain, which is inspected without execution. + /// The message text; null uses the standard invalid format and the property's friendly metadata text. + /// The validation exception. + public static ValidationException Create(Expression> propertyExpression, LText? text = null) where TEntity : class + => Create(propertyExpression, null, text); + + /// + /// Creates a validation exception for a direct or nested property using its JSON path and a formatted message. + /// + /// The root entity type. + /// The property chain, which is inspected without execution. + /// The composite format string. + /// The values included in the message; no property label or current value is prepended. + /// The validation exception. + public static ValidationException Create(Expression> propertyExpression, LText? format, params IEnumerable values) where TEntity : class + => Create(propertyExpression, null, format, values); + + /// + /// Creates a validation exception for a direct or nested property using explicit JSON naming options. + /// + /// The root entity type. + /// The property chain, which is inspected without execution. + /// The serializer options used for naming; defaults to . + /// The message text; null uses the standard invalid format and the property's friendly metadata text. + /// The validation exception. + public static ValidationException Create(Expression> propertyExpression, JsonSerializerOptions? options, LText? text) where TEntity : class + => new(MessageItem.CreateErrorMessage(propertyExpression, options, text)); + + /// + /// Creates a validation exception for a direct or nested property using explicit JSON naming options and a formatted message. + /// + /// The root entity type. + /// The property chain, which is inspected without execution. + /// The serializer options used for naming; defaults to . + /// The message text or composite format string. + /// The values included in the message; no property label or current value is prepended. + /// The validation exception. + public static ValidationException Create(Expression> propertyExpression, JsonSerializerOptions? options, LText? format, params IEnumerable values) where TEntity : class + { + var args = values.ToArray(); + return args.Length == 0 ? Create(propertyExpression, options, format) + : Create(RuntimeMetadata.GetJsonPath(propertyExpression, options), format ?? throw new ArgumentNullException(nameof(format)), args); + } + /// /// Initializes a new instance of the class. /// @@ -90,4 +138,4 @@ public override string ToString() return sb.ToString() + Environment.NewLine + base.ToString(); } -} \ No newline at end of file +} diff --git a/tests/CoreEx.Data.Test.Unit/DataResultTests.cs b/tests/CoreEx.Data.Test.Unit/DataResultTests.cs new file mode 100644 index 000000000..a0a8e4efc --- /dev/null +++ b/tests/CoreEx.Data.Test.Unit/DataResultTests.cs @@ -0,0 +1,80 @@ +namespace CoreEx.Data.Test.Unit; + +public class DataResultTests +{ + [TestCase(true)] + [TestCase(false)] + public void Map_TransformsValueOnceAndPreservesMutationFlag(bool wasMutated) + { + var source = new DataResult(42, wasMutated); + var calls = 0; + + DataResult mapped = source.Map(value => + { + calls++; + return value.ToString(); + }); + + mapped.Value.Should().Be("42"); + mapped.WasMutated.Should().Be(wasMutated); + calls.Should().Be(1); + source.Value.Should().Be(42); + source.WasMutated.Should().Be(wasMutated); + } + + [TestCase(true)] + [TestCase(false)] + public void Map_NullDelegate_Throws(bool wasMutated) + { + var source = new DataResult(42, wasMutated); + + Action action = () => source.Map(null!); // Intentionally violate the non-null delegate contract. + + action.Should().Throw().WithParameterName("mapper"); + } + + [TestCase(true)] + [TestCase(false)] + public void Map_MapperFailure_PropagatesUnchanged(bool wasMutated) + { + var source = new DataResult(42, wasMutated); + var error = new InvalidOperationException("Mapping failed."); + + Action action = () => source.Map(_ => throw error); + + action.Should().Throw().Which.Should().BeSameAs(error); + } + + [Test] + public void Map_NullSource_IsPassedToMapper() + { + var source = new DataResult(null, false); + + var mapped = source.Map(value => value is null ? "missing" : "present"); + + mapped.Value.Should().Be("missing"); + mapped.WasMutated.Should().BeFalse(); + } + + [Test] + public void Map_NullDestination_IsPreserved() + { + var source = new DataResult(42, false); + + DataResult mapped = source.Map(_ => null); + + mapped.Value.Should().BeNull(); + mapped.WasMutated.Should().BeFalse(); + } + + [Test] + public void Map_DefaultResult_MapsDefaultValueWithoutMutation() + { + DataResult source = default; + + var mapped = source.Map(value => value + 1); + + mapped.Value.Should().Be(1); + mapped.WasMutated.Should().BeFalse(); + } +} diff --git a/tests/CoreEx.RefData.Test.Unit/GlobalUsing.cs b/tests/CoreEx.RefData.Test.Unit/GlobalUsing.cs new file mode 100644 index 000000000..89ae57595 --- /dev/null +++ b/tests/CoreEx.RefData.Test.Unit/GlobalUsing.cs @@ -0,0 +1,6 @@ +global using CoreEx.Entities; +global using CoreEx.Localization; +global using CoreEx.Metadata; +global using CoreEx.Validation; +global using System.Text.Json; +global using System.Text.Json.Serialization; diff --git a/tests/CoreEx.RefData.Test.Unit/ReferenceDataMetadataValidationTests.cs b/tests/CoreEx.RefData.Test.Unit/ReferenceDataMetadataValidationTests.cs new file mode 100644 index 000000000..2c337e38a --- /dev/null +++ b/tests/CoreEx.RefData.Test.Unit/ReferenceDataMetadataValidationTests.cs @@ -0,0 +1,58 @@ +namespace CoreEx.RefData.Test.Unit; + +public partial class ReferenceDataOrchestratorTests +{ + [Contract] + internal partial class Booking + { + [ReferenceData] + [JsonPropertyName("sales_channel")] + [Localization("SalesChannel", "Sales channel")] + public partial string? ChannelCode { get; set; } + } + + internal class BookingValidator : Validator + { + public BookingValidator() => Property(x => x.Channel).Mandatory().IsValid(); + } + + [Test] + public async Task GeneratedNavigationProperty_CanBeValidatedWithoutChangingContractMetadata() + { + var properties = RuntimeMetadata.GetCachedProperties(); + properties.Keys.Should().ContainSingle().Which.Should().Be(nameof(Booking.ChannelCode)); + + var validator = BookingValidator.Default; + var missing = await validator.ValidateAsync(new Booking()); + missing.Messages.Should().ContainSingle().Which.Property.Should().Be("sales_channel"); + missing.Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Sales channel is required."); + + var invalid = await validator.ValidateAsync(new Booking { ChannelCode = "Z" }); + invalid.Messages.Should().ContainSingle().Which.Property.Should().Be("sales_channel"); + invalid.Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Sales channel is invalid."); + + var valid = await validator.ValidateAsync(new Booking { ChannelCode = "A" }); + valid.HasErrors.Should().BeFalse(); + RuntimeMetadata.GetCachedProperties(typeof(Booking)).Should().BeSameAs(properties); + properties.Keys.Should().ContainSingle().Which.Should().Be(nameof(Booking.ChannelCode)); + } + + [Test] + public void GeneratedNavigationProperty_ExpressionFactoriesUseReflectedJsonNameAndLabel() + { + var metadata = RuntimeMetadata.GetForExpression(x => x.Channel); + var reflected = RuntimeMetadata.GetForExpression(x => x.Channel, null); + reflected.Should().BeSameAs(metadata); + metadata.Name.Should().Be(nameof(Booking.Channel)); + metadata.GetJsonName().Should().Be("sales_channel"); + metadata.Text.ToString().Should().Be("Sales channel"); + + var options = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper }; + RuntimeMetadata.GetJsonPath(x => x.Channel, options).Should().Be("sales_channel"); + var message = MessageItem.CreateErrorMessage(x => x.Channel); + message.Property.Should().Be("sales_channel"); + message.Text.ToString().Should().Be("Sales channel is invalid."); + ValidationException.Create(x => x.Channel).Messages.Should().ContainSingle().Which.Should().Be(message); + RuntimeMetadata.GetCachedProperties().Should().NotContainKey(nameof(Booking.Channel)); + } +} diff --git a/tests/CoreEx.Test.Unit/Entities/MessageItemExpressionTests.cs b/tests/CoreEx.Test.Unit/Entities/MessageItemExpressionTests.cs new file mode 100644 index 000000000..2b8d48fd7 --- /dev/null +++ b/tests/CoreEx.Test.Unit/Entities/MessageItemExpressionTests.cs @@ -0,0 +1,58 @@ +namespace CoreEx.Test.Unit.Entities; + +[TestFixture] +public class MessageItemExpressionTests +{ + public class Request + { + [JsonPropertyName("product_id")] + [Localization("ProductReference", "Product reference")] + public int ProductId { get; set; } + public List Children { get; set; } = []; + public Dictionary References { get; set; } = []; + } + + [Test] + public void DefaultMessage_UsesFriendlyMetadataAndExplicitJsonName() + { + var message = MessageItem.CreateErrorMessage(x => x.ProductId); + message.Property.Should().Be("product_id"); + message.Text.ToString().Should().Be("Product reference is invalid."); + message.Type.Should().Be(MessageType.Error); + MessageItem.CreateErrorMessage(x => x.ProductId, null).Should().Be(message); + ValidationException.Create(x => x.ProductId).Messages.Should().ContainSingle().Which.Should().Be(message); + } + + [Test] + public void ExplicitOptionsAndPreformattedText_ArePreserved() + { + var options = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper, DictionaryKeyPolicy = JsonNamingPolicy.CamelCase }; + var text = new LText("MissingProduct", "Product {0} was not found.", 13); + var message = MessageItem.CreateErrorMessage(x => x.References["HomeAddress"].ProductId, options, text); + message.Property.Should().Be("REFERENCES.homeAddress.product_id"); + message.Text.Should().Be(text); + message.Text.ToString().Should().Be("Product 13 was not found."); + MessageItem.CreateErrorMessage(x => x.ProductId, text).Text.Should().Be(text); + MessageItem.CreateErrorMessage(x => x.Children[0].ProductId, options, null).Text.ToString().Should().Be("Product reference is invalid."); + } + + [Test] + public void Messages_CanBeAggregatedWithoutCreatingExceptionsPerItem() + { + var messages = new MessageItemCollection + { + MessageItem.CreateErrorMessage(x => x.ProductId), + MessageItem.CreateErrorMessage(x => x.Children[0].ProductId, "Child product is missing.") + }; + var exception = new ValidationException(messages); + exception.Messages.Should().HaveCount(2); + exception.Messages?.Select(x => x.Property).Should().BeEquivalentTo("product_id", "children[0].product_id"); + } + + [Test] + public void InvalidExpression_IsRejected() + { + var action = () => MessageItem.CreateErrorMessage(x => x.ProductId.ToString()); + action.Should().Throw().WithParameterName("propertyExpression"); + } +} diff --git a/tests/CoreEx.Test.Unit/GlobalUsing.cs b/tests/CoreEx.Test.Unit/GlobalUsing.cs index fbee4f0db..7ee9d7329 100644 --- a/tests/CoreEx.Test.Unit/GlobalUsing.cs +++ b/tests/CoreEx.Test.Unit/GlobalUsing.cs @@ -1,9 +1,14 @@ global using CoreEx.Data; +global using CoreEx.Entities; global using CoreEx.Http; +global using CoreEx.Localization; +global using CoreEx.Metadata; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Hosting; +global using System.Linq.Expressions; global using System.Net; global using System.Net.Http.Headers; global using System.Text; global using System.Text.Json; +global using System.Text.Json.Serialization; diff --git a/tests/CoreEx.Test.Unit/Results/ValidationErrorExpressionTests.cs b/tests/CoreEx.Test.Unit/Results/ValidationErrorExpressionTests.cs new file mode 100644 index 000000000..9f8e1a3b2 --- /dev/null +++ b/tests/CoreEx.Test.Unit/Results/ValidationErrorExpressionTests.cs @@ -0,0 +1,94 @@ +namespace CoreEx.Test.Unit.Results; + +[TestFixture] +public class ValidationErrorExpressionTests +{ + public class Person + { + public string? Name { get; set; } + public int Age { get; set; } + public List Children { get; set; } = []; + public Dictionary Contacts { get; set; } = []; + } + + [Test] + public void OmittedOrNullMessages_UseExceptionFactoryDefault() + { + var result = CoreEx.Results.Result.ValidationError(x => x.Name); + var typed = CoreEx.Results.Result.ValidationError(x => x.Name, null); + AssertError(result.Error, "name", "Name is invalid."); + AssertError(typed.Error, "name", "Name is invalid."); + var configured = CoreEx.Results.Result.ValidationError(x => x.Children[0].Name, configure: e => e.Messages.Should().ContainSingle()); + AssertError(configured.Error, "children[0].name", "Name is invalid."); + } + + [Test] + public void PlainTextAndPreformattedMessages_MatchExceptionFactory() + { + var text = new LText("DuplicateName", "Name {0} is taken.", "Casper"); + var result = CoreEx.Results.Result.ValidationError(x => x.Name, text); + var typed = CoreEx.Results.Result.ValidationError(x => x.Name, text); + + result.IsFailure.Should().BeTrue(); + typed.IsFailure.Should().BeTrue(); + AssertError(result.Error, "name", "Name Casper is taken."); + AssertError(typed.Error, "name", "Name Casper is taken."); + AssertError(CoreEx.Results.Result.ValidationError(x => x.Age, "Invalid.").Error, "age", "Invalid."); + } + + [Test] + public void FormattingAndIndexing_MatchExceptionFactory() + { + var text = new LText("DuplicateName", "Name {0} is taken.", "Casper"); + var result = CoreEx.Results.Result.ValidationError(x => x.Children[2].Name, text); + var typed = CoreEx.Results.Result.ValidationError(x => x.Children[2].Name, text); + + AssertError(result.Error, "children[2].name", "Name Casper is taken."); + AssertError(typed.Error, "children[2].name", "Name Casper is taken."); + } + + [Test] + public void AdvancedScenarios_UseExceptionFactoryFallback() + { + var options = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper, DictionaryKeyPolicy = JsonNamingPolicy.CamelCase }; + var key = "HomeAddress"; + var text = new LText("DuplicateName", "Name {0} is taken.", "Casper"); + CoreEx.Results.Result plain = ValidationException.Create(x => x.Contacts[key].Name, options, text); + CoreEx.Results.Result typedPlain = ValidationException.Create(x => x.Contacts[key].Name, options, text); + CoreEx.Results.Result formatted = ValidationException.Create(x => x.Contacts[key].Name, options, "Name {0} is taken.", "Casper"); + CoreEx.Results.Result typedFormatted = ValidationException.Create(x => x.Contacts[key].Name, options, "Name {0} is taken.", "Casper"); + + foreach (var error in new[] { plain.Error, typedPlain.Error, formatted.Error, typedFormatted.Error }) + AssertError(error, "CONTACTS.homeAddress.NAME", "Name Casper is taken."); + + } + + [Test] + public void Configuration_ReceivesTheCreatedException() + { + var exceptions = new List(); + var result = CoreEx.Results.Result.ValidationError(x => x.Name, "Invalid.", e => exceptions.Add(e)); + var typed = CoreEx.Results.Result.ValidationError(x => x.Name, "Invalid.", e => exceptions.Add(e)); + exceptions.Should().HaveCount(2); + result.Error.Should().BeSameAs(exceptions[0]); + typed.Error.Should().BeSameAs(exceptions[1]); + } + + [Test] + public void InvalidExpressions_AreRejectedRatherThanReturnedAsFailures() + { + var result = () => CoreEx.Results.Result.ValidationError(x => x.Age.ToString(), "Invalid."); + var typed = () => CoreEx.Results.Result.ValidationError(x => x.Age.ToString(), "Invalid."); + result.Should().Throw().WithParameterName("propertyExpression"); + typed.Should().Throw().WithParameterName("propertyExpression"); + } + + private static void AssertError(Exception? error, string path, string text) + { + var exception = error.Should().BeOfType().Subject; + var message = exception.Messages.Should().ContainSingle().Subject; + message.Property.Should().Be(path); + message.Text.ToString().Should().Be(text); + exception.StatusCode.Should().Be(HttpStatusCode.BadRequest); + } +} diff --git a/tests/CoreEx.Test.Unit/Runtime/MetadataSourceTests.cs b/tests/CoreEx.Test.Unit/Runtime/MetadataSourceTests.cs new file mode 100644 index 000000000..126ba9466 --- /dev/null +++ b/tests/CoreEx.Test.Unit/Runtime/MetadataSourceTests.cs @@ -0,0 +1,135 @@ +namespace CoreEx.Test.Unit.Runtime; + +[TestFixture] +public partial class MetadataSourceTests +{ + [Contract] + private partial class TypeFirstContract + { + public string? Name { get; set; } + + [ContractIgnore] + public string? Excluded { get; set; } + } + + [Contract] + private partial class GenericFirstContract + { + public string? Name { get; set; } + + [ContractIgnore] + public string? Excluded { get; set; } + } + + private class StaticEntity : IRuntimeMetadata + { + public StaticEntity() => throw new InvalidOperationException("Metadata must not construct entities."); + + [JsonPropertyName("reflectedName")] + public string? Name { get; set; } + + public string? Excluded => throw new InvalidOperationException("Metadata must not execute property getters."); + + private static readonly IPropertyRuntimeMetadata[] _properties = + [ + new PropertyRuntimeMetadata(nameof(Name), x => x.Name, text: () => "Custom friendly name", jsonName: "configuredName") + ]; + + static IEnumerable IRuntimeMetadata.GetStaticPropertyRuntimeMetadata() => _properties; + + public IEnumerable GetPropertyRuntimeMetadata() => _properties; + } + + private class Wrapper + { + public StaticEntity? Entity { get; set; } + } + + private class InstanceOnlyEntity : IRuntimeMetadataCore + { + public InstanceOnlyEntity() => throw new InvalidOperationException("Metadata must not construct entities."); + + [JsonPropertyName("reflectedName")] + public string? Name { get; set; } + + public IEnumerable GetPropertyRuntimeMetadata() => throw new InvalidOperationException("Instance metadata must not be invoked."); + } + + private class BrokenStaticEntity : IRuntimeMetadata + { + public string? Name { get; set; } + + static IEnumerable IRuntimeMetadata.GetStaticPropertyRuntimeMetadata() => throw new InvalidOperationException("Broken metadata."); + + public IEnumerable GetPropertyRuntimeMetadata() => throw new NotSupportedException(); + } + + [Test] + public void GeneratedMetadata_IsConsistentRegardlessOfCacheEntryPoint() + { + var typeFirst = RuntimeMetadata.GetCachedProperties(typeof(TypeFirstContract)); + typeFirst.Should().BeSameAs(RuntimeMetadata.GetCachedProperties()); + typeFirst.Keys.Should().ContainSingle().Which.Should().Be(nameof(TypeFirstContract.Name)); + + var genericFirst = RuntimeMetadata.GetCachedProperties(); + genericFirst.Should().BeSameAs(RuntimeMetadata.GetCachedProperties(typeof(GenericFirstContract))); + genericFirst.Keys.Should().ContainSingle().Which.Should().Be(nameof(GenericFirstContract.Name)); + } + + [Test] + public void TypeLookup_PrefersExplicitStaticMetadataAndCachesIt() + { + var first = RuntimeMetadata.GetCachedProperties(typeof(StaticEntity)); + var second = RuntimeMetadata.GetCachedProperties(); + + first.Should().BeSameAs(second); + first.Keys.Should().ContainSingle().Which.Should().Be(nameof(StaticEntity.Name)); + first[nameof(StaticEntity.Name)].GetJsonName().Should().Be("configuredName"); + first[nameof(StaticEntity.Name)].Text.ToString().Should().Be("Custom friendly name"); + RuntimeMetadata.GetCachedProperties(typeof(StaticEntity), nameof(StaticEntity.Name)).Should().BeEmpty(); + RuntimeMetadata.GetCachedProperties(typeof(StaticEntity)).Should().ContainKey(nameof(StaticEntity.Name)); + } + + [Test] + public void NestedPath_UsesStaticNamesAndTextWithoutInstances() + { + // Null-forgiving is safe because the expression is inspected without executing entity access. + var metadata = RuntimeMetadata.GetJsonPathAndText(x => x.Entity!.Name); + metadata.Path.Should().Be("entity.configuredName"); + metadata.Text.ToString().Should().Be("Custom friendly name"); + // No entity instance is required for inferred messages either. + var exception = ValidationException.Create(x => x.Entity!.Name); + exception.Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Custom friendly name is invalid."); + } + + [Test] + public void ExplicitPropertyLookup_ReflectsMissingPropertiesWithoutChangingStaticCollection() + { + var properties = RuntimeMetadata.GetCachedProperties(typeof(StaticEntity)); + var configured = RuntimeMetadata.GetForExpression(x => x.Name, null); + configured.Should().BeSameAs(properties[nameof(StaticEntity.Name)]); + configured.GetJsonName().Should().Be("configuredName"); + + var reflected = RuntimeMetadata.GetForExpression(x => x.Excluded, null); + reflected.Should().BeSameAs(RuntimeMetadata.GetForExpression(x => x.Excluded, null)); + reflected.GetJsonName().Should().Be("excluded"); + RuntimeMetadata.GetJsonPath(x => x.Entity!.Excluded).Should().Be("entity.excluded"); + properties.Keys.Should().ContainSingle().Which.Should().Be(nameof(StaticEntity.Name)); + } + + [Test] + public void InstanceOnlyMetadata_FallsBackToReflectionWithoutConstruction() + { + RuntimeMetadata.GetCachedProperties(typeof(InstanceOnlyEntity))[nameof(InstanceOnlyEntity.Name)].GetJsonName().Should().Be("reflectedName"); + RuntimeMetadata.GetJsonPath(x => x.Name).Should().Be("reflectedName"); + } + + [Test] + public void StaticProviderErrors_PropagateWithoutReflectionFallback() + { + var action = () => RuntimeMetadata.GetCachedProperties(typeof(BrokenStaticEntity)); + action.Should().Throw().WithMessage("Broken metadata."); + var expression = () => RuntimeMetadata.GetJsonPath(x => x.Name); + expression.Should().Throw().WithMessage("Broken metadata."); + } +} diff --git a/tests/CoreEx.Test.Unit/ValidationExceptionCreateTests.cs b/tests/CoreEx.Test.Unit/ValidationExceptionCreateTests.cs new file mode 100644 index 000000000..55023016b --- /dev/null +++ b/tests/CoreEx.Test.Unit/ValidationExceptionCreateTests.cs @@ -0,0 +1,225 @@ +namespace CoreEx.Test.Unit; + +[TestFixture] +public partial class ValidationExceptionCreateTests +{ + public class Person + { + public string? Name { get; set; } + public int Age { get; set; } + public int? OptionalAge { get; set; } + [System.ComponentModel.DataAnnotations.Display(Name = "Preferred contact name")] + public string? ContactName { get; set; } + public Address Address => throw new InvalidOperationException("The expression must not execute getters."); + public Person? Parent { get; set; } + public List
Addresses { get; set; } = []; + public Address[] AddressArray { get; set; } = []; + public IList
AddressList { get; set; } = new List
(); + public IReadOnlyList
ReadOnlyAddresses { get; set; } = []; + public Dictionary AddressesByType { get; set; } = []; + public IDictionary AddressDictionary { get; set; } = new Dictionary(); + public IReadOnlyDictionary> AddressGroups { get; set; } = new Dictionary>(); + public Dictionary NumericAddresses { get; set; } = []; + public int ComputedIndex => throw new InvalidOperationException("Index getters must not execute."); + public static string StaticName => "Name"; + public string Field = ""; + } + + public class Address + { + [Localization("SpookyStreet", "Street address")] + public string? Street { get; set; } + public int HouseNumber { get; set; } + } + + public class InheritedPerson : Person; + + [Contract] + private partial class NamedPerson + { + [JsonPropertyName("homeAddress")] + public NamedAddress? Address { get; set; } + } + + [Contract] + private partial class NamedAddress + { + [JsonPropertyName("street_name")] + public string? Street { get; set; } + } + + [Test] + public void OmittedOrNullMessage_UsesMetadataFriendlyText() + { + ValidationException.Create(x => x.Name).Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Name is invalid."); + ValidationException.Create(x => x.Name, null).Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Name is invalid."); + ValidationException.Create(x => x.ContactName).Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Preferred contact name is invalid."); + ValidationException.Create(x => x.Address.Street, (LText?)null).Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Street address is invalid."); + var options = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper }; + var exception = ValidationException.Create(x => x.Addresses[0].Street, options, (LText?)null); + var message = exception.Messages.Should().ContainSingle().Subject; + message.Property.Should().Be("ADDRESSES[0].STREET"); + message.Text.ToString().Should().Be("Street address is invalid."); + var metadata = RuntimeMetadata.GetJsonPathAndText(x => x.Address.HouseNumber); + metadata.Path.Should().Be("address.houseNumber"); + metadata.Text.ToString().Should().Be("House number"); + ValidationException.Create(x => x.Addresses[0]).Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Addresses is invalid."); + ValidationException.Create(x => x.AddressesByType["home"].Street).Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Street address is invalid."); + } + + [Test] + public void DirectProperty_UsesDefaultJsonNamingAndPreservesExceptionSemantics() + { + var exception = ValidationException.Create(x => x.Name, "Name is spooky."); + + var message = exception.Messages.Should().ContainSingle().Subject; + message.Property.Should().Be(RuntimeMetadata.GetForExpression(x => x.Name).GetJsonName()); + message.Text.ToString().Should().Be("Name is spooky."); + message.Type.Should().Be(MessageType.Error); + exception.StatusCode.Should().Be(HttpStatusCode.BadRequest); + exception.ErrorType.Should().Be("validation"); + } + + [Test] + public void NestedProperty_ResolvesEverySegmentWithoutExecutingGetters() + { + var exception = ValidationException.Create(x => x.Address.Street, "Street is spooky."); + + exception.Messages.Should().ContainSingle().Which.Property.Should().Be("address.street"); + // Null-forgiving is safe because nullable intermediate values are never evaluated. + RuntimeMetadata.GetJsonPath(x => x.Parent!.Parent!.Address.Street).Should().Be("parent.parent.address.street"); + RuntimeMetadata.GetJsonPath(x => x.Address.Street).Should().Be("address.street"); + } + + [Test] + public void ExplicitNames_TakePrecedenceAtEveryLevel() + { + // Null-forgiving is safe: only the expression structure is inspected, not the nullable address value. + var exception = ValidationException.Create(x => x.Address!.Street, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper }, "Spooky."); + + exception.Messages.Should().ContainSingle().Which.Property.Should().Be("homeAddress.street_name"); + } + + [Test] + public void ExplicitOptions_ApplyToEverySegment() + { + var options = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper }; + var exception = ValidationException.Create(x => x.Address.HouseNumber, options, "House {0} is spooky.", 13); + + var message = exception.Messages.Should().ContainSingle().Subject; + message.Property.Should().Be("ADDRESS.HOUSE_NUMBER"); + message.Text.ToString().Should().Be("House 13 is spooky."); + RuntimeMetadata.GetJsonPath(x => x.Address.HouseNumber, new JsonSerializerOptions()).Should().Be("Address.HouseNumber"); + } + + [Test] + public void Boxing_SupportsValueTypesAndNullableValueTypes() + { + ValidationException.Create(x => x.Age, "Spooky.").Messages.Should().ContainSingle().Which.Property.Should().Be("age"); + ValidationException.Create(x => x.OptionalAge, "Spooky.").Messages.Should().ContainSingle().Which.Property.Should().Be("optionalAge"); + ValidationException.Create(x => x.Address.HouseNumber, "Spooky.").Messages.Should().ContainSingle().Which.Property.Should().Be("address.houseNumber"); + } + + [Test] + public void FormattedAndPreformattedLText_PreserveExistingFactoryBehavior() + { + var expected = ValidationException.Create("name", "Name {0} is spooky.", "Casper"); + var actual = ValidationException.Create(x => x.Name, "Name {0} is spooky.", "Casper"); + actual.Messages.Should().ContainSingle().Which.Text.Should().Be(expected.Messages.Should().ContainSingle().Subject.Text); + + var text = new LText("SpookyName", "Name {0} is spooky.", "Casper"); + ValidationException.Create(x => x.Name, text).Messages.Should().ContainSingle().Which.Text.Should() + .Be(ValidationException.Create("name", text).Messages.Should().ContainSingle().Subject.Text); + ValidationException.Create(x => x.Name, text).Messages.Should().ContainSingle().Which.Text.ToString().Should().Be("Name Casper is spooky."); + ValidationException.Create(x => x.Name, null, text).Messages.Should().ContainSingle().Which.Property.Should().Be("name"); + } + + [Test] + public void NullExpression_IsRejected() + { + Expression>? expression = null; + // Deliberately pass null to verify the runtime guard. + var action = () => ValidationException.Create(expression!, "Spooky."); + action.Should().Throw().WithParameterName("propertyExpression"); + } + + [Test] + public void ArrayAndListIndexes_UseValidatorNotationWithoutAccessingItems() + { + RuntimeMetadata.GetJsonPath(x => x.AddressArray[2].Street).Should().Be("addressArray[2].street"); + RuntimeMetadata.GetJsonPath(x => x.Addresses[2].Street).Should().Be("addresses[2].street"); + RuntimeMetadata.GetJsonPath(x => x.AddressList[2].Street).Should().Be("addressList[2].street"); + RuntimeMetadata.GetJsonPath(x => x.ReadOnlyAddresses[2].Street).Should().Be("readOnlyAddresses[2].street"); + RuntimeMetadata.GetJsonPath(x => x.AddressArray[2]).Should().Be("addressArray[2]"); + RuntimeMetadata.GetJsonPath(x => x.AddressesByType["home"].HouseNumber).Should().Be("addressesByType.home.houseNumber"); + } + + [TestCase("home")] + [TestCase("HomeAddress")] + [TestCase("home.with[brackets]")] + [TestCase("")] + public void DictionaryKeys_UseSerializerPolicyAsInValidators(string key) + { + var options = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper, DictionaryKeyPolicy = JsonNamingPolicy.SnakeCaseUpper }; + ValidationException.Create(x => x.AddressesByType[key].Street, options, "Spooky.") + .Messages.Should().ContainSingle().Which.Property.Should().Be($"ADDRESSES_BY_TYPE.{options.DictionaryKeyPolicy.ConvertName(key)}.STREET"); + RuntimeMetadata.GetJsonPath(x => x.AddressDictionary[key].Street, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }) + .Should().Be($"addressDictionary.{key}.street"); + } + + [Test] + public void CapturedLocals_UseCurrentValuesAndSupportMixedNesting() + { + var index = 1; + var key = "home"; + Expression> expression = x => x.AddressGroups[key][index].Street; + RuntimeMetadata.GetJsonPath(expression).Should().Be("addressGroups.home[1].street"); + index = 3; + key = "work"; + RuntimeMetadata.GetJsonPath(expression).Should().Be("addressGroups.work[3].street"); + } + + [Test] + public void NullDictionaryKey_IsRejected() + { + string? key = null; + // Deliberately pass a null key to verify rejection without accessing the dictionary. + var action = () => RuntimeMetadata.GetJsonPath(x => x.AddressesByType[key!].Street); + action.Should().Throw().WithParameterName("propertyExpression"); + } + + [Test] + public void UnsupportedExpressions_AreRejected() + { + var captured = new Person(); + Expression>[] expressions = + [ + x => x, + x => "constant", + x => x.Age.ToString(), + x => captured.Name, + x => Person.StaticName, + x => x.Field, + x => x.Addresses.ToArray()[0].Street, + x => x.Addresses[-1].Street, + x => x.Addresses[x.Age].Street, + x => x.Addresses[captured.ComputedIndex].Street, + x => x.Addresses[GetIndex()].Street, + x => x.Addresses[1 + captured.Age].Street, + x => x.NumericAddresses[0].Street, + x => x.AddressesByType[GetKey()].Street, + x => (long)x.Age, + x => ((Person)(object)x).Name + ]; + + foreach (var expression in expressions) + { + var action = () => ValidationException.Create(expression, "Spooky."); + action.Should().Throw().WithParameterName("propertyExpression"); + } + } + + private static int GetIndex() => throw new InvalidOperationException("Index methods must not execute."); + + private static string GetKey() => throw new InvalidOperationException("Key methods must not execute."); +} diff --git a/tests/CoreEx.Validation.Test.Unit/JsonPathParityTests.cs b/tests/CoreEx.Validation.Test.Unit/JsonPathParityTests.cs new file mode 100644 index 000000000..7571a0e0d --- /dev/null +++ b/tests/CoreEx.Validation.Test.Unit/JsonPathParityTests.cs @@ -0,0 +1,109 @@ +namespace CoreEx.Validation.Test.Unit; + +[TestFixture] +public class JsonPathParityTests +{ + public class Person + { + [System.Text.Json.Serialization.JsonPropertyName("addressEntries")] + public List
Addresses { get; set; } = []; + public Address[] AddressArray { get; set; } = []; + public Dictionary AddressesByType { get; set; } = []; + public Dictionary NumericAddresses { get; set; } = []; + public Dictionary Values { get; set; } = []; + } + + public class Address + { + [System.Text.Json.Serialization.JsonPropertyName("street_name")] + public string? Street { get; set; } + } + + [TestCase("home")] + [TestCase("HomeAddress")] + [TestCase("home.with[brackets]")] + [TestCase("")] + [TestCase("ETag")] + public async Task DictionaryFactoryPath_MatchesValidatorExactly(string key) + { + var addressValidator = Validator.Create
().HasProperty(x => x.Street, r => r.Mandatory()); + var validator = Validator.Create().HasProperty(x => x.AddressesByType, r => r.Dictionary(w => w.WithValueValidator(addressValidator))); + var options = new System.Text.Json.JsonSerializerOptions + { + PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.SnakeCaseUpper, + DictionaryKeyPolicy = System.Text.Json.JsonNamingPolicy.SnakeCaseUpper + }; + var result = await validator.ValidateAsync(new Person { AddressesByType = new() { [key] = new() } }, new ValidationArgs { JsonSerializerOptions = options }); + var exception = ValidationException.Create(x => x.AddressesByType[key].Street, options, "Spooky."); + + exception.Messages.Should().ContainSingle().Which.Property.Should().Be(result.Messages.Should().ContainSingle().Subject.Property); + } + + [TestCase("HomeAddress", "default")] + [TestCase("ETag", "default")] + [TestCase("HomeAddress", "none")] + [TestCase("HomeAddress", "snake")] + public async Task DictionaryPaths_MatchSerializedJsonAndPreserveOriginalKey(string key, string policy) + { + var options = policy switch + { + "none" => new System.Text.Json.JsonSerializerOptions { PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.CamelCase }, + "snake" => new System.Text.Json.JsonSerializerOptions { PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.CamelCase, DictionaryKeyPolicy = System.Text.Json.JsonNamingPolicy.SnakeCaseUpper }, + _ => CoreEx.Json.JsonDefaults.SerializerOptions + }; + var person = new Person { AddressesByType = new() { [key] = new() } }; + var json = System.Text.Json.JsonSerializer.SerializeToElement(person, options); + var dictionary = json.GetProperty("addressesByType"); + var serializedKey = dictionary.EnumerateObject().Single().Name; + var expected = $"addressesByType.{serializedKey}"; + var addressValidator = Validator.Create
().HasProperty(x => x.Street, r => r.Mandatory().When(c => c.GetDictionaryKey() == key)); + var valueValidator = Validator.Create().HasProperty(x => x.AddressesByType, r => r.Dictionary(w => w.WithValueValidator(addressValidator))); + var keyValidator = Validator.Create().HasProperty(x => x.AddressesByType, r => r.Dictionary(w => w.WithKeyValidator(k => k.MaximumLength(1)))); + var args = new ValidationArgs { JsonSerializerOptions = options }; + + var valueResult = await valueValidator.ValidateAsync(person, args); + valueResult.Messages.Should().ContainSingle().Which.Property.Should().Be(expected + ".street_name"); + var keyResult = await keyValidator.ValidateAsync(person, args); + keyResult.Messages.Should().ContainSingle().Which.Property.Should().Be(expected); + ValidationException.Create(x => x.AddressesByType[key].Street, options, "Spooky.") + .Messages.Should().ContainSingle().Which.Property.Should().Be(expected + ".street_name"); + + var clrValue = await valueValidator.ValidateAsync(person, args with { UseJsonNames = false }); + clrValue.Messages.Should().ContainSingle().Which.Property.Should().Be($"AddressesByType[{key}].Street"); + var clrKey = await keyValidator.ValidateAsync(person, args with { UseJsonNames = false }); + clrKey.Messages.Should().ContainSingle().Which.Property.Should().Be($"AddressesByType.{key}"); + person.AddressesByType.Keys.Should().ContainSingle().Which.Should().Be(key); + } + + [Test] + public async Task InlineValueAndNonStringKeyPaths_PreserveExpectedBehavior() + { + var validator = Validator.Create() + .HasProperty(x => x.Values, r => r.Dictionary(w => w.WithValueValidator(v => v.MaximumLength(1)))) + .HasProperty(x => x.NumericAddresses, r => r.Dictionary(w => w.WithValueValidator( + Validator.Create
().HasProperty(x => x.Street, r => r.Mandatory())))); + var result = await validator.ValidateAsync(new Person + { + Values = new() { ["ETag"] = "too long" }, + NumericAddresses = new() { [13] = new() } + }); + + result.Messages.Should().NotBeNull(); + result.Messages?.Select(x => x.Property).Should().BeEquivalentTo("values.etag", "numericAddresses.13.street_name"); + } + + [Test] + public async Task ListAndArrayFactoryPaths_MatchValidatorExactly() + { + var addressValidator = Validator.Create
().HasProperty(x => x.Street, r => r.Mandatory()); + var validator = Validator.Create() + .HasProperty(x => x.Addresses, r => r.Collection(w => w.WithItemValidator(addressValidator))) + .HasProperty(x => x.AddressArray, r => r.Collection(w => w.WithItemValidator(addressValidator))); + var result = await validator.ValidateAsync(new Person { Addresses = [new()], AddressArray = [new()] }); + + var listPath = ValidationException.Create(x => x.Addresses[0].Street, "Spooky.").Messages.Should().ContainSingle().Subject.Property; + var arrayPath = ValidationException.Create(x => x.AddressArray[0].Street, "Spooky.").Messages.Should().ContainSingle().Subject.Property; + result.Messages.Should().NotBeNull(); + result.Messages?.Select(x => x.Property).Should().BeEquivalentTo(listPath, arrayPath); + } +} From 51493266a46ff7cdd30c5943bcb070013707a4f4 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Thu, 22 Oct 2026 17:00:21 -0700 Subject: [PATCH 23/25] Preserve dictionary keys and correct JSON Merge Patch semantics Use case-sensitive RFC 7396 member matching by default, recursively merge object patches, and avoid changes for absent-member removals. Preserve dictionary keys in serializer defaults and align regression coverage and documentation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- CHANGELOG.md | 5 +- src/CoreEx/Json/JsonDefaults.cs | 5 +- src/CoreEx/Json/JsonMergePatch.cs | 70 ++++++++---------- src/CoreEx/Json/JsonMergePatchOptions.cs | 7 +- src/CoreEx/Json/README.md | 14 ++-- .../WebApiTestsBase.MergePatch.cs | 15 +++- .../Json/JsonDefaultsTests.cs | 60 +++++++++++++++ .../Json/JsonMergePatchRfcTests.cs | 74 +++++++++++++++++++ .../JsonPathParityTests.cs | 2 +- 9 files changed, 198 insertions(+), 54 deletions(-) create mode 100644 tests/CoreEx.Test.Unit/Json/JsonDefaultsTests.cs create mode 100644 tests/CoreEx.Test.Unit/Json/JsonMergePatchRfcTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 738be1450..d4df22850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,13 +18,16 @@ Represents the **NuGet** versions. - *Enhancement:* Added OData-style free-text `$search` support to the `QueryArgs` querying pipeline: `QueryArgs.Search`; `QueryArgsConfig.WithSearch(Action)` with `QuerySearchMode` (`Keyword` executed by `QueryExtensions.Where` as a provider-agnostic predicate where every term must match a configured keyword field; `FullText`/`Semantic`/`Hybrid` are relevance-ranked and executed by the repository via `QueryArgsParseResult.IsRankedSearch`/`SearchResult.Text`, with `$orderby` rejected); `QuerySearchParserException`; `WithSearch` on `HttpRequestMessage`/`HttpRequestBuilder`; `QueryAttribute.SupportsSearch` (NSwag `$search` parameter); and a GraphQL-lite `search` argument on list roots. `$search` is rejected where not configured. The Products sample (and its MCP `search_products` tool) now uses `$search` to demonstrate. - *Enhancement:* Added `DataResult.Map` to transform mutation-result values while preserving `WasMutated`, including successful no-ops; simplified Shopping repository mapping and aligned usage guidance. - *Behavior change:* Added a string-specific `Required()` validation overload that rejects null, empty and whitespace values with a `ValidationException`. Existing string call sites now reject empty/whitespace strings rather than only null. +- *Behavior change:* `JsonDefaults` now preserves dictionary keys by default (`DictionaryKeyPolicy = null`) instead of camel-casing/substituting them, preventing key identity changes and casing collisions during JSON round-trips. Property naming is unchanged; explicitly configured dictionary key policies remain supported. Dictionary validation/error paths preserve keys by default. Existing consumers relying on transformed keys must explicitly configure their previous policy. +- *Behavior change:* JSON Merge Patch now matches member names case-sensitively by default (`StringComparer.Ordinal`) to follow RFC 7396 and distinguish dictionary keys such as `P1` and `p1`. Clients must use exact serialized member names; explicitly configure `PropertyNameComparer = StringComparer.OrdinalIgnoreCase` only for previous non-standard compatibility. - *Behavior change:* Preserve Shopping aggregate mutation metadata through repository mapping and gate Create/Update/Checkout mutation events with `WhereMutated`; align repository/service guidance and add successful-no-op publication regressions. - *Behavior change:* Aligned Domain guidance and Shopping pricing/discount examples on business errors for aggregate/entity/value-object invariants, distinct from request validation. Pricing rejections (negative values, invalid/inactive unit of measure, quantity scale) now throw coded `BusinessException`; invalid/inactive discount application returns a coded business failure. Request validators retain field errors; shared argument/reference-data guards keep their existing exception semantics. - *Maintenance:* Updated framework dependencies and corresponding `CoreEx.Template` dependencies, including Aspire 13.6.1, EF/.NET servicing packages and FusionCache 2.9.0. The SQL Server package now explicitly references `System.Security.Cryptography.Pkcs` 10.0.13; the Aspire template's MessagePack override is updated to 2.5.307. - *Maintenance:* Pinned local development, CI and package publishing to the stable .NET SDK in `global.json`; updated `nuget-publish.ps1` to build before packing and remove matching package versions from the local NuGet cache by default. - *Documentation:* Aligned application/API instructions, policy/service/adapter skills, and package/sample guidance on missing target resources (404) versus invalid request-body references (field-level 400), including query filters, expected downstream lookup outcomes, and deliberate endpoint semantics. - *Fixed:* Named/typed HTTP client registration now binds standard resilience settings from the nested `Resilience` configuration section rather than the parent client section. -- *Fixed:* Dictionary validation JSON paths now apply the effective serializer's `DictionaryKeyPolicy` to string keys for both key and value errors, matching serialized JSON and expression-based exception factories. Original keys remain unchanged in CLR paths and `GetDictionaryKey()`; affected JSON error paths now use the serialized casing (for example, `HomeAddress` becomes `homeAddress` with the default policy). +- *Fixed:* JSON Merge Patch now recursively processes object patches for new members and non-object targets, removing null-valued patch members rather than copying them literally. Removing a nonexistent member no longer reports a change. +- *Fixed:* Dictionary validation JSON paths now apply the effective serializer's `DictionaryKeyPolicy` to string keys for both key and value errors, matching serialized JSON and expression-based exception factories. Original keys remain unchanged in CLR paths and `GetDictionaryKey()`; JSON error paths preserve keys by default and use transformed casing only when a dictionary key policy is explicitly configured. - *Fixed:* Shared cached property metadata lookups now consistently prefer static `IRuntimeMetadata` providers, including explicit implementations, with reflection as the fallback for other types. JSON path/friendly-label resolution respects custom/generated metadata for both root and nested types regardless of cache-population order. Explicit expression lookups separately cache reflected metadata for properties omitted by the static provider (including generated reference-data navigation properties), preserving validation support without changing contract equality/copy/clean property collections. - *Fixed:* GraphQL-lite search parsing errors now preserve actionable per-property validation details in `extensions.messages["$search"]` while retaining the `SEARCH_PARSE_ERROR` code, including unsupported search, length/term limits and custom validation failures. - *Fixed:* `CoreEx.UnitTesting` and the sample/template MockHost now explicitly reference MessagePack 2.5.307 so their transitive Aspire dependency cannot resolve the vulnerable 2.5.302 version; a central version entry alone does not pin transitive dependencies. diff --git a/src/CoreEx/Json/JsonDefaults.cs b/src/CoreEx/Json/JsonDefaults.cs index 902c61f0b..81472ae18 100644 --- a/src/CoreEx/Json/JsonDefaults.cs +++ b/src/CoreEx/Json/JsonDefaults.cs @@ -29,6 +29,7 @@ public sealed class JsonDefaultConfiguration /// /// Initializes a new instance of the class with default settings. /// + /// Property names use the substitute naming policy; dictionary keys are preserved by default. public JsonDefaultConfiguration() { SerializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) @@ -36,7 +37,7 @@ public JsonDefaultConfiguration() DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault, WriteIndented = false, PropertyNamingPolicy = _jsonSubstituteNamingPolicy, - DictionaryKeyPolicy = _jsonSubstituteNamingPolicy, + DictionaryKeyPolicy = null, Converters = { new JsonStringEnumConverter(), new JsonReferenceDataConverter(), new JsonResultConverter(), new JsonDataMapConverterFactory() } }; } @@ -46,4 +47,4 @@ public JsonDefaultConfiguration() ///
public JsonSerializerOptions SerializerOptions { get; set => field = value.ThrowIfNull(); } } -} \ No newline at end of file +} diff --git a/src/CoreEx/Json/JsonMergePatch.cs b/src/CoreEx/Json/JsonMergePatch.cs index 82d8a3e26..95a9f9318 100644 --- a/src/CoreEx/Json/JsonMergePatch.cs +++ b/src/CoreEx/Json/JsonMergePatch.cs @@ -217,38 +217,12 @@ private bool TryMerge(JsonElement patch, JsonElement target, [NotNullWhen(true)] /// private bool TryMerge(JsonElement patch, JsonElement target, Utf8JsonWriter writer, bool changed) { - // Where the kinds are different then simply accept the merge and acknowledge as changed. - if (patch.ValueKind != target.ValueKind) - { - patch.WriteTo(writer); - return true; - } - - // Where the kinds are the same then process accordingly. - switch (patch.ValueKind) - { - case JsonValueKind.Object: - // An object is a property-by-property merge. - return TryObjectMerge(patch, target, writer, changed); - - case JsonValueKind.Array: - // An array is always a replacement. - patch.WriteTo(writer); - if (!DeepEquals(patch, target)) - changed = true; - - break; - - default: - // Accept merge as-is. - patch.WriteTo(writer); - if (!DeepEquals(patch, target)) - changed = true; - - break; - } + // Object patches merge recursively, treating a non-object target as an empty object. + if (patch.ValueKind == JsonValueKind.Object) + return TryObjectMerge(patch, target, writer, changed || target.ValueKind != JsonValueKind.Object); - return changed; + patch.WriteTo(writer); + return changed || !DeepEquals(patch, target); } /// @@ -264,10 +238,11 @@ private bool TryObjectMerge(JsonElement patch, JsonElement target, Utf8JsonWrite // Where the property is new then add. if (!TryGetProperty(target, j.Name, out var t)) { - if (j.Value.ValueKind != JsonValueKind.Null) - j.WriteTo(writer); + if (j.Value.ValueKind == JsonValueKind.Null) + continue; - changed = true; + writer.WritePropertyName(j.Name); + changed = TryMerge(j.Value, default, writer, true); continue; } @@ -282,13 +257,16 @@ private bool TryObjectMerge(JsonElement patch, JsonElement target, Utf8JsonWrite } // Add existing target properties not being merged. - foreach (var t in target.EnumerateObject()) + if (target.ValueKind == JsonValueKind.Object) { - // Where found then consider as handled above and ignore! - if (TryGetProperty(patch, t.Name, out _)) - continue; + foreach (var t in target.EnumerateObject()) + { + // Where found then consider as handled above and ignore! + if (TryGetProperty(patch, t.Name, out _)) + continue; - t.WriteTo(writer); + t.WriteTo(writer); + } } writer.WriteEndObject(); @@ -299,7 +277,17 @@ private bool TryObjectMerge(JsonElement patch, JsonElement target, Utf8JsonWrite /// Performs the TryGetProperty using the configured comparer. /// private bool TryGetProperty(JsonElement json, string propertyName, out JsonElement value) - => Options.PropertyNameComparer is null ? json.TryGetProperty(propertyName, out value) : TryGetPropertyWithComparer(json, propertyName, Options.PropertyNameComparer, out value); + { + if (json.ValueKind != JsonValueKind.Object) + { + value = default; + return false; + } + + return Options.PropertyNameComparer is null || Options.PropertyNameComparer == StringComparer.Ordinal + ? json.TryGetProperty(propertyName, out value) + : TryGetPropertyWithComparer(json, propertyName, Options.PropertyNameComparer, out value); + } /// /// Performs the TryGetProperty using the specified comparer (this will impact performance). @@ -381,4 +369,4 @@ internal static bool DeepEquals(JsonElement left, JsonElement right) return true; } } -} \ No newline at end of file +} diff --git a/src/CoreEx/Json/JsonMergePatchOptions.cs b/src/CoreEx/Json/JsonMergePatchOptions.cs index adc4f6da2..6c980d8f2 100644 --- a/src/CoreEx/Json/JsonMergePatchOptions.cs +++ b/src/CoreEx/Json/JsonMergePatchOptions.cs @@ -12,7 +12,8 @@ public class JsonMergePatchOptions(JsonSerializerOptions? jsonSerializerOptions public JsonSerializerOptions JsonSerializerOptions { get; } = jsonSerializerOptions ?? JsonDefaults.SerializerOptions; /// - /// Gets or sets the for matching the JSON name (defaults to ). + /// Gets or sets the for matching the JSON name (defaults to ). /// - public StringComparer PropertyNameComparer { get; set; } = StringComparer.OrdinalIgnoreCase; -} \ No newline at end of file + /// RFC 7396 requires case-sensitive member names. A different comparer is a non-standard compatibility option and applies to every object, including dictionary members. + public StringComparer PropertyNameComparer { get; set; } = StringComparer.Ordinal; +} diff --git a/src/CoreEx/Json/README.md b/src/CoreEx/Json/README.md index 91cb54ff3..0d8ced862 100644 --- a/src/CoreEx/Json/README.md +++ b/src/CoreEx/Json/README.md @@ -8,11 +8,15 @@ Beyond basic serialization, the namespace provides two advanced document-level operations: `JsonMergePatch` (RFC 7396) for partial update semantics used in `PATCH` endpoints, and `JsonFilter` for surgical include/exclude projection of JSON properties — used by the `WebApi` helper to honor `$fields` query parameters. Custom converters handle CoreEx entity types (`DataMap` and reference data codes) transparently. +Property names use camelCase and the configured substitutions (for example, `ETag` becomes `etag`), but dictionary keys are data and are preserved exactly by default (`DictionaryKeyPolicy = null`). This preserves identifiers such as `P1` across JSON round-trips, including when an unrelated merge-patch change causes the target to be deserialized again. Validation and expression-based error paths preserve dictionary keys by default and follow an explicitly configured `DictionaryKeyPolicy` when supplied. Applications that intentionally transform keys can configure that policy at startup; deserialization does not reverse it. + +Merge-patch member matching is case-sensitive by default (`JsonMergePatchOptions.PropertyNameComparer = StringComparer.Ordinal`), as required by RFC 7396, including for dictionary members. Clients should use the exact serialized property names: `name` and `Name` are distinct JSON members. The serializer's case-insensitive CLR property binding does not change JSON member identity; typed results remain subject to the configured deserializer and cannot necessarily represent distinct names that bind to the same CLR property. Setting `PropertyNameComparer = StringComparer.OrdinalIgnoreCase` explicitly restores the previous non-standard matching behavior at every object level. Object patches are always processed recursively, treating non-object targets as empty objects; null removes existing members and is ignored for nonexistent members. `HasChanges` describes changes to the JSON document, not necessarily the resulting CLR value. + ## Key capabilities - ⚙️ **Centralized defaults**: `JsonDefaults.Configuration` provides a single, customizable `JsonDefaultConfiguration` instance; `JsonDefaults.SerializerOptions` resolves the runtime options from `ExecutionContext` or the static configuration. - 🔤 **Substitute naming policy**: `JsonSubstituteNamingPolicy` wraps any naming policy and applies registered name substitutions (e.g. `Id` → `id`), enabling per-property JSON name overrides without `[JsonPropertyName]` attributes on every type. -- 🔀 **JSON Merge Patch (RFC 7396)**: `JsonMergePatch` merges a patch JSON document into a target object, returning a `JsonMergePatchResult` carrying the merged value and the set of changed property paths. +- 🔀 **JSON Merge Patch (RFC 7396)**: `JsonMergePatch` merges a patch JSON document into a target object, returning a `JsonMergePatchResult` carrying the merged value and a `HasChanges` flag. - 📐 **JSON property filtering**: `JsonFilter` applies include or exclude path-based filters to a `JsonNode` or raw JSON string, removing unwanted properties. Used to implement `$fields` response projection. Supports recursive descent paths (`$..Foo`) to match a property at any depth in the hierarchy. - ⚡ **Streaming exclude**: `JsonFilter.TryExcludeUtf8Json` applies an exclude filter directly against UTF-8 JSON bytes via a single-pass `Utf8JsonReader`/`Utf8JsonWriter` copy, without ever materializing a `JsonNode` document — used internally wherever an exclude filter runs on raw JSON (e.g. `EventFormatter.DataExcludePaths`), meaningfully cheaper for high-volume paths than the `JsonNode`-based route that `Include` still requires. - 🗃️ **Reference data converter**: `JsonReferenceDataConverter` serializes reference data types as their `Code` string and deserializes back to the full reference data instance via `ReferenceDataOrchestrator`. @@ -26,9 +30,9 @@ Beyond basic serialization, the namespace provides two advanced document-level o | **[`JsonDefaults`](./JsonDefaults.cs)** | Central access point for the application's `JsonSerializerOptions`; resolves from `ExecutionContext` or the static `JsonDefaultConfiguration`. | | **[`JsonDefaults.JsonDefaultConfiguration`](./JsonDefaults.cs)** | Sealed inner class holding the default `JsonSerializerOptions` with CoreEx-specific converters and naming policy; customize at startup before any serialization occurs. | | **[`JsonSubstituteNamingPolicy`](./JsonSubstituteNamingPolicy.cs)** | `JsonNamingPolicy` implementation that applies registered name substitutions on top of a delegate naming policy (default: camelCase). | -| **[`JsonMergePatch`](./JsonMergePatch.cs)** | Performs RFC 7396 JSON Merge Patch: merges a patch `BinaryData`/string into a typed target, returning `Result>` with the merged value and changed paths. | -| **[`JsonMergePatchOptions`](./JsonMergePatchOptions.cs)** | Options for `JsonMergePatch`: controls whether `null` means removal (RFC default) or is treated as a value, and `StringComparison` for property matching. | -| **[`JsonMergePatchResult`](./JsonMergePatchResult.cs)** | Result record from `JsonMergePatch.Merge()`: carries the merged `Value`, the collection of `ChangedPaths`, and a `HasChanges` flag. | +| **[`JsonMergePatch`](./JsonMergePatch.cs)** | Performs RFC 7396 JSON Merge Patch: merges a patch `BinaryData`/string into a typed target, returning `Result>` with the merged value and a change flag. | +| **[`JsonMergePatchOptions`](./JsonMergePatchOptions.cs)** | Options for `JsonMergePatch`: serializer options and the `PropertyNameComparer` (default: `StringComparer.Ordinal`). | +| **[`JsonMergePatchResult`](./JsonMergePatchResult.cs)** | Result from `JsonMergePatch.Merge()`: carries the `Merged` value and a `HasChanges` flag. | | **[`JsonFilter`](./JsonFilter.cs)** | Static utility applying include/exclude JSON path filters to a `JsonNode`, JSON string, or (via `TryExcludeUtf8Json`, exclude-only) raw UTF-8 bytes with no `JsonNode` allocation; exact path matching with optional indexed or non-indexed path support, plus recursive descent (`$..Foo`) paths that match at any depth. | | **[`JsonFilterOption`](./JsonFilterOption.cs)** | Enum controlling `JsonFilter` behavior: `Include` retains only listed paths; `Exclude` removes listed paths. | | [`JsonReferenceDataConverter`](./JsonReferenceDataConverter.cs) | `JsonConverter` that serializes reference data instances as their `Code` string and deserializes by looking up the code in `ReferenceDataOrchestrator`. | @@ -44,4 +48,4 @@ Beyond basic serialization, the namespace provides two advanced document-level o ## Additional Resources -- [RFC 7396 — JSON Merge Patch](https://tools.ietf.org/html/rfc7396) - The specification that `JsonMergePatch` implements. \ No newline at end of file +- [RFC 7396 — JSON Merge Patch](https://tools.ietf.org/html/rfc7396) - The specification that `JsonMergePatch` implements. diff --git a/tests/CoreEx.AspNetCore.Test.Unit/WebApiTestsBase.MergePatch.cs b/tests/CoreEx.AspNetCore.Test.Unit/WebApiTestsBase.MergePatch.cs index abda4f920..fa93316b5 100644 --- a/tests/CoreEx.AspNetCore.Test.Unit/WebApiTestsBase.MergePatch.cs +++ b/tests/CoreEx.AspNetCore.Test.Unit/WebApiTestsBase.MergePatch.cs @@ -81,6 +81,19 @@ public void MergePatch_No_Changes() .AssertValue(Person.GetPerson("abcdef")); } + [Test] + public void MergePatch_DifferentlyCasedName_DoesNotRemoveProperty() + { + var hr = Test.CreateHttpRequest(HttpMethod.Patch, "test", """{"Age":null}""", r => r.ContentType = HttpNames.MergePatchJsonMediaTypeName); + hr.Headers.IfMatch = new EntityTagHeaderValue("\"abcdef\"", true).ToString(); + + Test.Type() + .Run(async w => await w.PatchAsync(hr, (ro, ct) => Task.FromResult(Person.GetPerson("abcdef")), (ro, ct) => throw new InvalidOperationException())) + .ToHttpResponseMessageAssertor() + .AssertOK() + .AssertValue(Person.GetPerson("abcdef")); + } + [Test] public void MergePatch_With_Changes() { @@ -96,4 +109,4 @@ public void MergePatch_With_Changes() .AssertOK() .AssertValue(ep, "etag"); } -} \ No newline at end of file +} diff --git a/tests/CoreEx.Test.Unit/Json/JsonDefaultsTests.cs b/tests/CoreEx.Test.Unit/Json/JsonDefaultsTests.cs new file mode 100644 index 000000000..3c94255ce --- /dev/null +++ b/tests/CoreEx.Test.Unit/Json/JsonDefaultsTests.cs @@ -0,0 +1,60 @@ +namespace CoreEx.Test.Unit.Json; + +[TestFixture] +public class JsonDefaultsTests +{ + [Test] + public void Defaults_PreserveDictionaryKeysAndPropertyNaming() + { + var options = new CoreEx.Json.JsonDefaults.JsonDefaultConfiguration().SerializerOptions; + var target = new DictionaryDocument + { + DisplayName = "Original", + ETag = "version", + Values = new() { ["P1"] = "upper", ["p1"] = "lower", ["HomeAddress"] = "home", ["ETag"] = "key" } + }; + + options.DictionaryKeyPolicy.Should().BeNull(); + var json = JsonSerializer.Serialize(target, options); + json.Should().Be("""{"displayName":"Original","etag":"version","values":{"P1":"upper","p1":"lower","HomeAddress":"home","ETag":"key"}}"""); + var restored = JsonSerializer.Deserialize(json, options); + restored.Should().NotBeNull(); + restored!.Values.Should().BeEquivalentTo(target.Values); + } + + [Test] + public void ExplicitDictionaryKeyPolicy_RemainsSupported() + { + var options = new CoreEx.Json.JsonDefaults.JsonDefaultConfiguration().SerializerOptions; + options.DictionaryKeyPolicy = JsonNamingPolicy.CamelCase; + + JsonSerializer.Serialize(new Dictionary { ["HomeAddress"] = "home" }, options) + .Should().Be("""{"homeAddress":"home"}"""); + } + + [TestCase("""{}""", false, "Original")] + [TestCase("""{"displayName":"Updated"}""", true, "Updated")] + public void MergePatch_PreservesUnpatchedDictionaryKeys(string patch, bool hasChanges, string displayName) + { + var target = new DictionaryDocument + { + DisplayName = "Original", + Values = new() { ["P1"] = "upper", ["p1"] = "lower", ["ETag"] = "key" } + }; + + var result = new CoreEx.Json.JsonMergePatch().Merge(patch, target).ThrowOnError().Value; + + result.HasChanges.Should().Be(hasChanges); + result.Merged.Should().NotBeNull(); + result.Merged!.DisplayName.Should().Be(displayName); + result.Merged.Values.Should().BeEquivalentTo(target.Values); + target.Values.Keys.Should().BeEquivalentTo("P1", "p1", "ETag"); + } + + public class DictionaryDocument + { + public string? DisplayName { get; set; } + public string? ETag { get; set; } + public Dictionary Values { get; set; } = []; + } +} diff --git a/tests/CoreEx.Test.Unit/Json/JsonMergePatchRfcTests.cs b/tests/CoreEx.Test.Unit/Json/JsonMergePatchRfcTests.cs new file mode 100644 index 000000000..fedca90e6 --- /dev/null +++ b/tests/CoreEx.Test.Unit/Json/JsonMergePatchRfcTests.cs @@ -0,0 +1,74 @@ +namespace CoreEx.Test.Unit.Json; + +[TestFixture] +public class JsonMergePatchRfcTests +{ + [TestCase("""{"Name":"old"}""", """{"name":"new"}""", """{"Name":"old","name":"new"}""", true)] + [TestCase("""{"Name":"old"}""", """{"name":null}""", """{"Name":"old"}""", false)] + [TestCase("""{"values":{"P1":1,"p1":2}}""", """{"values":{"P1":3}}""", """{"values":{"P1":3,"p1":2}}""", true)] + [TestCase("""{"values":{"P1":1,"p1":2}}""", """{"values":{"P1":null}}""", """{"values":{"p1":2}}""", true)] + [TestCase("""{"values":{"P1":1,"p1":2}}""", """{"values":{"P1":1}}""", """{"values":{"P1":1,"p1":2}}""", false)] + [TestCase("""{}""", """{"missing":null}""", """{}""", false)] + [TestCase("""{"values":{"P1":1}}""", """{"values":{"missing":null}}""", """{"values":{"P1":1}}""", false)] + [TestCase("""[1,2]""", """{"keep":3,"remove":null}""", """{"keep":3}""", true)] + [TestCase("""null""", """{"remove":null}""", """{}""", true)] + [TestCase("""42""", """{}""", """{}""", true)] + [TestCase("""{}""", """{"values":{"nested":{"remove":null}}}""", """{"values":{"nested":{}}}""", true)] + [TestCase("""{"values":42}""", """{"values":{"remove":null,"keep":3}}""", """{"values":{"keep":3}}""", true)] + [TestCase("""{"values":null}""", """{"values":{"remove":null}}""", """{"values":{}}""", true)] + [TestCase("""{"values":[1,2]}""", """{"values":{"remove":null}}""", """{"values":{}}""", true)] + [TestCase("""{"values":[1,2]}""", """{"values":[3]}""", """{"values":[3]}""", true)] + [TestCase("""{"values":[1,2]}""", """{"values":[1,2]}""", """{"values":[1,2]}""", false)] + [TestCase("""{"values":1}""", """null""", """null""", true)] + [TestCase("""{"values":1}""", "\"replacement\"", "\"replacement\"", true)] + [TestCase("""{"untouched":null}""", """{"added":1}""", """{"untouched":null,"added":1}""", true)] + [TestCase("""{"values":[{"Name":1}]}""", """{"values":[{"name":1}]}""", """{"values":[{"name":1}]}""", true)] + public void Merge_UsesRfcMemberIdentityAndRecursiveObjectSemantics(string target, string patch, string expected, bool hasChanges) + { + var merger = new CoreEx.Json.JsonMergePatch(); + var merged = JsonSerializer.Deserialize(merger.Merge(patch, target)); + AssertJson(merged, expected); + + merger.TryMerge(patch, target, out string? changed).Should().Be(hasChanges); + if (hasChanges) + AssertJson(JsonSerializer.Deserialize(changed!), expected); + else + changed.Should().BeNull(); + + var targetElement = JsonSerializer.Deserialize(target); + var patchElement = JsonSerializer.Deserialize(patch); + merger.TryMerge(patchElement, targetElement, out JsonElement result).Should().Be(hasChanges); + AssertJson(result, expected); + } + + [Test] + public void CaseInsensitiveMatching_RequiresExplicitCompatibilityOption() + { + var options = new CoreEx.Json.JsonMergePatchOptions { PropertyNameComparer = StringComparer.OrdinalIgnoreCase }; + var merger = new CoreEx.Json.JsonMergePatch(options); + + merger.Merge("""{"name":"new"}""", """{"Name":"old"}""").Should().Be("""{"name":"new"}"""); + } + + [TestCase("""{"values":{"P1":"updated"}}""", "updated")] + [TestCase("""{"values":{"P1":null}}""", null)] + public void TypedMerge_DistinguishesDictionaryKeysByCase(string patch, string? upperValue) + { + var target = new Dictionary> { ["values"] = new() { ["P1"] = "upper", ["p1"] = "lower" } }; + var result = new CoreEx.Json.JsonMergePatch().Merge(patch, target).ThrowOnError().Value; + + result.HasChanges.Should().BeTrue(); + result.Merged.Should().NotBeNull(); + var values = result.Merged!["values"]; + values["p1"].Should().Be("lower"); + if (upperValue is null) + values.Should().NotContainKey("P1"); + else + values["P1"].Should().Be(upperValue); + + target["values"]["P1"].Should().Be("upper"); + } + + private static void AssertJson(JsonElement actual, string expected) + => System.Text.Json.Nodes.JsonNode.DeepEquals(System.Text.Json.Nodes.JsonNode.Parse(actual.GetRawText()), System.Text.Json.Nodes.JsonNode.Parse(expected)).Should().BeTrue(); +} diff --git a/tests/CoreEx.Validation.Test.Unit/JsonPathParityTests.cs b/tests/CoreEx.Validation.Test.Unit/JsonPathParityTests.cs index 7571a0e0d..9ad6d5c82 100644 --- a/tests/CoreEx.Validation.Test.Unit/JsonPathParityTests.cs +++ b/tests/CoreEx.Validation.Test.Unit/JsonPathParityTests.cs @@ -89,7 +89,7 @@ public async Task InlineValueAndNonStringKeyPaths_PreserveExpectedBehavior() }); result.Messages.Should().NotBeNull(); - result.Messages?.Select(x => x.Property).Should().BeEquivalentTo("values.etag", "numericAddresses.13.street_name"); + result.Messages?.Select(x => x.Property).Should().BeEquivalentTo("values.ETag", "numericAddresses.13.street_name"); } [Test] From 7db10730521596fd6b69489ad15b348f33a9ca34 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Fri, 23 Oct 2026 06:28:03 -0700 Subject: [PATCH 24/25] Align sample and template health checks with preserved dictionary keys Update expected SQL Server, PostgreSQL and Redis health entry names to match their registered casing across sample and generated host tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- samples/tests/Contoso.Customers.Test.Api/HostTests.cs | 2 +- .../tests/Contoso.Orders.Test.Api/OtherTests.Health.cs | 6 +++--- samples/tests/Contoso.Products.Test.Api/HostTests.cs | 6 +++--- .../Contoso.Products.Test.Relay/OtherTests.Health.cs | 4 ++-- .../Contoso.Shopping.Test.Relay/OtherTests.Health.cs | 2 +- .../CoreEx.Api/tests/solution-name.Test.Api/HostTests.cs | 8 ++++---- .../tests/solution-name.Test.Relay/HostTests.cs | 6 +++--- .../tests/solution-name.Test.Subscribe/HostTests.cs | 8 ++++---- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/samples/tests/Contoso.Customers.Test.Api/HostTests.cs b/samples/tests/Contoso.Customers.Test.Api/HostTests.cs index 589b5560d..d3121fc00 100644 --- a/samples/tests/Contoso.Customers.Test.Api/HostTests.cs +++ b/samples/tests/Contoso.Customers.Test.Api/HostTests.cs @@ -53,7 +53,7 @@ public void Health(string path) public void Health_Detailed(string path, bool minimal) { // Liveness reflects the process, not downstream dependencies. - string[] paths = ["$.entries.reference-data-orchestrator", "$.entries.cosmos-database", "$.entries['stackExchange.Redis']"]; + string[] paths = ["$.entries.reference-data-orchestrator", "$.entries.cosmos-database", "$.entries['StackExchange.Redis']"]; var r = Test.Http() .Run(HttpMethod.Get, path) diff --git a/samples/tests/Contoso.Orders.Test.Api/OtherTests.Health.cs b/samples/tests/Contoso.Orders.Test.Api/OtherTests.Health.cs index 46178316c..cb6433b60 100644 --- a/samples/tests/Contoso.Orders.Test.Api/OtherTests.Health.cs +++ b/samples/tests/Contoso.Orders.Test.Api/OtherTests.Health.cs @@ -20,8 +20,8 @@ public void Health_Detailed(string path, bool minimal) string[] requiredServices = [ "reference-data-orchestrator", - "stackExchange.Redis", - "sqlServer" + "StackExchange.Redis", + "SqlServer" ]; var r = Test.Http() @@ -36,4 +36,4 @@ public void Health_Detailed(string path, bool minimal) else json.Should().ContainAll(requiredServices); } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.Products.Test.Api/HostTests.cs b/samples/tests/Contoso.Products.Test.Api/HostTests.cs index 78cbd53ad..098319dbc 100644 --- a/samples/tests/Contoso.Products.Test.Api/HostTests.cs +++ b/samples/tests/Contoso.Products.Test.Api/HostTests.cs @@ -55,8 +55,8 @@ public void Health_Detailed(string path, bool minimal) string[] _paths = [ "$.entries.reference-data-orchestrator", - "$.entries['stackExchange.Redis']", - "$.entries.postgreSql" + "$.entries['StackExchange.Redis']", + "$.entries.PostgreSql" ]; var r = Test.Http() @@ -71,4 +71,4 @@ public void Health_Detailed(string path, bool minimal) else json.ContainAll(_paths); } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.Products.Test.Relay/OtherTests.Health.cs b/samples/tests/Contoso.Products.Test.Relay/OtherTests.Health.cs index 1bf537550..b10071526 100644 --- a/samples/tests/Contoso.Products.Test.Relay/OtherTests.Health.cs +++ b/samples/tests/Contoso.Products.Test.Relay/OtherTests.Health.cs @@ -19,7 +19,7 @@ public void Health_Detailed(string path, bool minimal) { string[] _requiredServices = [ - "postgreSql", + "PostgreSql", "postgres-outbox-relay-00", "postgres-outbox-relay-01", "postgres-outbox-relay-02", @@ -38,4 +38,4 @@ public void Health_Detailed(string path, bool minimal) else json.Should().ContainAll(_requiredServices); } -} \ No newline at end of file +} diff --git a/samples/tests/Contoso.Shopping.Test.Relay/OtherTests.Health.cs b/samples/tests/Contoso.Shopping.Test.Relay/OtherTests.Health.cs index 3e783bfd0..4d5681055 100644 --- a/samples/tests/Contoso.Shopping.Test.Relay/OtherTests.Health.cs +++ b/samples/tests/Contoso.Shopping.Test.Relay/OtherTests.Health.cs @@ -19,7 +19,7 @@ public void Health_Detailed(string path, bool minimal) { string[] requiredServices = [ - "sqlServer", + "SqlServer", "sqlserver-outbox-relay-00", "sqlserver-outbox-relay-01", "sqlserver-outbox-relay-02", diff --git a/src/CoreEx.Template/content/CoreEx.Api/tests/solution-name.Test.Api/HostTests.cs b/src/CoreEx.Template/content/CoreEx.Api/tests/solution-name.Test.Api/HostTests.cs index e8231040b..9e1736b6e 100644 --- a/src/CoreEx.Template/content/CoreEx.Api/tests/solution-name.Test.Api/HostTests.cs +++ b/src/CoreEx.Template/content/CoreEx.Api/tests/solution-name.Test.Api/HostTests.cs @@ -63,13 +63,13 @@ public void Health_Detailed(string path, bool minimal) // #if refdata-enabled "$.entries.reference-data-orchestrator", // #endif - "$.entries['stackExchange.Redis']", + "$.entries['StackExchange.Redis']", // #if implement-sqlserver - "$.entries.sqlServer" + "$.entries.SqlServer" // #elif implement-cosmos "$.entries.cosmos-database" // #elif implement-postgres - "$.entries.postgreSql" + "$.entries.PostgreSql" // #endif ]; @@ -85,4 +85,4 @@ public void Health_Detailed(string path, bool minimal) else json.ContainAll(_paths); } -} \ No newline at end of file +} diff --git a/src/CoreEx.Template/content/CoreEx.Relay/tests/solution-name.Test.Relay/HostTests.cs b/src/CoreEx.Template/content/CoreEx.Relay/tests/solution-name.Test.Relay/HostTests.cs index 6a4455971..5e98836d3 100644 --- a/src/CoreEx.Template/content/CoreEx.Relay/tests/solution-name.Test.Relay/HostTests.cs +++ b/src/CoreEx.Template/content/CoreEx.Relay/tests/solution-name.Test.Relay/HostTests.cs @@ -32,7 +32,7 @@ public void Health_Detailed(string path, bool minimal) string[] _paths = [ // #if implement-sqlserver - "$.entries.sqlServer", + "$.entries.SqlServer", "$.entries.sqlserver-outbox-relay-00", "$.entries.sqlserver-outbox-relay-01", "$.entries.sqlserver-outbox-relay-02", @@ -46,7 +46,7 @@ public void Health_Detailed(string path, bool minimal) "$.entries.cosmos-outbox-relay-ref-data-03", // #endif // #elif implement-postgres - "$.entries.postgreSql", + "$.entries.PostgreSql", "$.entries.postgres-outbox-relay-00", "$.entries.postgres-outbox-relay-01", "$.entries.postgres-outbox-relay-02", @@ -102,4 +102,4 @@ public void HostedService_Pause_And_Resume() .Value.Should().BeOneOf("Running", "Sleeping"); } // #endif -} \ No newline at end of file +} diff --git a/src/CoreEx.Template/content/CoreEx.Subscribe/tests/solution-name.Test.Subscribe/HostTests.cs b/src/CoreEx.Template/content/CoreEx.Subscribe/tests/solution-name.Test.Subscribe/HostTests.cs index f0f880e95..bf7f95b31 100644 --- a/src/CoreEx.Template/content/CoreEx.Subscribe/tests/solution-name.Test.Subscribe/HostTests.cs +++ b/src/CoreEx.Template/content/CoreEx.Subscribe/tests/solution-name.Test.Subscribe/HostTests.cs @@ -38,13 +38,13 @@ public void Health_Detailed(string path, bool minimal) // #if refdata-enabled "$.entries.reference-data-orchestrator", // #endif - "$.entries['stackExchange.Redis']", + "$.entries['StackExchange.Redis']", // #if implement-sqlserver - "$.entries.sqlServer", + "$.entries.SqlServer", // #elif implement-cosmos "$.entries.cosmos-database", // #elif implement-postgres - "$.entries.postgreSql", + "$.entries.PostgreSql", // #endif // #if implement-servicebus "$.entries.hosted-subscriber-events" @@ -93,4 +93,4 @@ public void HostedService_Pause_And_Resume() .Run(HttpMethod.Get, $"/hosted-services/{Service}/status") .Value.Should().BeOneOf("Running", "Sleeping"); } -} \ No newline at end of file +} From 249df992962013aa99feb4f49c2d519606d624b9 Mon Sep 17 00:00:00 2001 From: Eric Sibly Date: Sat, 24 Oct 2026 10:13:52 -0700 Subject: [PATCH 25/25] Add Optional request values and refine validation plumbing Support presence-aware JSON, validation, reference-data generation and OpenAPI. Add extensible typed wrapper cleaning, preserve nullable validation state and normalization, and update guidance and release notes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Eric Sibly --- .github/copilot-instructions.md | 3 + .../coreex-contracts.instructions.md | 19 + .../coreex-conventions.instructions.md | 10 +- .../coreex-validators.instructions.md | 24 ++ .github/skills/coreex-contract/SKILL.md | 1 + .../coreex-contract/references/workflow.md | 2 + .github/skills/coreex-refdata/SKILL.md | 1 + .../coreex-refdata/references/workflow.md | 2 + .github/skills/coreex-validator/SKILL.md | 1 + .../coreex-validator/references/workflow.md | 6 + AGENTS.md | 3 + CHANGELOG.md | 4 + gen/CoreEx.Generator/ContractModel.cs | 23 +- gen/CoreEx.Generator/PropertyModel.cs | 15 + gen/CoreEx.Generator/Templates/Contract.cs.hb | 6 +- samples/docs/application-layer.md | 16 + samples/docs/contracts-layer.md | 13 + .../NSwagOpenApiOperationProcessor.cs | 60 ++- src/CoreEx.AspNetCore.NSwag/README.md | 1 + src/CoreEx.Validation/AGENTS.md | 15 + .../Abstractions/IPropertyContextNullState.cs | 12 + .../Abstractions/InlineValidator.cs | 8 +- src/CoreEx.Validation/CommonValidatorT.cs | 3 +- src/CoreEx.Validation/PropertyContext.cs | 66 +++- src/CoreEx.Validation/README.md | 39 +- src/CoreEx.Validation/Rules/EntityRule.cs | 8 +- src/CoreEx.Validation/Rules/SpecifiedRule.cs | 22 ++ src/CoreEx.Validation/Rules/UnwrapRule.cs | 34 ++ .../ValidationContext.Utility.cs | 14 +- .../ValidationExtensions.BetweenRule.cs | 14 +- .../ValidationExtensions.CommonRule.cs | 4 +- ...alidationExtensions.ComparePropertyRule.cs | 2 +- .../ValidationExtensions.CompareValueRule.cs | 30 +- .../ValidationExtensions.CompareValuesRule.cs | 6 +- .../ValidationExtensions.EntityRule.cs | 6 +- .../ValidationExtensions.Optional.cs | 28 ++ .../ValidationExtensions.SpecifiedRule.cs | 26 ++ .../ValidationExtensions.StringRule.cs | 18 +- .../ValidationExtensions.UnwrapRule.cs | 30 ++ src/CoreEx.Validation/ValidationExtensions.cs | 3 +- src/CoreEx/AGENTS.md | 2 + src/CoreEx/Abstractions/Internal.cs | 10 +- .../Entities/Abstractions/IOptionalValue.cs | 13 + .../Entities/Abstractions/IWrappedValue.cs | 13 + .../Abstractions/IWrappedValueCleaner.cs | 16 + src/CoreEx/Entities/Optional.cs | 66 ++++ src/CoreEx/Entities/README.md | 12 + src/CoreEx/Json/JsonDefaults.cs | 2 +- .../Json/OptionalJsonConverterFactory.cs | 37 ++ .../Metadata/PropertyRuntimeMetadata.cs | 4 + .../RuntimeMetadata.Clean.WrappedValue.cs | 49 +++ src/CoreEx/Metadata/RuntimeMetadata.Clean.cs | 13 + src/CoreEx/README.md | 7 + .../Controllers/OtherController.cs | 6 + .../Entities/OptionalPatchRequest.cs | 42 +++ .../OtherApiTests.cs | 46 +++ .../Entities/OptionalTests.cs | 105 ++++++ .../Entities/WrappedValueCleanerTests.cs | 94 +++++ .../Json/OptionalJsonConverterTests.cs | 48 +++ tests/CoreEx.Validation.Test.Unit/Helper.cs | 3 +- .../Rules/ChainRuleTests.cs | 44 +++ .../Rules/OptionalRuleTests.cs | 222 +++++++++++ .../Rules/UnwrapRuleTests.cs | 91 +++++ .../Rules/WrappedContextTests.cs | 349 ++++++++++++++++++ 64 files changed, 1805 insertions(+), 87 deletions(-) create mode 100644 src/CoreEx.Validation/Abstractions/IPropertyContextNullState.cs create mode 100644 src/CoreEx.Validation/Rules/SpecifiedRule.cs create mode 100644 src/CoreEx.Validation/Rules/UnwrapRule.cs create mode 100644 src/CoreEx.Validation/ValidationExtensions.Optional.cs create mode 100644 src/CoreEx.Validation/ValidationExtensions.SpecifiedRule.cs create mode 100644 src/CoreEx.Validation/ValidationExtensions.UnwrapRule.cs create mode 100644 src/CoreEx/Entities/Abstractions/IOptionalValue.cs create mode 100644 src/CoreEx/Entities/Abstractions/IWrappedValue.cs create mode 100644 src/CoreEx/Entities/Abstractions/IWrappedValueCleaner.cs create mode 100644 src/CoreEx/Entities/Optional.cs create mode 100644 src/CoreEx/Json/OptionalJsonConverterFactory.cs create mode 100644 src/CoreEx/Metadata/RuntimeMetadata.Clean.WrappedValue.cs create mode 100644 tests/CoreEx.AspNetCore.Test.Api/Entities/OptionalPatchRequest.cs create mode 100644 tests/CoreEx.Test.Unit/Entities/OptionalTests.cs create mode 100644 tests/CoreEx.Test.Unit/Entities/WrappedValueCleanerTests.cs create mode 100644 tests/CoreEx.Test.Unit/Json/OptionalJsonConverterTests.cs create mode 100644 tests/CoreEx.Validation.Test.Unit/Rules/ChainRuleTests.cs create mode 100644 tests/CoreEx.Validation.Test.Unit/Rules/OptionalRuleTests.cs create mode 100644 tests/CoreEx.Validation.Test.Unit/Rules/UnwrapRuleTests.cs create mode 100644 tests/CoreEx.Validation.Test.Unit/Rules/WrappedContextTests.cs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index daa58fd46..c473a8957 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -88,6 +88,7 @@ Connection strings for each service in development are in each host's `appsettin - Mutable contracts often implement `IIdentifier`, `IETag`, and `IChangeLog`. - Use `[NonNullable]` for properties that are mandatory/required but declared nullable (`string?`, `int?`) for DTO flexibility, so OpenAPI reports them as non-nullable. - Use `[ReadOnly(true)]` for server-managed fields and `[ReferenceData]` for reference-data-backed code properties. +- Use `Optional` only on explicit patch/input DTOs when omission must differ from supplied `null` or default values; validate with `.Unwrap()` and map only specified values into ordinary application/domain objects. Do not leak wrappers into response/event or persistence models. - Canonical casing transformations belong in property setters when already established by the model (for example `Sku` uppercasing in `ProductBase`). - Favor the existing source-generation approach; do not hand-write members that are meant to be generated. @@ -161,6 +162,8 @@ Connection strings for each service in development are in each host's `appsettin ### House Rules - Code comments end with a period/full stop. +- Use `///` XML documentation for types and members, including private nested types, methods, and properties; use `` for implementations and overrides. Private fields and test fixtures/helpers retain their existing exemptions. +- Do not hard-wrap prose in READMEs, AGENTS files, skills, instructions, or other Markdown documents for visual formatting. Keep each paragraph on one physical line; use editor soft wrapping, LF line endings, and line breaks only for Markdown structure or code blocks. - Always use `.ConfigureAwait(false)` in service/repository code. - `enable` and `enable` are set in `Directory.Build.props` — treat nullable warnings as errors, never suppress them with `!` without justification. - Every project has a single `GlobalUsing.cs` at the project root. All `using` statements go there — never in individual source files. The code generator (`*.CodeGen`) emits no `using` statements and depends on this. diff --git a/.github/instructions/coreex-contracts.instructions.md b/.github/instructions/coreex-contracts.instructions.md index a2722c0f0..eb7dca52f 100644 --- a/.github/instructions/coreex-contracts.instructions.md +++ b/.github/instructions/coreex-contracts.instructions.md @@ -395,6 +395,25 @@ public partial class BookingTravellerNotesPatch : IETag } ``` +### Optional patch properties + +Use `Optional` only on a dedicated request/patch contract when omitted, explicit `null`, and supplied default values must remain distinct. A default wrapper is unspecified; assignment or `Optional.Specified(value)` marks it present. Do not combine `required` with `Optional`: presence is already represented by the wrapper, and the generator reports `CoreEx013` for that contradictory declaration. Do not use `[NonNullable]` to express presence; it describes only the inner value's OpenAPI nullability. + +```csharp +[Contract] +public partial class TravellerNotesPatch +{ + /// Gets or sets the optionally supplied notes. + public Optional Notes { get; set; } +} +``` + +The converter uses the inner value's scalar/object/collection JSON shape. Missing JSON properties remain unspecified, explicit `null` becomes specified-null for nullable inner types, and supplied defaults such as `0` or `false` remain specified. CoreEx's `JsonDefaults` omits the default wrapper when writing; custom serializer options must also ignore default values. OpenAPI unwraps the inner type, marks the property non-required, and preserves the inner nullability. + +For reference data, annotate the optional `*Code` property with `[ReferenceData]` and declare it `partial Optional`; the generated typed navigation accessor is also optional and shares the code property's specified state. Preserve the scalar code wire contract. + +Validate the request DTO with `.Unwrap()` and map only specified values into an ordinary contract, aggregate, or model. Do not put `Optional` on shared response/event contracts or in repository and persistence models. See [optional validation guidance](./coreex-validators.instructions.md#optional-patch-validation). + ## Reference Data Contracts Reference data contracts are **generated, not hand-authored**. The source of truth is the `entities:` section of `ref-data.yaml` in the domain's `*.CodeGen` project. Running the CodeGen generates all artefacts across every layer -- contract class, API endpoint, service method, repository interface, repository implementation, and mapper -- as `.g.cs` files that must never be edited directly. diff --git a/.github/instructions/coreex-conventions.instructions.md b/.github/instructions/coreex-conventions.instructions.md index a1dbbaa40..6792c0a99 100644 --- a/.github/instructions/coreex-conventions.instructions.md +++ b/.github/instructions/coreex-conventions.instructions.md @@ -179,13 +179,13 @@ public async Task CreateAsync(Employee value, CancellationToken cancel ## XML Documentation Comments -Document the public surface with XML doc comments, and **never duplicate** a description that an interface already provides: +Document types and members with `///` XML doc comments regardless of accessibility, and **never duplicate** a description that an interface already provides: - **Interfaces** — give **every** member (method, property, event) a `` describing the operation. Document parameters/returns (``, ``) where it adds clarity. - **Contract / DTO properties** — every property on a contract (including `[Contract]` partial classes) gets a ``. (Standard `Id`/`ETag`/`ChangeLog` members that implement `IIdentifier`/`IETag`/`IChangeLog` may use ``.) - **Implementing / overriding members** — on the concrete class member that implements an interface member (or overrides a base member), use **``** rather than repeating the summary. - **Public / protected / internal types and members** — a type or member at these access levels that is **not** an interface implementation/override gets its own `` (classes, standalone methods, properties, constructors, etc.), plus ``/`` where applicable. -- **Private methods** (including local functions) — still get a `` describing what they do; ``/`` are not required. Private fields are exempt. +- **Private types and members** — private nested types, methods (including local functions), and properties still get a `` describing what they do; use `` for implementations and overrides. ``/`` are not required for private methods. Private fields are exempt. - **Test projects (`*.Test*`)** — test fixture classes, `[Test]`/`[TestCase]` methods, and private test helpers/fixtures are exempt from this doc requirement (established convention across the repo's test suites). > **Do not invert this** (a common mistake): the `` goes on the **interface** member and on **contract properties**; the concrete class member that *implements* an interface gets **``**, not a fresh summary. Leaving interfaces or contract properties undocumented while summarising the implementing class is **backwards** — fix it the right way round. @@ -245,6 +245,10 @@ private readonly IUnitOfWork _unitOfWork; private readonly ILogger _logger; ``` +## Markdown Prose + +Do not hard-wrap prose in READMEs, AGENTS files, skills, instructions, or other Markdown documents to meet a visual line width. Keep each paragraph on one physical line and let the editor soft-wrap it. Use LF line endings; retain line breaks required for Markdown structure (headings, lists, tables, and paragraph boundaries) and code blocks. + ## End of File Every file ends with **exactly one** line terminator — the closing `}` (or last line of content) followed by a single newline. No blank lines after it, and no trailing whitespace. This applies to every file you create or edit (`.cs`, `.json`, `.yaml`, `.md`, `.csproj`, `.sql`, etc.), including generated-then-hand-tweaked output and template content. @@ -270,5 +274,5 @@ Create/edit tooling frequently appends extra `\n`s (particularly when `new_str` - Do not replace a private backing field with an auto-property simply because it could be one — backing fields are a valid developer choice. - Do not leave interface members or contract properties undocumented — each gets a ``. - Do not invert the doc convention — summaries go on **interfaces and contract properties**; the **implementing** class member gets `` (not a fresh summary). Summarising the concrete class while leaving the interface/contract undocumented is backwards. -- Do not leave a private method undocumented — it still needs a `` (params/returns optional), unless it's a test helper in a `*.Test*` project. +- Do not leave private nested types, methods, or properties undocumented — use `/// ` (params/returns optional for private methods), or `/// ` for implementations and overrides, unless covered by the test fixture/helper exemption. - Do not invent a plausible-sounding CoreEx member name when unsure (`MapToEntity`, `OnMapToPrimary`/`OnMapToSecondary`, etc.) — check the relevant instructions file, then just build (a compile error is authoritative), before ever searching GitHub blind. diff --git a/.github/instructions/coreex-validators.instructions.md b/.github/instructions/coreex-validators.instructions.md index dcf0a074a..df251715b 100644 --- a/.github/instructions/coreex-validators.instructions.md +++ b/.github/instructions/coreex-validators.instructions.md @@ -150,6 +150,7 @@ The table above lists the most common rules; the full set of fluent extension me |---|---| | Presence (required) | `Mandatory()`, `NotNull()`, `NotEmpty()` | | Absence (must be unset) | `Null()`, `None()`, `Empty()` | +| Optional patch input | `Unwrap()` validates the inner value only when specified; `Specified()` requires outer presence | | Strings | `MaximumLength(n)`, `MinimumLength(n)`, `Length(exact)`, `String(maxLength)`, `String(min, max, regex)`, `Matches(regex)`, `Wildcard()`, `Email()` / `Email(maxLength)` | | Numbers / decimals | `Numeric(allowNegatives)`, `Positive()`, `Decimal(precision, scale)`, `PrecisionScale(precision, scale)` | | Comparisons (value) | `Equal(v)`, `NotEqual(v)`, `LessThan(v)`, `LessThanOrEqualTo(v)`, `GreaterThan(v)`, `GreaterThanOrEqualTo(v)`, `Compare(op, v)` — each with a delegate overload for runtime values (see below) | @@ -166,6 +167,29 @@ The table above lists the most common rules; the full set of fluent extension me > - **Do not use `Mandatory()` on a non-nullable value type where the default is a legitimate value** (e.g. a `decimal Salary` where `0` is allowed) — it will wrongly reject `0`. Use a range rule instead, e.g. `GreaterThanOrEqualTo(0)` (allows `0`) or `GreaterThan(0)` (requires positive). To require presence *without* the default-check, pass `Mandatory(mustNotBeDefault: false)`. > - **The expected unit-test error follows the same rule:** a `Mandatory()` on a non-nullable `decimal`/`DateOnly` produces the *required* error for `0`/`MinValue`, so a test arranging that value must expect the mandatory error (not a pass). +## Optional patch validation + +Unwrapping is interface-based: custom `IWrappedValue` wrappers validate their inner value, while `IOptionalValue` wrappers skip the inner chain when unspecified, before `.Value` is read. Both interfaces live in `CoreEx.Entities.Abstractions`. `Optional` and reference-type wrappers support `.Unwrap()` with inferred types; custom struct wrappers require the static helper `ValidationExtensions.Unwrap(rule)` because C# cannot infer the inner type from a generic constraint. Only wrapper-specific receivers are exposed as extensions; the explicitly typed helpers retain interface constraints without appearing as extensions on unrelated property rules. + +On a patch request property typed `Optional`, `.Unwrap()` lets the existing rules validate its inner value while skipping the entire chain when the field was omitted. A specified `null` or default value is not omitted and follows the inner type's ordinary rule behavior. `.Specified()` is a separate outer-property rule that requires presence but permits specified-null. + +```csharp +public class TravellerNotesPatchValidator : Validator +{ + public TravellerNotesPatchValidator() + { + Property(x => x.Notes).Unwrap().Mandatory(); + Property(x => x.Category).Specified(); + } +} +``` + +Use `.Unwrap().Collection(...)` for an optional collection. Only root `Optional` properties are unwrapped; optional collection elements and dictionary keys/values are not supported. Keep optional wrappers in request DTOs and map only specified values into normal application/domain objects. Reference-data patch contracts validate the generated optional navigation property after unwrapping, not the code string. + +`.Unwrap().Entity(...)` validates a supplied child entity using its nested error paths. Normalizing rules (for example, `CompareValues(..., overrideValueWhereMatched: true)`) write back to writable `Optional` properties, and subsequent rules see the normalized value. Custom wrappers expose a read-only inner-value contract; do not assume they can be reconstructed or normalized. + +`CommonValidator` preserves the caller's label, formatting and originating null state, including when composed after `.Unwrap()`. A `CommonValidator` can validate `int?` without treating null as zero: range rules skip null; `.NotNull()` rejects null but allows zero; default `.Mandatory()` rejects both null and zero. Normalization respects property writability and is visible to later rules and `AdditionalAsync` callbacks. + ### Comparisons Prefer the dedicated comparison rules — `.GreaterThanOrEqualTo(value)`, `.LessThanOrEqualTo(value)`, `.GreaterThan(value)`, `.LessThan(value)`, `.Equal(value)`. For the general form use the **`.Compare(...)`** extension with a `CompareOperator` value: diff --git a/.github/skills/coreex-contract/SKILL.md b/.github/skills/coreex-contract/SKILL.md index 7cedd0635..2477c13d3 100644 --- a/.github/skills/coreex-contract/SKILL.md +++ b/.github/skills/coreex-contract/SKILL.md @@ -51,6 +51,7 @@ Guides you through creating or modifying a hand-authored contract (DTO/entity) i - Every property needs a `` XML doc comment - Same contract type for both API response and event payload — never split them - Patch/subset contracts (merge-patch, partial-update requests): standalone class with only the needed fields (+ `IETag`) — never inherit a sibling and suppress the rest; inheritance is for genuine supersets only +- For patch fields that distinguish omission from explicit `null` or a supplied default, use `Optional` on the request DTO; do not use it on response/event or persistence contracts For full workflow, decision trees, and code examples see [`references/workflow.md`](references/workflow.md). diff --git a/.github/skills/coreex-contract/references/workflow.md b/.github/skills/coreex-contract/references/workflow.md index 121bdcb82..8a0a83110 100644 --- a/.github/skills/coreex-contract/references/workflow.md +++ b/.github/skills/coreex-contract/references/workflow.md @@ -166,6 +166,8 @@ public partial class ProductLite : ProductBase **A5 applies to genuine supersets only** — every inherited property must stay meaningful on the derived type. For a patch/merge-patch or other partial-update request that needs only a *subset* of a sibling contract's fields, do **not** inherit it and suppress the rest (`new`, `[JsonIgnore]`, ignoring members): author a standalone `[Contract]` class with just the properties it needs (+ `IETag` where concurrency applies). Overlapping field names alone are not a reason to extract a base class or inherit. Path B (request contracts) is the right home for these. +When the patch must distinguish an omitted field from explicit `null` or a supplied default, declare that field as `Optional` on this standalone request contract. Do not mark it C# `required`, and do not carry the wrapper into shared response/event or persistence models. Validate via `.Unwrap()` and map only specified values into the ordinary target. See the [Optional patch property guidance](/.github/instructions/coreex-contracts.instructions.md#optional-patch-properties). + --- ## Path B — New Subordinate or Request Contract diff --git a/.github/skills/coreex-refdata/SKILL.md b/.github/skills/coreex-refdata/SKILL.md index bfccf0ac0..f6ca86ff6 100644 --- a/.github/skills/coreex-refdata/SKILL.md +++ b/.github/skills/coreex-refdata/SKILL.md @@ -36,6 +36,7 @@ Guides you through the complete end-to-end workflow for adding or modifying a re ## Quick Reference - Consume codes through [generated implicit conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion) to concrete reference-data types, not custom `RefData` helpers. Contracts use annotated partial `XxxCode` properties, not plain code auto-properties; conversion does not validate requiredness, validity or activity. +- For a patch DTO that must distinguish an omitted code from an explicit clear, use an annotated `partial Optional XxxCode` property; the generator supplies a presence-sharing optional navigation accessor. Keep this wrapper out of persisted/reference-data entities. > **Resolve project-wide choices from state before asking.** Read the solution-root `AGENTS.md` > **Feature Configuration** for `refdata-enabled` (gates this whole skill) and `data-provider` diff --git a/.github/skills/coreex-refdata/references/workflow.md b/.github/skills/coreex-refdata/references/workflow.md index 3c2bab10a..cb0653dc4 100644 --- a/.github/skills/coreex-refdata/references/workflow.md +++ b/.github/skills/coreex-refdata/references/workflow.md @@ -244,6 +244,8 @@ public partial string? BrandCode { get; set; } 3. Rebuild — the Roslyn generator emits the typed `Brand` navigation property automatically. Do not hand-author it. 4. Set `BrandCode` when mapping/arranging DTOs and use `Brand` for typed behavior. For a concrete typed destination, use `Contracts.Brand? brand = source.BrandCode`; no helper or null guard is needed. Follow [Reference-Data Code Conversion](/.github/instructions/coreex-contracts.instructions.md#reference-data-code-conversion) and validate separately. +For partial-update request DTOs that need to distinguish omission from an explicit clear, declare the annotated code property as `partial Optional BrandCode`. The source generator emits an optional typed `Brand` navigation property sharing the code property's presence. Validate supplied codes through that navigation after unwrapping; map only specified codes into the ordinary target contract. Do not use `Optional` on the persisted reference-data entity. + --- ## Phase 3 — CoreEx CodeGen (edit ref-data.yaml + generate) diff --git a/.github/skills/coreex-validator/SKILL.md b/.github/skills/coreex-validator/SKILL.md index a225869c2..ae8147675 100644 --- a/.github/skills/coreex-validator/SKILL.md +++ b/.github/skills/coreex-validator/SKILL.md @@ -52,6 +52,7 @@ Guides you through creating or modifying a CoreEx validator (`Application/Valida **Key rules at a glance:** - Request/input validation uses field-level validation errors; aggregate/entity/value-object business invariants use business errors with domain messages/codes. Keep both checks; never derive request paths in Domain or blanket-convert business errors. See [domain invariant conventions](/.github/instructions/coreex-domain.instructions.md#invariant-failures-versus-request-validation). - Ref-data: `.IsValid()` on the **navigation property** (`Gender`), never on `*Code` string +- Optional patch input: `.Unwrap()` validates the inner value only when specified; `.Specified()` requires presence while allowing specified-null - `Mandatory()` on a non-nullable value type errors on `default` (`0`, `MinValue`) — use a range rule if `0` is valid - Runtime-computed thresholds: use delegate overloads (`.LessThanOrEqualTo(_ => ...)`) — prefer over `OnValidateAsync` imperative logic - `context.HasErrors` guard before async I/O; `context.HasError(x => x.Prop)` for per-property guards diff --git a/.github/skills/coreex-validator/references/workflow.md b/.github/skills/coreex-validator/references/workflow.md index b60a16858..f1c1b2f3b 100644 --- a/.github/skills/coreex-validator/references/workflow.md +++ b/.github/skills/coreex-validator/references/workflow.md @@ -60,6 +60,8 @@ public class {Name}Validator : Validator | Required string | `.Mandatory().MaximumLength(n)` | | Required ref-data | `.Mandatory().IsValid()` | | Optional ref-data (valid if supplied) | `.IsValid()` | +| Optional patch value (validate only when present) | `.Unwrap().{existing rules}` | +| Require a patch field to be present | `.Specified()` | | Non-negative decimal | `.GreaterThanOrEqualTo(0, _ => "zero")` | | Decimal precision | `.PrecisionScale(precision, scale)` — pass `null` for precision to skip | | Required + precision | `.PrecisionScale(null, 2).GreaterThanOrEqualTo(0, _ => "zero")` | @@ -70,6 +72,10 @@ public class {Name}Validator : Validator | Compare to another property on the same entity | `.CompareProperty(CompareOperator.GreaterThanOrEqualTo, x => x.StartsOn)` — compares only when both have a value and are valid; add `.Mandatory()` first to require it | | Runtime-computed threshold | `.LessThanOrEqualTo(_ => DateOnly.FromDateTime(Runtime.UtcNow.UtcDateTime.AddYears(-16)), _ => "the minimum age of 16")` | +For patch inputs, exercise omitted, specified-null (where applicable), supplied default and valid/invalid supplied values. Use `.Unwrap().Entity(...)` or `.Unwrap().Collection(...)` for supplied complex values; assert the nested error path. If a rule normalizes an input, assert the value written back into the writable `Optional` property and that a subsequent rule observes it; custom read-only wrapper contracts do not imply write-back support. + +When reusing a `CommonValidator`, keep `T` non-nullable for reuse across nullable/non-nullable properties. Check null versus zero explicitly for numeric presence requirements: `.NotNull()` permits zero, default `.Mandatory()` does not. Shared rules preserve caller labels/formatting and normalization across chained rules and `AdditionalAsync` callbacks; cover these interactions when the application relies on them. + When a property's rules are conditional on the dependent property's value, use the predicate overload instead of repeating the dependency/null condition in `WhenEntity`: ```csharp diff --git a/AGENTS.md b/AGENTS.md index 82524cd53..9481bbf9f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,10 +67,13 @@ Never create or edit `*.g.cs`, `*.g.sql`, or `*.g.pgsql` files directly. Each ge Rules that are easy to violate and cause real breakage or wrong choices: +- **Markdown prose** — keep each paragraph on one physical line; do not insert hard line breaks mid-sentence for visual formatting. Use editor soft wrapping and LF line endings; preserve structural line breaks and code blocks. +- **XML documentation** — document types and members with `///`, including private nested types, methods, and properties; use `` for implementations and overrides. Private fields and test fixtures/helpers retain their existing exemptions. - **`GlobalUsings.cs`** — every project has a single `GlobalUsings.cs` at the project root; all `using` statements go there, never in individual source files. The Roslyn code generator emits no `using` statements and depends on this. - **`AwesomeAssertions` not FluentAssertions** — tests use the `AwesomeAssertions` NuGet package. Do not reach for FluentAssertions. - **Polyglot data** — Products uses PostgreSQL (`CoreEx.Database.Postgres`); Shopping uses SQL Server (`CoreEx.Database.SqlServer`); Customers uses Azure Cosmos DB (`CoreEx.Cosmos`). Do not assume SQL Server when working on Products, do not assume EF Core/DbEx in Cosmos domains, and do not mix outbox/publisher helpers across domains. - **No AutoMapper** — do not introduce AutoMapper. All mapping is explicit via `Mapper<>` (application layer) or `BiDirectionMapper<>` (infrastructure layer). +- **Optional patch inputs** — use `Optional` only on explicit request/patch DTOs when omission must differ from a supplied `null` or default value. Validate with `.Unwrap()` and map only specified values into ordinary application/domain objects; do not leak wrappers into response/event or persistence models. - **Aspire external infrastructure** — use `CoreEx.UnitTesting`'s `AddExternalConnectionString` extension (`UnitTestEx` namespace) for visible graph nodes, preserving connection names and Compose lifecycle ownership. Plain `AddConnectionString(name)` creates parameter-only nodes excluded from Aspire 13.5.4's Graph/Table views. Cosmos's configured `AccountEndpoint` is exposed as a credential-free resource URL for trace matching. Node status reflects configuration resolution, not connectivity; hosts own readiness probes. See [Aspire guidance](./samples/docs/aspire.md). - **`.ConfigureAwait(false)`** — always use it in service and repository code. - **File-scoped namespaces** — `namespace Foo.Bar;` only; never block-scoped `namespace Foo.Bar { }`. diff --git a/CHANGELOG.md b/CHANGELOG.md index d4df22850..66794a115 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ Represents the **NuGet** versions. ## v4.2.0 +- *Enhancement:* Added `Optional` for explicit request/patch DTOs to distinguish omitted fields from supplied null/default values, with JSON conversion, contract cleaning and generated reference-data support, and inner-type OpenAPI schemas. Validation uses `.Unwrap()` for supplied values and `.Specified()` to require presence; only specified fields should be mapped into ordinary application/domain models. Custom value wrappers (optional or always present) can independently opt into runtime cleaning through the public `IWrappedValueCleaner` capability, with cached constrained dispatch avoiding wrapper boxing on typed cleaning paths. - *Enhancement:* Added expression-based `MessageItem.CreateErrorMessage` factories for aggregating field errors, with inferred friendly invalid messages and optional JSON naming options. Validation exception factories share this message construction; policy guidance now distinguishes known request-contract expressions from explicit paths in reusable policies. - *Enhancement:* Added single-use fluent HTTP request builders with `Get`/`Post`/`Put`/`Patch`/`Delete` shortcuts, `WithJson` for JSON bodies and `WithContent` for explicit HTTP content (also available on `HttpRequestMessage`). Exactly one body assignment is permitted; repeated execution, configuration after execution starts, and conflicting body assignments throw `InvalidOperationException`. PATCH does not imply JSON Merge Patch. - *Enhancement:* Added `WithStatusResult` / `WithErrorStatus` to HTTP request builders, including validated builders, to map selected response statuses to CoreEx results before normal deserialization and validation. `ValidatedHttpRequestBuilder` integrates response validation with fluent request execution and automatic response disposal. @@ -26,6 +27,9 @@ Represents the **NuGet** versions. - *Maintenance:* Pinned local development, CI and package publishing to the stable .NET SDK in `global.json`; updated `nuget-publish.ps1` to build before packing and remove matching package versions from the local NuGet cache by default. - *Documentation:* Aligned application/API instructions, policy/service/adapter skills, and package/sample guidance on missing target resources (404) versus invalid request-body references (field-level 400), including query filters, expected downstream lookup outcomes, and deliberate endpoint semantics. - *Fixed:* Named/typed HTTP client registration now binds standard resilience settings from the nested `Resilience` configuration section rather than the parent client section. +- *Fixed:* Common validators now preserve nullable null state instead of treating null as zero, retain caller labels/value formatting, and propagate normalized values through subsequent rules and additional validation callbacks. Unwrapped validation uses inner values for messages and nested entity paths, and writes normalization back to writable `Optional` properties. +- *Fixed:* String, comparison, entity and common validation extension receivers now accept nullable reference inputs without nullable-annotation warnings, preserving existing fluent return types and runtime null behavior. +- *Fixed:* Exact-type internal casts now take their fast path before null/pattern checks, avoiding unnecessary value-type boxing in typed metadata reads. Common validation tracks null state separately from typed values rather than boxing values to represent null. - *Fixed:* JSON Merge Patch now recursively processes object patches for new members and non-object targets, removing null-valued patch members rather than copying them literally. Removing a nonexistent member no longer reports a change. - *Fixed:* Dictionary validation JSON paths now apply the effective serializer's `DictionaryKeyPolicy` to string keys for both key and value errors, matching serialized JSON and expression-based exception factories. Original keys remain unchanged in CLR paths and `GetDictionaryKey()`; JSON error paths preserve keys by default and use transformed casing only when a dictionary key policy is explicitly configured. - *Fixed:* Shared cached property metadata lookups now consistently prefer static `IRuntimeMetadata` providers, including explicit implementations, with reflection as the fallback for other types. JSON path/friendly-label resolution respects custom/generated metadata for both root and nested types regardless of cache-population order. Explicit expression lookups separately cache reflected metadata for properties omitted by the static provider (including generated reference-data navigation properties), preserving validation support without changing contract equality/copy/clean property collections. diff --git a/gen/CoreEx.Generator/ContractModel.cs b/gen/CoreEx.Generator/ContractModel.cs index 2700df6f8..9eb3eb813 100644 --- a/gen/CoreEx.Generator/ContractModel.cs +++ b/gen/CoreEx.Generator/ContractModel.cs @@ -156,6 +156,7 @@ private static ContractModel CreateForStandard(GeneratorAttributeSyntaxContext c } // Get the list of properties which as a minimum do a get. + var optionalTypeSymbol = context.SemanticModel.Compilation.GetTypeByMetadataName("CoreEx.Entities.Optional`1"); foreach (var p in symbol.GetMembers().OfType().Where(p => p.GetMethod is not null)) { if (p.GetAttributes().FirstOrDefault(a => a.AttributeClass?.ToDisplayString() == "CoreEx.Entities.ContractIgnoreAttribute") is not null) @@ -175,6 +176,20 @@ private static ContractModel CreateForStandard(GeneratorAttributeSyntaxContext c Format = GetDisplayFormatAttributeDataFormatString(p) }; + emp.IsOptional = p.Type is INamedTypeSymbol optionalType && SymbolEqualityComparer.Default.Equals(optionalType.OriginalDefinition, optionalTypeSymbol); + if (emp.IsOptional && emp.IsRequired) + { + var descriptor = new DiagnosticDescriptor( + id: "CoreEx013", + title: "Required Optional property is contradictory.", + messageFormat: "Contract property '{0}' cannot be declared 'required' because Optional already represents whether a value was specified.", + category: "CoreEx", + defaultSeverity: DiagnosticSeverity.Error, + isEnabledByDefault: true); + + model.Diagnostics.Add(Diagnostic.Create(descriptor, p.Locations.FirstOrDefault(), p.Name)); + } + if (model.IsRecord && emp.Name == "EqualityContract") continue; @@ -484,9 +499,13 @@ internal static void ManageReferenceDataAttributeProperty(IPropertySymbol proper return; model.IsRefData = true; - model.RefDataType = FormatTypeWithNullability(att.AttributeClass.TypeArguments.FirstOrDefault()?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)!, propertySymbol.NullableAnnotation); + var optionalType = propertySymbol.Type as INamedTypeSymbol; + var innerType = model.IsOptional && optionalType is not null ? optionalType.TypeArguments[0] : propertySymbol.Type; + model.IsOptionalRefData = model.IsOptional; + model.OptionalRefDataCodeType = innerType.ToDisplayString(ContractGenerator.FullyQualifiedWithNullability); + model.RefDataType = FormatTypeWithNullability(att.AttributeClass.TypeArguments.FirstOrDefault()?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)!, innerType.NullableAnnotation); - if (propertySymbol.Type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat) != "string" && propertySymbol.Type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat) != "string?") + if (innerType.SpecialType != SpecialType.System_String) { var descriptor = new DiagnosticDescriptor( id: "CoreEx001", diff --git a/gen/CoreEx.Generator/PropertyModel.cs b/gen/CoreEx.Generator/PropertyModel.cs index b32d7929b..29c75809a 100644 --- a/gen/CoreEx.Generator/PropertyModel.cs +++ b/gen/CoreEx.Generator/PropertyModel.cs @@ -52,6 +52,11 @@ internal class PropertyModel /// public bool IsRequired { get; set; } + /// + /// Indicates whether the property is an Optional<T>. + /// + public bool IsOptional { get; set; } + /// /// Indicates whether the property has a . /// @@ -127,6 +132,16 @@ internal class PropertyModel /// public string? RefDataType { get; set; } + /// + /// Indicates whether the reference-data code property is an Optional<T>. + /// + public bool IsOptionalRefData { get; set; } + + /// + /// Gets or sets the inner type of an optional reference-data code. + /// + public string? OptionalRefDataCodeType { get; set; } + /// /// Indicates whether the json serialization attribute is required. /// diff --git a/gen/CoreEx.Generator/Templates/Contract.cs.hb b/gen/CoreEx.Generator/Templates/Contract.cs.hb index 9d2a2b74c..83326e3e6 100644 --- a/gen/CoreEx.Generator/Templates/Contract.cs.hb +++ b/gen/CoreEx.Generator/Templates/Contract.cs.hb @@ -64,7 +64,11 @@ namespace {{Namespace}}; {{#if HasText}} {{indent}} [global::CoreEx.Localization.Localization("{{KeyAndOrText}}"{{#if HasFallbackText}}, "{{FallbackText}}"{{/if}})] {{/if}} + {{#if IsOptionalRefData}} +{{indent}} public global::CoreEx.Entities.Optional<{{RefDataType}}> {{RefDataName}} { get => {{Name}}.IsSpecified ? global::CoreEx.Entities.Optional<{{RefDataType}}>.Specified(({{RefDataType}}){{Name}}.Value) : default; set => {{Name}} = value.IsSpecified ? global::CoreEx.Entities.Optional<{{OptionalRefDataCodeType}}>.Specified(value.Value) : default; } + {{else}} {{indent}} public {{RefDataType}} {{RefDataName}} { get => ({{RefDataType}}){{Name}}; {{#if IsSettable}}set => {{Name}} = value; {{/if}} } + {{/if}} {{#if IsRefDataText}} {{indent}} /// Gets the related text where explicitly requested. @@ -74,7 +78,7 @@ namespace {{Namespace}}; {{#if HasRefDataTextJsonName}} {{indent}} [global::System.Text.Json.Serialization.JsonPropertyName("{{RefDataTextJsonName}}")] {{/if}} -{{indent}} public string? {{RefDataName}}Text { get => field ?? global::CoreEx.ExecutionContext.GetRelatedText(() => {{Name}} is null ? null : {{RefDataName}}?.Text); init => field = value; } +{{indent}} public string? {{RefDataName}}Text { get => field ?? global::CoreEx.ExecutionContext.GetRelatedText(() => {{#if IsOptionalRefData}}{{Name}}.IsSpecified && {{Name}}.Value is not null ? {{RefDataName}}.Value?.Text : null{{else}}{{Name}} is null ? null : {{RefDataName}}?.Text{{/if}}); init => field = value; } {{/if}} {{else}} {{#if IsRefDataCodeCollection}} diff --git a/samples/docs/application-layer.md b/samples/docs/application-layer.md index b37613706..9898d1285 100644 --- a/samples/docs/application-layer.md +++ b/samples/docs/application-layer.md @@ -84,6 +84,22 @@ Validators live in `Application/Validators/` and extend CoreEx's `Validator`. The two phases integrate naturally: `OnValidateAsync` should guard with `if (context.HasErrors) return;` to fail fast and avoid unnecessary I/O when declarative rules have already failed. +For a patch contract using `Optional`, `.Unwrap()` applies ordinary rules only when a field was supplied, and `.Specified()` can require field presence independently. A specified null/default is still supplied and is handled by the inner rules: + +```csharp +public class TravellerNotesPatchValidator : Validator +{ + public TravellerNotesPatchValidator() => Property(x => x.Notes).Unwrap().Mandatory(); +} +``` + +After validation, map the patch into the existing ordinary contract/domain object only when the field is specified; this preserves untouched fields and prevents request-only wrappers from reaching repositories or persistence models: + +```csharp +if (patch.Notes.IsSpecified) + traveller.Notes = patch.Notes.Value; +``` + ```csharp // samples/src/Contoso.Products.Application/Validators/ProductValidator.cs // Declarative-only — all rules expressed in the constructor. diff --git a/samples/docs/contracts-layer.md b/samples/docs/contracts-layer.md index 39be968ca..c9ffb74d4 100644 --- a/samples/docs/contracts-layer.md +++ b/samples/docs/contracts-layer.md @@ -69,6 +69,19 @@ public partial class Product : ProductBase, IETag, IChangeLog } ``` +For partial-update requests, use a standalone patch contract and `Optional` when omission must remain distinct from explicit `null` or a supplied default. The wrapper belongs at the request boundary; the regular resource contract remains unchanged: + +```csharp +[Contract] +public partial class TravellerNotesPatch +{ + /// Gets or sets the optionally supplied notes. + public Optional Notes { get; set; } +} +``` + +`default(Optional)` is omitted, while assignment marks a value specified. The converter preserves the inner JSON shape, and CoreEx serializer defaults omit unspecified values. The OpenAPI schema uses the inner type/nullability but does not require the property. For a reference-data patch field, declare `[ReferenceData] public partial Optional GenderCode { get; set; }`; the generated typed navigation property shares the code property's presence. + An abstract base class (e.g. [`ProductBase`](../src/Contoso.Products.Contracts/ProductBase.cs)) is used where multiple projections of the same resource share common fields — for example `Product` (full detail) and `ProductLite` (query list projection). > **See also**: [`IIdentifier`](../../src/CoreEx/Entities/IIdentifier.cs) · [`IETag`](../../src/CoreEx/Entities/IETag.cs) · [`IChangeLog`](../../src/CoreEx/Entities/IChangeLog.cs) diff --git a/src/CoreEx.AspNetCore.NSwag/NSwagOpenApiOperationProcessor.cs b/src/CoreEx.AspNetCore.NSwag/NSwagOpenApiOperationProcessor.cs index 38b3b1065..43d0b4e23 100644 --- a/src/CoreEx.AspNetCore.NSwag/NSwagOpenApiOperationProcessor.cs +++ b/src/CoreEx.AspNetCore.NSwag/NSwagOpenApiOperationProcessor.cs @@ -221,17 +221,73 @@ private void ApplyNotNull(AspNetCoreOperationProcessorContext context, Type? typ foreach (var pi in type.GetProperties(BindingFlags.Public | BindingFlags.Instance)) { + if (pi.IsDefined(typeof(JsonIgnoreAttribute), true)) + continue; + var name = pi.GetCustomAttribute()?.Name ?? jso?.PropertyNamingPolicy?.ConvertName(pi.Name) ?? pi.Name; if (!schema.ActualProperties.TryGetValue(name, out var property)) continue; + if (pi.PropertyType.IsGenericType && pi.PropertyType.GetGenericTypeDefinition() == typeof(Optional<>)) + ApplyOptionalProperty(context, schema, property, pi, name); + if (pi.IsDefined(typeof(NonNullableAttribute), true)) { property.IsNullableRaw = false; property.Type &= ~JsonObjectType.Null; } - ApplyNotNull(context, pi.PropertyType, visited); + ApplyNotNull(context, pi.PropertyType.IsGenericType && pi.PropertyType.GetGenericTypeDefinition() == typeof(Optional<>) + ? pi.PropertyType.GetGenericArguments()[0] + : pi.PropertyType, visited); + } + } + + /// + /// Replaces an Optional wrapper schema with its inner schema and makes the containing property non-required. + /// + private static void ApplyOptionalProperty(AspNetCoreOperationProcessorContext context, JsonSchema schema, JsonSchemaProperty property, PropertyInfo propertyInfo, string jsonName) + { + var optionalType = propertyInfo.PropertyType.GetGenericArguments()[0]; + var nullableInfo = new NullabilityInfoContext().Create(propertyInfo); + var innerIsNullable = Nullable.GetUnderlyingType(optionalType) is not null + || nullableInfo.GenericTypeArguments.FirstOrDefault()?.ReadState == NullabilityState.Nullable; + + var innerType = Nullable.GetUnderlyingType(optionalType) ?? optionalType; + var generatedInnerSchema = context.SchemaGenerator.Generate(innerType, context.SchemaResolver); + var innerSchema = generatedInnerSchema.ActualSchema; + var innerJsonType = generatedInnerSchema.Reference is not null ? JsonObjectType.None + : innerType == typeof(string) ? JsonObjectType.String + : innerType == typeof(bool) ? JsonObjectType.Boolean + : innerType == typeof(byte) || innerType == typeof(short) || innerType == typeof(int) || innerType == typeof(long) ? JsonObjectType.Integer + : innerType == typeof(float) || innerType == typeof(double) || innerType == typeof(decimal) ? JsonObjectType.Number + : innerSchema.Type; + + schema.RequiredProperties.Remove(jsonName); + property.IsRequired = false; + property.Reference = generatedInnerSchema.Reference; + property.Type = innerJsonType; + property.Format = innerSchema.Format; + property.IsNullableRaw = innerIsNullable; + property.Type = innerIsNullable ? property.Type | JsonObjectType.Null : property.Type & ~JsonObjectType.Null; + + if (generatedInnerSchema.Reference is null) + { + property.Item = innerSchema.Item; + property.AdditionalPropertiesSchema = innerSchema.AdditionalPropertiesSchema; + property.Properties.Clear(); + foreach (var innerProperty in innerSchema.Properties) + property.Properties.Add(innerProperty.Key, innerProperty.Value); + property.RequiredProperties.Clear(); + foreach (var requiredProperty in innerSchema.RequiredProperties) + property.RequiredProperties.Add(requiredProperty); + } + + if (innerSchema.Enumeration.Count > 0) + { + property.Enumeration.Clear(); + foreach (var value in innerSchema.Enumeration) + property.Enumeration.Add(value); } } @@ -290,4 +346,4 @@ private void HandleRequestFieldQueryString(AspNetCoreOperationProcessorContext c /// Determines whether the is the framework namespace or one of its dotted sub-namespaces (e.g. System.Collections, but not Systematic). /// private static bool IsFrameworkNamespace(string ns, string root) => ns.StartsWith(root, StringComparison.Ordinal) && (ns.Length == root.Length || ns[root.Length] == '.'); -} \ No newline at end of file +} diff --git a/src/CoreEx.AspNetCore.NSwag/README.md b/src/CoreEx.AspNetCore.NSwag/README.md index 23a4d08d5..9ad590128 100644 --- a/src/CoreEx.AspNetCore.NSwag/README.md +++ b/src/CoreEx.AspNetCore.NSwag/README.md @@ -17,6 +17,7 @@ - 📦 **Request body content types**: Reads `[AcceptsAttribute]` and populates the operation `RequestBody` with the declared content type(s) and NSwag-inferred JSON schema for the body type. - 🔑 **Idempotency-key header**: Reads `[IdempotencyKeyAttribute]` and adds an `Idempotency-Key` header parameter to the operation. - ✅ **Non-nullable properties**: Walks the request/response types of each operation and marks any property decorated with `CoreEx.Entities.NonNullableAttribute` as `nullable: false` in the schema, regardless of whether it is declared nullable in code (e.g. `string?`, `int?`). +- 📝 **Optional patch properties**: Unwraps `Optional` to its inner schema, keeps the property out of the required list, and derives nullability from the inner type. - 🚫 **Not-found response**: Reads `[ProducesNotFoundProblemAttribute]` and adds a `404 application/problem+json` response entry. - ⚠️ **ProblemDetails responses**: Optionally injects `application/problem+json` response entries for the HTTP status codes configured via `OpenApiOptions.IncludeProblemDetailsHttpStatusCodes` / `OpenApiOptions.IncludeValidationProblemDetailsHttpStatusCodes` and their corresponding status-code lists; `500` is only included when it is present in the configured list, not by default via a separate `IncludeStandardProblemDetailsResponses` option. - 📡 **Fields query string**: When `OpenApiOptions.IncludeFieldsRequestHeaders` is set, adds the `$fields` query-string parameter for response field projection. diff --git a/src/CoreEx.Validation/AGENTS.md b/src/CoreEx.Validation/AGENTS.md index 6bf9e81e5..685652537 100644 --- a/src/CoreEx.Validation/AGENTS.md +++ b/src/CoreEx.Validation/AGENTS.md @@ -28,6 +28,17 @@ public class ProductValidator : Validator } ``` +## Optional patch properties + +Use `.Unwrap()` on `Optional` request properties to validate the wrapped value with normal rules only when it was supplied. `.Specified()` requires presence but allows specified-null. Keep ordinary value rules off the outer `Optional` property. + +```csharp +Property(x => x.Notes).Unwrap().Mandatory(); +Property(x => x.Category).Specified(); +``` + +Keep `Optional` in the request contract and apply changes to ordinary application/domain objects only after validation. See the [validation README](./README.md#optional-patch-validation). + ## Call from a Service Always call `ValidateAndThrowAsync` before mutating state. This produces a `ValidationException` with all field-level errors at once. @@ -99,6 +110,10 @@ public static readonly CommonValidator SkuValidator = Validator.CreateCo HasRuleFor(p => p.Sku).Common(SkuValidator); ``` +Common validators preserve the caller's label, value formatting and null state. Prefer non-nullable `TValue` for reuse on both nullable and non-nullable properties. `CommonValidator` does not turn an originating null into zero: range rules skip null, `.NotNull()` rejects null but permits zero, and default `.Mandatory()` rejects both. Normalizing rules write back to writable properties (including unwrapped `Optional`) and subsequent rules/`AdditionalAsync` callbacks observe the updated value. Custom wrapper interfaces expose read-only inner values; do not invent reconstruction semantics. + +When changing validation plumbing, preserve typed metadata reads and track adapted null state separately; do not route every value through `object` merely to represent null. Validation is not allocation-free: type-erased adapters, error construction and other infrastructure can still allocate. Keep regression coverage for `string`, `string?`, `int` and `int?`, including omitted/specified values, chained normalization, read-only properties and nested error paths. See the [package README](./README.md#optional-patch-validation) for behavior details. + ## Do Not - Do not compare two properties of the same entity imperatively in `OnValidateAsync` — use `CompareProperty(op, p => p.Other)`; it already handles nulls and cascading errors. diff --git a/src/CoreEx.Validation/Abstractions/IPropertyContextNullState.cs b/src/CoreEx.Validation/Abstractions/IPropertyContextNullState.cs new file mode 100644 index 000000000..510b50851 --- /dev/null +++ b/src/CoreEx.Validation/Abstractions/IPropertyContextNullState.cs @@ -0,0 +1,12 @@ +namespace CoreEx.Validation.Abstractions; + +/// +/// Preserves an adapted property's null state independently of its typed value. +/// +internal interface IPropertyContextNullState +{ + /// + /// Gets the optional function that reads the originating property's current null state. + /// + Func? GetIsValueNull { get; } +} diff --git a/src/CoreEx.Validation/Abstractions/InlineValidator.cs b/src/CoreEx.Validation/Abstractions/InlineValidator.cs index 60240ebfd..924862ac5 100644 --- a/src/CoreEx.Validation/Abstractions/InlineValidator.cs +++ b/src/CoreEx.Validation/Abstractions/InlineValidator.cs @@ -45,19 +45,23 @@ internal InlineValidator(Action? configure) internal async Task ValidateAsync(PropertyContext context, CancellationToken cancellationToken) where TEntity : class { var root = new RootPropertyRule, TValue>( - new PropertyRuntimeMetadata, TValue>(OverrideName ?? context.Name, _ => context.Metadata.GetValue(context.Entity), text: () => OverrideText ?? context.Metadata.Text, jsonName: OverrideJsonName ?? OverrideName ?? context.Metadata.JsonName), + new PropertyRuntimeMetadata, TValue>(OverrideName ?? context.Name, _ => context.Value, + context.Metadata.IsReadOnly ? null : (_, value) => context.Override(value), + text: () => OverrideText ?? context.Text, jsonName: OverrideJsonName ?? OverrideName ?? context.Metadata.JsonName), context.IsValueNullable ? _ => context.GetNullableValueOrDefault() : null, context.IsValueNullable ? _ => context.IsNullableValueDefault() : null); ValidationExtensions.Chain(root, _validator); + ((IRootPropertyRule>)root).SetFormat(context.ValueFormatter.Format, context.ValueFormatter.FormatProvider, context.ValueFormatter.QuotingCharacter); var vv = new ValidationValue(context.Value); var vc = new ValidationContext>(vv, context.CreateValidationArgs(true)); - var pc = new PropertyContext, TValue>(root, vc); + var pc = new PropertyContext, TValue>(root, vc, () => context.IsValueNull); // Execute the primary validation. await root.ValidateAsync(pc, cancellationToken).ConfigureAwait(false); // Execute the secondary validation. + pc.RefreshFromEntity(); await OnValidateAsync(pc, cancellationToken).ConfigureAwait(false); // Merge results. diff --git a/src/CoreEx.Validation/CommonValidatorT.cs b/src/CoreEx.Validation/CommonValidatorT.cs index cb7ed9e3f..d0b4dbe12 100644 --- a/src/CoreEx.Validation/CommonValidatorT.cs +++ b/src/CoreEx.Validation/CommonValidatorT.cs @@ -33,7 +33,8 @@ protected async override Task OnValidateAsync(PropertyContext /// The entity . /// The property . -public struct PropertyContext : IPropertyContext where TEntity : class +public struct PropertyContext : IPropertyContext, IPropertyContextNullState where TEntity : class { private const string _dictionaryKeyParameterName = "__dictionaryKey"; private const string _collectionIndexParameterName = "__collectionIndex"; @@ -13,21 +13,26 @@ public struct PropertyContext : IPropertyContext _root; private readonly IPropertyRuntimeMetadata _metadata; private readonly ValidationContext _owner; + private readonly bool _isValueNullable; + private readonly Func? _getIsValueNull; /// /// Initializes a new instance of the class. /// /// The . /// The validation context for the parent entity. - internal PropertyContext(RootPropertyRule root, ValidationContext context) + /// Optional originating null-state reader for an adapted value. + internal PropertyContext(RootPropertyRule root, ValidationContext context, Func? getIsValueNull = null) { _root = root.ThrowIfNull(); _metadata = root.Metadata.ThrowIfNull(); _owner = context.ThrowIfNull(); + _isValueNullable = root.IsValueNullable; + _getIsValueNull = getIsValueNull; JsonName = _metadata.GetJsonName(context.JsonSerializerOptions); Value = _metadata.GetValue(context.Value); - IsValueNull = Value is null; + IsValueNull = _getIsValueNull?.Invoke() ?? Value is null; ValueFormatter = root.ValueFormatter; Text = root.Text ?? _metadata.Text; @@ -50,15 +55,19 @@ internal PropertyContext(RootPropertyRule root, ValidationCo /// /// The . /// The property value. - internal PropertyContext(IPropertyContext context, TProperty value) + /// Indicates whether the unwrapped value is . + /// Optional adapted metadata for an unwrapped value. + internal PropertyContext(IPropertyContext context, TProperty value, bool? isValueNull = null, IPropertyRuntimeMetadata? metadata = null) { _root = context.RootPropertyRule; - _metadata = context.Metadata; + _metadata = metadata ?? context.Metadata; _owner = (ValidationContext)context.Owner; + _isValueNullable = isValueNull.HasValue ? Nullable.GetUnderlyingType(typeof(TProperty)) is not null : context.IsValueNullable; + _getIsValueNull = metadata is null ? (context as IPropertyContextNullState)?.GetIsValueNull : null; JsonName = context.JsonName; Value = value; - IsValueNull = context.IsValueNull; + IsValueNull = isValueNull ?? context.IsValueNull; ValueFormatter = context.ValueFormatter; Text = context.Text; FullyQualifiedPropertyName = context.FullyQualifiedPropertyName; @@ -68,6 +77,9 @@ internal PropertyContext(IPropertyContext context, TProperty value) /// readonly IRootPropertyRule IPropertyContext.RootPropertyRule => _root; + /// + readonly Func? IPropertyContextNullState.GetIsValueNull => _getIsValueNull; + /// readonly IValidationContext IPropertyContext.Owner => _owner; @@ -127,7 +139,7 @@ internal PropertyContext(IPropertyContext context, TProperty value) /// The values that form part of the message text. /// The . /// The property friendly text and value are automatically prepended to the as the first two arguments where the does not have already have . - public readonly MessageItem AddError(LText format, params object?[] values) => _owner.AddError(_metadata, Text, ValueFormatter, JsonName, format, values); + public readonly MessageItem AddError(LText format, params object?[] values) => _owner.AddError(_metadata, Text, ValueFormatter, JsonName, IsValueNull, format, values); /// public readonly string CreateFullyQualifiedPropertyName(string name) => _owner.CreateFullyQualifiedPropertyName(name); @@ -167,20 +179,38 @@ internal PropertyContext(IPropertyContext context, TProperty value) /// /// Indicates whether the originating property type is . /// - public readonly bool IsValueNullable => _root.IsValueNullable; + public readonly bool IsValueNullable => _isValueNullable; /// /// Gets the originating property or (where ). /// /// The property . /// The originating property or . - public readonly T GetNullableValueOrDefault() => _root.GetNullableValueOrDefault(Entity); + public readonly T GetNullableValueOrDefault() + { + if (_root.IsValueNullable) + return _root.GetNullableValueOrDefault(Entity); + + if (!IsValueNullable) + throw new InvalidOperationException("The property must be Nullable."); + + return Value is null ? default! : Internal.Cast(Value); + } /// /// Indicates whether the originating property is (where ). /// /// where ; otherwise, . - public readonly bool IsNullableValueDefault() => _root.IsNullableValueDefault(Entity); + public readonly bool IsNullableValueDefault() + { + if (_root.IsValueNullable) + return _root.IsNullableValueDefault(Entity); + + if (!IsValueNullable) + throw new InvalidOperationException("The property must be Nullable."); + + return Value is null || Equals(Value, Activator.CreateInstance(Nullable.GetUnderlyingType(typeof(TProperty)) ?? typeof(TProperty))); + } /// /// An will be thrown where the underlying property is read-only. @@ -203,15 +233,19 @@ public void Override(TProperty value) /// Because is a value type that is passed by value through the rule chain (see ), a call to /// made by one rule only updates that rule's own copy of the context. This method allows the chain to re-synchronize its copy with the entity's current value before invoking /// the next chained rule, so that an overridden value is correctly observed by subsequent rules. - /// Skipped where the underlying metadata's declared type does not match (i.e. the rewrap adapter created by ), - /// as the value cannot be safely re-read via the metadata in that case. + /// Nullable value adapters retain the originating null state while exposing the underlying value to subsequent rules. internal void RefreshFromEntity() { - if (_metadata.Type != typeof(TProperty)) + if (_metadata.Type == typeof(TProperty)) + { + Value = _metadata.GetValue(Entity); + IsValueNull = _getIsValueNull?.Invoke() ?? Value is null; return; + } - Value = _metadata.GetValue(Entity); - IsValueNull = Value is null; + var value = _metadata.GetValue(Entity); + Value = Internal.Cast(value); + IsValueNull = _getIsValueNull?.Invoke() ?? value is null; } /// @@ -282,4 +316,4 @@ internal readonly void SetCollectionIndex(int? index) else _owner.Parameters[_collectionIndexParameterName] = index; } -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/README.md b/src/CoreEx.Validation/README.md index ed61117d8..147441d6c 100644 --- a/src/CoreEx.Validation/README.md +++ b/src/CoreEx.Validation/README.md @@ -6,10 +6,44 @@ `CoreEx.Validation` fills the gap between primitive data-annotation validation and a fully featured domain validation library. The framework is built around the concept of a `Validator` that owns a set of `PropertyRule` chains — one chain per property. Each chain is declared fluently using `HasProperty`/`HasRuleFor`, and individual rules are appended to the chain with extension methods defined in `ValidationExtensions`. -Every rule invoked on a property chain runs asynchronously and receives a `PropertyContext` that carries the current entity, the resolved property value, the accumulated error list, and a reference to the owning `ValidationContext`. When a rule adds an error it does so through the context, which means all rules in the chain continue executing and all errors for all properties are collected before the framework decides to throw or return. +Every rule invoked on a property chain runs asynchronously and receives a `PropertyContext` that carries the current entity, the resolved property value, the accumulated error list, and a reference to the owning `ValidationContext`. A property error stops subsequent rules in that chain; other properties continue validating so their errors can be collected before the framework decides to throw or return. Validation is initiated by calling `ValidateAsync` on a validator instance and inspecting the returned `IValidationContext`, or by calling `ValidateAndThrowAsync` which converts errors to a CoreEx `ValidationException` automatically. +## Optional patch validation + +`.Unwrap()` also supports custom wrappers implementing `CoreEx.Entities.Abstractions.IWrappedValue`. Ordinary wrappers always validate their inner value; wrappers implementing `IOptionalValue` skip the inner chain when unspecified, without reading `.Value`. `Optional` and reference-type wrappers infer the inner type automatically. For custom struct wrappers, use the static helper `ValidationExtensions.Unwrap(rule)` because C# cannot infer `TValue` from a generic constraint and variance does not apply to value types. Extension receivers are restricted to wrappers, not arbitrary property types. + +Use `Optional` on an explicit partial-update request DTO when omitted fields must be distinguished from supplied `null` or default values. `.Unwrap()` runs the normal validation chain on the inner value only when the wrapper is specified. Therefore, an omitted value is skipped, while a specified `null` or `0` is handled by the ordinary rules for the inner type. Add `.Specified()` on the outer property when presence itself is required; it does not reject a specified `null`. + +`.Specified()` is a validation rule, not a conditional clause. It supports standard message overrides and conditional clauses, and can precede `.Unwrap()` to require presence before validating the supplied value: `Property(x => x.Title).Specified().Unwrap().String(maxLength: 100)`. Presence validation also supports custom `IOptionalValue` wrappers; custom struct wrappers require the static helper `ValidationExtensions.Specified(rule)`. + +```csharp +[Contract] +public partial class TravellerNotesPatch +{ + /// Gets or sets the optionally supplied title. + public Optional Title { get; set; } +} + +public class TravellerNotesPatchValidator : Validator +{ + public TravellerNotesPatchValidator() => Property(x => x.Title).Unwrap().String(maxLength: 100); +} +``` + +Use `Optional` when explicit `null` is a valid supplied value; `.Unwrap().Mandatory()` rejects that null, while `.Specified()` alone checks only that the field was supplied. `Optional>` can likewise be unwrapped before the normal `.Collection(...)` rules. Unwrapping is supported for root properties only, not wrapped collection items or dictionary values. + +String, comparison, entity, and `.Common(...)` rules accept nullable reference inputs without nullable-annotation warnings and retain their existing fluent return types. Null values keep the normal rule behavior: string constraints skip null, while `.Mandatory()` rejects it. For `Optional`, specified-null is skipped by range rules and rejected by `.Mandatory()`; `.Mandatory()` also rejects specified zero for both `int` and `int?`, matching existing CoreEx behavior. Use `.NotNull()` or `.Mandatory(mustNotBeDefault: false)` when zero is valid. Common validators consume the unwrapped value while preserving the outer property's error path. + +Unwrapped rules use the inner value for error formatting and child-entity validation while retaining the outer property's label and path. Normalizing rules such as `.CompareValues(..., overrideValueWhereMatched: true)` write a specified value back into writable `Optional` properties; subsequent rules observe the normalized value. Custom wrappers expose only a read-only `.Value` contract, so normalization that requires replacing their inner value throws the normal read-only property error rather than guessing how to rebuild the wrapper. + +Common validators preserve the caller's label and value formatting unless explicitly overridden by the common validator. Their normalizing rules write back to writable properties, including unwrapped `Optional` properties, and both subsequent rules and secondary validation observe the updated value. Read-only properties remain read-only. + +When a `CommonValidator` validates an `int?` value, null remains null throughout the chain; it is not substituted with zero for range checks or error formatting. `.NotNull()` therefore rejects null but permits zero, while `.Mandatory()` retains its existing default-value checks. + +Common validation keeps typed metadata getters and tracks the originating null state separately, rather than boxing every value into `object` to represent null. This does not make validation allocation-free: object-based metadata access, error construction, and other framework infrastructure can still allocate. + ## Motivation - The standard .NET `DataAnnotations` model is attribute-only, non-composable, and cannot express cross-property rules or async checks cleanly. @@ -20,13 +54,14 @@ Validation is initiated by calling `ValidateAsync` on a validator instance and i ## Key capabilities -- ✅ **Fluent property-rule chains**: properties are declared with `HasProperty`/`HasRuleFor` and rules are appended via extension methods; the chain executes every rule in order, collecting all errors before reporting. +- ✅ **Fluent property-rule chains**: properties are declared with `HasProperty`/`HasRuleFor` and rules are appended via extension methods; each chain runs in order until its first error, while errors across properties are collected before reporting. - 📋 **Rich built-in rule set**: mandatory, string (length, regex), numeric, decimal/floating-point precision, between/range, comparison (value, property, values), enum, email, wildcard, null/none/empty, error, collection, dictionary, entity, common, reference data, and interop rules cover the vast majority of domain validation needs. - 🧩 **Conditional clauses**: `When`/`WhenValue`/`WhenHasValue` and `DependsOn` clauses short-circuit a rule when a condition is not met, enabling context-sensitive rules without branching in the rule body. - ⚡ **Fully async**: every rule and clause is `async`-capable; `PredicateAsync` delegates allow awaiting external services within a clause. - 🔧 **Common and inline validators**: `CommonValidator` encapsulates reusable validation logic; `InlineValidator` is the compositional base used by both `CommonValidator` and `CommonRule`. - 📝 **Localised error messages**: every rule accepts an optional `LText` override; default messages are defined in `ValidatorStrings` and resolved through CoreEx localisation. - 🔍 **Context-aware property paths**: `PropertyContext` tracks fully qualified property names (both .NET and JSON) so error messages map precisely to the correct field in API responses. +- 📝 **Presence-aware request fields**: `.Unwrap()` applies ordinary validation rules to a specified `Optional` value while skipping omitted fields; `.Specified()` independently requires presence. - 💬 **RuleSet grouping**: `RuleSet` collects a block of rules that only execute when a `Predicate>` returns `true`. - 🧩 **Base-validator include**: `Validator`/`AbstractValidator` and the fluent `Include` method allow a base validator to be composed into a derived validator. - **Interop bridge**: `InteropRule` and the `.Interop()` extension forward validation to any `IValidator`/`IValidator` (e.g. a FluentValidation validator) and merge errors into the CoreEx context. diff --git a/src/CoreEx.Validation/Rules/EntityRule.cs b/src/CoreEx.Validation/Rules/EntityRule.cs index 9bba0ea01..299cafec0 100644 --- a/src/CoreEx.Validation/Rules/EntityRule.cs +++ b/src/CoreEx.Validation/Rules/EntityRule.cs @@ -52,7 +52,7 @@ public void WithValidator(IValidatorEx validator) validator.ThrowIfNull(); _rule._validationAsync = async (context, args, cancellationToken) => { - var value = context.Metadata.GetValue(context.Entity); + var value = context.Value; if (validator is ValidatingInlineValidator vilv) return await vilv.ValidateEntityAsync(value, args, cancellationToken).ConfigureAwait(false); @@ -67,7 +67,7 @@ public void WithValidator(IValidatorEx validator) public void WithValidator() where TValidator : IValidatorEx => _rule._validationAsync = async (context, args, cancellationToken) => { var validator = Validator.Get(args.ServiceProvider); - var value = context.Metadata.GetValue(context.Entity); + var value = context.Value; return await validator.ValidateAsync(value, args, cancellationToken).ConfigureAwait(false); }; @@ -79,8 +79,8 @@ public void WithValidator() where TValidator : IValidatorEx(object? serviceKey) where TValidator : IValidatorEx => _rule._validationAsync = async (context, args, cancellationToken) => { var validator = Validator.GetKeyed(serviceKey, args.ServiceProvider); - var value = context.Metadata.GetValue(context.Entity); + var value = context.Value; return await validator.ValidateAsync(value, args, cancellationToken).ConfigureAwait(false); }; } -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/Rules/SpecifiedRule.cs b/src/CoreEx.Validation/Rules/SpecifiedRule.cs new file mode 100644 index 000000000..847d7c177 --- /dev/null +++ b/src/CoreEx.Validation/Rules/SpecifiedRule.cs @@ -0,0 +1,22 @@ +namespace CoreEx.Validation.Rules; + +/// +/// Requires an optional wrapper to have been specified. +/// +/// The owning entity type. +/// The optional wrapper type. +/// The wrapped value type. +internal sealed class SpecifiedRule : PropertyRuleBase where TEntity : class where TWrapper : IOptionalValue +{ + /// + protected override bool ValidateWhenNull => true; + + /// + protected override Task OnValidateAsync(PropertyContext context, CancellationToken cancellationToken) + { + if (context.Value is null || !context.Value.IsSpecified) + context.AddError(ErrorText ?? ValidatorStrings.MandatoryFormat); + + return Task.CompletedTask; + } +} diff --git a/src/CoreEx.Validation/Rules/UnwrapRule.cs b/src/CoreEx.Validation/Rules/UnwrapRule.cs new file mode 100644 index 000000000..a018a981d --- /dev/null +++ b/src/CoreEx.Validation/Rules/UnwrapRule.cs @@ -0,0 +1,34 @@ +namespace CoreEx.Validation.Rules; + +/// +/// Adapts a wrapped property's value to the existing typed property-rule chain. +/// +/// The owning entity type. +/// The wrapper type. +/// The wrapped value type. +/// An optional factory to write overridden values back into the wrapper. +internal sealed class UnwrapRule(Func? wrapValue = null) : PropertyRuleBase, IPropertyRule where TEntity : class where TWrapper : IWrappedValue +{ + /// + async Task IPropertyRule.ValidateAsync(IPropertyContext context, CancellationToken cancellationToken) + { + if (context.Value is not TWrapper wrapper) + throw new InvalidOperationException($"The property value must be {typeof(TWrapper)}."); + + if (wrapper is IOptionalValue optional && !optional.IsSpecified) + return; + + var outerMetadata = context.Metadata; + var metadata = new PropertyRuntimeMetadata(outerMetadata.Name, + entity => outerMetadata.GetValue(entity).Value, + wrapValue is not null && !outerMetadata.IsReadOnly ? (entity, value) => outerMetadata.SetValue(entity, wrapValue(value)) : null, + text: () => outerMetadata.Text, jsonName: outerMetadata.JsonName, format: outerMetadata.Format); + + var value = wrapper.Value; + var propertyContext = new PropertyContext(context, value, value is null, metadata); + await ((IPropertyRuleEx)this).ValidateAsync(propertyContext, cancellationToken).ConfigureAwait(false); + } + + /// + protected override Task OnValidateAsync(PropertyContext context, CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/src/CoreEx.Validation/ValidationContext.Utility.cs b/src/CoreEx.Validation/ValidationContext.Utility.cs index a81d3e18c..a4da68774 100644 --- a/src/CoreEx.Validation/ValidationContext.Utility.cs +++ b/src/CoreEx.Validation/ValidationContext.Utility.cs @@ -39,7 +39,7 @@ public bool Check(Expression> propertyExpres if (!predicate(pm.GetValue(Value))) return false; - AddError(pm, null, null, null, format, values); + AddError(pm, null, null, null, null, format, values); return true; } @@ -63,7 +63,7 @@ public bool Check(Expression> propertyExpres if (!when) return false; - AddError(pm, null, null, null, format, values); + AddError(pm, null, null, null, null, format, values); return true; } @@ -77,7 +77,7 @@ public bool Check(Expression> propertyExpres /// The . /// The property friendly text and value are automatically prepended to the as the first two arguments where the does not have already have . public MessageItem AddError(Expression> propertyExpression, LText format, params object?[] values) - => AddError(RuntimeMetadata.GetForExpression(propertyExpression), null, null, null, format, values); + => AddError(RuntimeMetadata.GetForExpression(propertyExpression), null, null, null, null, format, values); /// /// Adds an to the for the specified property, explicit text format and and additional values included in the text. @@ -87,17 +87,19 @@ public MessageItem AddError(Expression> prop /// The property override. /// The . /// The JSON property name override. + /// Optional originating null state for an adapted value. /// The composite format string. /// The values that form part of the message text. /// The . /// The property friendly text and value are automatically prepended to the as the first two arguments where the does not have already have . - internal MessageItem AddError(IPropertyRuntimeMetadata propertyMetadata, LText? text, ValueFormatter? valueFormatter, string? jsonNameOverride, LText format, params object?[] values) + internal MessageItem AddError(IPropertyRuntimeMetadata propertyMetadata, LText? text, ValueFormatter? valueFormatter, string? jsonNameOverride, bool? isValueNull, LText format, params object?[] values) { format = format.EnsureNoArgsWhen(values); text ??= propertyMetadata.Text; jsonNameOverride ??= propertyMetadata.GetJsonName(JsonSerializerOptions); - object?[] std = [text, valueFormatter is null ? propertyMetadata.GetValue(Value) : valueFormatter.Value.ToLText(propertyMetadata.GetValue(Value))]; + var value = isValueNull == true ? null : propertyMetadata.GetValue(Value); + object?[] std = [text, valueFormatter is null ? value : valueFormatter.Value.ToLText(value)]; if (propertyMetadata is ISelfRuntimeMetadata) return AddError(format, [.. std, .. values]); @@ -160,4 +162,4 @@ internal MessageItem AddError(LText format, params object?[] values) /// /// The JSON property name. public string CreateFullyQualifiedJsonPropertyName(string jsonName) => FullyQualifiedJsonEntityName is null ? jsonName : (jsonName.StartsWith('[') ? FullyQualifiedJsonEntityName + jsonName : FullyQualifiedJsonEntityName + "." + jsonName); -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/ValidationExtensions.BetweenRule.cs b/src/CoreEx.Validation/ValidationExtensions.BetweenRule.cs index b38e4c117..9b20d10f6 100644 --- a/src/CoreEx.Validation/ValidationExtensions.BetweenRule.cs +++ b/src/CoreEx.Validation/ValidationExtensions.BetweenRule.cs @@ -14,7 +14,7 @@ public static partial class ValidationExtensions /// The maximum text (used in the error message); otherwise, uses the resulting value. /// Indicates whether the between comparison is exclusive or inclusive (default). /// The optional . - public static IPropertyRule Between(this IPropertyRule rule, TProperty min, TProperty max, LText? minText = null, LText? maxText = null, bool exclusiveBetween = false, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule Between(this IPropertyRule rule, TProperty min, TProperty max, LText? minText = null, LText? maxText = null, bool exclusiveBetween = false, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new BetweenRule(_ => min, _ => max, _ => minText, _ => maxText, exclusiveBetween, comparer)); /// @@ -29,7 +29,7 @@ public static IPropertyRule Between(this /// The maximum text formatter (used in the error message); otherwise, uses the resulting value. /// Indicates whether the between comparison is exclusive or inclusive (default). /// The optional . - public static IPropertyRule Between(this IPropertyRule rule, Func, TProperty> min, Func, TProperty> max, Func? minText = null, Func? maxText = null, bool exclusiveBetween = false, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule Between(this IPropertyRule rule, Func, TProperty> min, Func, TProperty> max, Func? minText = null, Func? maxText = null, bool exclusiveBetween = false, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new BetweenRule(c => min.ThrowIfNull()(c), c => max.ThrowIfNull()(c), minText, maxText, exclusiveBetween, comparer)); /// @@ -43,7 +43,7 @@ public static IPropertyRule Between(this /// The minimum text (used in the error message); otherwise, uses the resulting value. /// The maximum text (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule InclusiveBetween(this IPropertyRule rule, TProperty min, TProperty max, LText? minText = null, LText? maxText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule InclusiveBetween(this IPropertyRule rule, TProperty min, TProperty max, LText? minText = null, LText? maxText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new BetweenRule(_ => min, _ => max, _ => minText, _ => maxText, false, comparer)); /// @@ -57,7 +57,7 @@ public static IPropertyRule InclusiveBetweenThe minimum text formatter (used in the error message); otherwise, uses the resulting value. /// The maximum text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule InclusiveBetween(this IPropertyRule rule, Func, TProperty> min, Func, TProperty> max, Func? minText = null, Func? maxText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule InclusiveBetween(this IPropertyRule rule, Func, TProperty> min, Func, TProperty> max, Func? minText = null, Func? maxText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new BetweenRule(c => min.ThrowIfNull()(c), c => max.ThrowIfNull()(c), minText, maxText, false, comparer)); /// @@ -71,7 +71,7 @@ public static IPropertyRule InclusiveBetweenThe minimum text (used in the error message); otherwise, uses the resulting value. /// The maximum text (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule ExclusiveBetween(this IPropertyRule rule, TProperty min, TProperty max, LText? minText = null, LText? maxText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule ExclusiveBetween(this IPropertyRule rule, TProperty min, TProperty max, LText? minText = null, LText? maxText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new BetweenRule(_ => min, _ => max, _ => minText, _ => maxText, true, comparer)); /// @@ -85,7 +85,7 @@ public static IPropertyRule ExclusiveBetweenThe minimum text formatter (used in the error message); otherwise, uses the resulting value. /// The maximum text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule ExclusiveBetween(this IPropertyRule rule, Func, TProperty> min, Func, TProperty> max, Func? minText = null, Func? maxText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule ExclusiveBetween(this IPropertyRule rule, Func, TProperty> min, Func, TProperty> max, Func? minText = null, Func? maxText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new BetweenRule(c => min.ThrowIfNull()(c), c => max.ThrowIfNull()(c), minText, maxText, true, comparer)); /* Nullable/Struct */ @@ -175,4 +175,4 @@ public static IPropertyRule ExclusiveBetweenThe optional . public static IPropertyRule ExclusiveBetween(this IPropertyRule rule, Func, TProperty> min, Func, TProperty> max, Func? minText = null, Func? maxText = null, IComparer? comparer = null) where TEntity : class where TProperty : struct, IComparable => Chain(rule, new BetweenRule(c => min.ThrowIfNull()(c), c => max.ThrowIfNull()(c), minText, maxText, true, comparer)); -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/ValidationExtensions.CommonRule.cs b/src/CoreEx.Validation/ValidationExtensions.CommonRule.cs index c0087fade..9202819a5 100644 --- a/src/CoreEx.Validation/ValidationExtensions.CommonRule.cs +++ b/src/CoreEx.Validation/ValidationExtensions.CommonRule.cs @@ -9,7 +9,7 @@ public static partial class ValidationExtensions /// The property . /// The being extended. /// The . - public static IPropertyRule Common(this IPropertyRule rule, InlineValidator commonValidator) where TEntity : class + public static IPropertyRule Common(this IPropertyRule rule, InlineValidator commonValidator) where TEntity : class => Chain(rule, new CommonRule(commonValidator.ThrowIfNull())); /// @@ -21,4 +21,4 @@ public static IPropertyRule Common(this /// The . public static IPropertyRule Common(this IPropertyRule rule, InlineValidator commonValidator) where TEntity : class where TProperty : struct => Chain(rule, new CommonRule(commonValidator.ThrowIfNull())); -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/ValidationExtensions.ComparePropertyRule.cs b/src/CoreEx.Validation/ValidationExtensions.ComparePropertyRule.cs index 8bccb9417..ca57d7aba 100644 --- a/src/CoreEx.Validation/ValidationExtensions.ComparePropertyRule.cs +++ b/src/CoreEx.Validation/ValidationExtensions.ComparePropertyRule.cs @@ -14,7 +14,7 @@ public static partial class ValidationExtensions /// The compare-to value text formatter (used in the error message); otherwise, uses the resulting compare-to value. /// The optional . /// Where the is null then then is used. - public static IPropertyRule CompareProperty(this IPropertyRule rule, CompareOperator compareOperator, Expression> compareToPropertyExpression, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule CompareProperty(this IPropertyRule rule, CompareOperator compareOperator, Expression> compareToPropertyExpression, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new ComparePropertyRule(compareOperator, compareToPropertyExpression, compareToText, comparer)); /// diff --git a/src/CoreEx.Validation/ValidationExtensions.CompareValueRule.cs b/src/CoreEx.Validation/ValidationExtensions.CompareValueRule.cs index 0b931823d..7d8418549 100644 --- a/src/CoreEx.Validation/ValidationExtensions.CompareValueRule.cs +++ b/src/CoreEx.Validation/ValidationExtensions.CompareValueRule.cs @@ -12,7 +12,7 @@ public static partial class ValidationExtensions /// The compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule Compare(this IPropertyRule rule, CompareOperator compareOperator, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule Compare(this IPropertyRule rule, CompareOperator compareOperator, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(compareOperator, _ => compareTo, compareToText, comparer)); /// @@ -25,7 +25,7 @@ public static IPropertyRule Compare(this /// The function to get the compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule Compare(this IPropertyRule rule, CompareOperator compareOperator, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule Compare(this IPropertyRule rule, CompareOperator compareOperator, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(compareOperator, c => compareTo.ThrowIfNull()(c), compareToText, comparer)); /// @@ -37,7 +37,7 @@ public static IPropertyRule Compare(this /// The compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule Equal(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule Equal(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.Equal, _ => compareTo, compareToText, comparer)); /// @@ -49,7 +49,7 @@ public static IPropertyRule Equal(this I /// The function to get the compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule Equal(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule Equal(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.Equal, c => compareTo.ThrowIfNull()(c), compareToText, comparer)); /// @@ -61,7 +61,7 @@ public static IPropertyRule Equal(this I /// The compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule NotEqual(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule NotEqual(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.NotEqual, _ => compareTo, compareToText, comparer)); /// @@ -73,7 +73,7 @@ public static IPropertyRule NotEqual(thi /// The function to get the compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule NotEqual(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule NotEqual(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.NotEqual, c => compareTo.ThrowIfNull()(c), compareToText, comparer)); /// @@ -85,7 +85,7 @@ public static IPropertyRule NotEqual(thi /// The compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule LessThan(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule LessThan(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.LessThan, _ => compareTo, compareToText, comparer)); /// @@ -97,7 +97,7 @@ public static IPropertyRule LessThan(thi /// The function to get the compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule LessThan(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule LessThan(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.LessThan, c => compareTo.ThrowIfNull()(c), compareToText, comparer)); /// @@ -109,7 +109,7 @@ public static IPropertyRule LessThan(thi /// The compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule LessThanOrEqualTo(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule LessThanOrEqualTo(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.LessThanOrEqualTo, _ => compareTo, compareToText, comparer)); /// @@ -121,7 +121,7 @@ public static IPropertyRule LessThanOrEqualToThe function to get the compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule LessThanOrEqualTo(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule LessThanOrEqualTo(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.LessThanOrEqualTo, c => compareTo.ThrowIfNull()(c), compareToText, comparer)); /// @@ -133,7 +133,7 @@ public static IPropertyRule LessThanOrEqualToThe compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule GreaterThanOrEqualTo(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule GreaterThanOrEqualTo(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.GreaterThanOrEqualTo, _ => compareTo, compareToText, comparer)); /// @@ -145,7 +145,7 @@ public static IPropertyRule GreaterThanOrEqualToThe function to get the compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule GreaterThanOrEqualTo(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule GreaterThanOrEqualTo(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.GreaterThanOrEqualTo, c => compareTo.ThrowIfNull()(c), compareToText, comparer)); /// @@ -157,7 +157,7 @@ public static IPropertyRule GreaterThanOrEqualToThe compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule GreaterThan(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule GreaterThan(this IPropertyRule rule, TProperty compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.GreaterThan, _ => compareTo, compareToText, comparer)); /// @@ -169,7 +169,7 @@ public static IPropertyRule GreaterThan( /// The function to get the compare-to value. /// The value text formatter (used in the error message); otherwise, uses the resulting value. /// The optional . - public static IPropertyRule GreaterThan(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable + public static IPropertyRule GreaterThan(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : IComparable => Chain(rule, new CompareValueRule(CompareOperator.GreaterThan, c => compareTo.ThrowIfNull()(c), compareToText, comparer)); /* Nullable/Struct */ @@ -343,4 +343,4 @@ public static IPropertyRule GreaterThan( /// The optional . public static IPropertyRule GreaterThan(this IPropertyRule rule, Func, TProperty> compareTo, Func? compareToText = null, IComparer? comparer = null) where TEntity : class where TProperty : struct, IComparable => Chain(rule, new CompareValueRule(CompareOperator.GreaterThan, c => compareTo.ThrowIfNull()(c), compareToText, comparer)); -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/ValidationExtensions.CompareValuesRule.cs b/src/CoreEx.Validation/ValidationExtensions.CompareValuesRule.cs index 26950c447..3f1e4b9a3 100644 --- a/src/CoreEx.Validation/ValidationExtensions.CompareValuesRule.cs +++ b/src/CoreEx.Validation/ValidationExtensions.CompareValuesRule.cs @@ -11,7 +11,7 @@ public static partial class ValidationExtensions /// The compare-to value(s). /// The optional . /// Indicates whether to override the underlying property value with the corresponding matched value. - public static IPropertyRule CompareValues(this IPropertyRule rule, IEnumerable values, IEqualityComparer? comparer = null, bool overrideValueWhereMatched = false) where TEntity : class where TProperty : IEquatable + public static IPropertyRule CompareValues(this IPropertyRule rule, IEnumerable values, IEqualityComparer? comparer = null, bool overrideValueWhereMatched = false) where TEntity : class where TProperty : IEquatable => Chain(rule, new CompareValuesRule(_ => values, comparer, overrideValueWhereMatched)); /// @@ -23,7 +23,7 @@ public static IPropertyRule CompareValuesThe function to get the compare-to value(s). /// The optional . /// Indicates whether to override the underlying property value with the corresponding matched value. - public static IPropertyRule CompareValues(this IPropertyRule rule, Func, IEnumerable> values, IEqualityComparer? comparer = null, bool overrideValueWhereMatched = false) where TEntity : class where TProperty : IEquatable + public static IPropertyRule CompareValues(this IPropertyRule rule, Func, IEnumerable> values, IEqualityComparer? comparer = null, bool overrideValueWhereMatched = false) where TEntity : class where TProperty : IEquatable => Chain(rule, new CompareValuesRule(c => values.ThrowIfNull()(c), comparer, overrideValueWhereMatched)); /* Nullable/Struct */ @@ -51,4 +51,4 @@ public static IPropertyRule CompareValuesIndicates whether to override the underlying property value with the corresponding matched value. public static IPropertyRule CompareValues(this IPropertyRule rule, Func, IEnumerable> values, IEqualityComparer? comparer = null, bool overrideValueWhereMatched = false) where TEntity : class where TProperty : struct, IEquatable => Chain(rule, new CompareValuesRule(c => values.ThrowIfNull()(c), comparer, overrideValueWhereMatched)); -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/ValidationExtensions.EntityRule.cs b/src/CoreEx.Validation/ValidationExtensions.EntityRule.cs index 2ec5b04ca..4427906ca 100644 --- a/src/CoreEx.Validation/ValidationExtensions.EntityRule.cs +++ b/src/CoreEx.Validation/ValidationExtensions.EntityRule.cs @@ -9,7 +9,7 @@ public static partial class ValidationExtensions /// The property . /// The being extended. /// Extends configuration . - public static IPropertyRule Entity(this IPropertyRule rule, Action.With> with) where TEntity : class where TProperty : class? + public static IPropertyRule Entity(this IPropertyRule rule, Action.With> with) where TEntity : class where TProperty : class? => Chain(rule, new EntityRule(with)); /// @@ -20,6 +20,6 @@ public static IPropertyRule Entity(this /// The being extended. /// The . /// Consider using the as this provides additional validator options (where applicable). - public static IPropertyRule Entity(this IPropertyRule rule, IValidatorEx validator) where TEntity : class where TProperty : class? + public static IPropertyRule Entity(this IPropertyRule rule, IValidatorEx validator) where TEntity : class where TProperty : class? => Entity(rule, w => w.WithValidator(validator)); -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/ValidationExtensions.Optional.cs b/src/CoreEx.Validation/ValidationExtensions.Optional.cs new file mode 100644 index 000000000..4a3909303 --- /dev/null +++ b/src/CoreEx.Validation/ValidationExtensions.Optional.cs @@ -0,0 +1,28 @@ +namespace CoreEx.Validation; + +public static partial class ValidationExtensions +{ + /// + /// Requires the optional property to be specified. + /// + /// The owning entity type. + /// The wrapped value type. + /// The optional property rule. + /// The presence rule to support fluent-style method-chaining. + public static IPropertyRule> Specified(this IPropertyRule> rule) where TEntity : class + => Specified, TValue>(rule); + + /// + /// Validates the wrapped value when the optional property is specified. + /// + /// The owning entity type. + /// The wrapped value type. + /// The optional property rule. + /// A rule for the wrapped value. + public static IPropertyRule Unwrap(this IPropertyRule> rule) where TEntity : class + { + var unwrapRule = new UnwrapRule, TValue>(Optional.Specified); + rule.ThrowIfNull().Chain(unwrapRule); + return unwrapRule; + } +} diff --git a/src/CoreEx.Validation/ValidationExtensions.SpecifiedRule.cs b/src/CoreEx.Validation/ValidationExtensions.SpecifiedRule.cs new file mode 100644 index 000000000..087772b5f --- /dev/null +++ b/src/CoreEx.Validation/ValidationExtensions.SpecifiedRule.cs @@ -0,0 +1,26 @@ +namespace CoreEx.Validation; + +public static partial class ValidationExtensions +{ + /// + /// Requires an optional wrapper to be specified without rejecting a specified null value. + /// + /// The owning entity type. + /// The wrapped value type. + /// The optional property rule. + /// The presence rule to support fluent-style method-chaining. + public static IPropertyRule> Specified(this IPropertyRule> rule) where TEntity : class + => Specified, TValue>(rule); + + /// + /// Requires an optional wrapper of an explicitly specified type to be specified. + /// + /// The owning entity type. + /// The optional wrapper type. + /// The wrapped value type. + /// The optional property rule. + /// The presence rule to support fluent-style method-chaining. + /// Call this static helper with explicit type arguments for custom struct wrappers because generic variance cannot infer their wrapped value type. + public static IPropertyRule Specified(IPropertyRule rule) where TEntity : class where TWrapper : IOptionalValue + => Chain(rule, new SpecifiedRule()); +} diff --git a/src/CoreEx.Validation/ValidationExtensions.StringRule.cs b/src/CoreEx.Validation/ValidationExtensions.StringRule.cs index 1548096c9..82b9b52cb 100644 --- a/src/CoreEx.Validation/ValidationExtensions.StringRule.cs +++ b/src/CoreEx.Validation/ValidationExtensions.StringRule.cs @@ -9,7 +9,7 @@ public static partial class ValidationExtensions /// The being extended. /// The maximum string length. /// The . - public static IPropertyRule String(this IPropertyRule rule, int maxLength) where TEntity : class + public static IPropertyRule String(this IPropertyRule rule, int maxLength) where TEntity : class => Chain(rule, new StringRule(maxLength: _ => maxLength)); /// @@ -23,7 +23,7 @@ public static IPropertyRule String(this IPropertyRule< /// The . /// The is executed directly against the value being validated; where the value may originate from an untrusted source, construct the with an explicit /// matchTimeout to guard against catastrophic backtracking (ReDoS), as a timeout cannot be applied to an already-constructed instance. - public static IPropertyRule String(this IPropertyRule rule, int minLength, int? maxLength, Regex? regex = null) where TEntity : class + public static IPropertyRule String(this IPropertyRule rule, int minLength, int? maxLength, Regex? regex = null) where TEntity : class => Chain(rule, new StringRule(minLength: _ => minLength, maxLength: _ => maxLength, regex: _ => regex)); /// @@ -37,7 +37,7 @@ public static IPropertyRule String(this IPropertyRule< /// The . /// The is executed directly against the value being validated; where the value may originate from an untrusted source, construct the with an explicit /// matchTimeout to guard against catastrophic backtracking (ReDoS), as a timeout cannot be applied to an already-constructed instance. - public static IPropertyRule String(this IPropertyRule rule, Func, int>? minLength, Func, int?>? maxLength, Func, Regex?>? regex = null) where TEntity : class + public static IPropertyRule String(this IPropertyRule rule, Func, int>? minLength, Func, int?>? maxLength, Func, Regex?>? regex = null) where TEntity : class => Chain(rule, new StringRule(minLength: minLength, maxLength: maxLength, regex: regex)); /// @@ -49,7 +49,7 @@ public static IPropertyRule String(this IPropertyRule< /// The . /// The is executed directly against the value being validated; where the value may originate from an untrusted source, construct the with an explicit /// matchTimeout to guard against catastrophic backtracking (ReDoS), as a timeout cannot be applied to an already-constructed instance. - public static IPropertyRule String(this IPropertyRule rule, Regex regex) where TEntity : class + public static IPropertyRule String(this IPropertyRule rule, Regex regex) where TEntity : class => Chain(rule, new StringRule(regex: _ => regex)); /// @@ -61,7 +61,7 @@ public static IPropertyRule String(this IPropertyRule< /// The . /// The is executed directly against the value being validated; where the value may originate from an untrusted source, construct the with an explicit /// matchTimeout to guard against catastrophic backtracking (ReDoS), as a timeout cannot be applied to an already-constructed instance. - public static IPropertyRule Matches(this IPropertyRule rule, Regex regex) where TEntity : class + public static IPropertyRule Matches(this IPropertyRule rule, Regex regex) where TEntity : class => Chain(rule, new StringRule(regex: _ => regex)); /// @@ -72,7 +72,7 @@ public static IPropertyRule Matches(this IPropertyRule /// The exact string length. /// The optional . /// The . - public static IPropertyRule Length(this IPropertyRule rule, int exactLength, Regex? regex = null) where TEntity : class + public static IPropertyRule Length(this IPropertyRule rule, int exactLength, Regex? regex = null) where TEntity : class => Chain(rule, new StringRule(minLength: _ => exactLength, maxLength: _ => exactLength, regex: _ => regex)); /// @@ -82,7 +82,7 @@ public static IPropertyRule Length(this IPropertyRule< /// The being extended. /// The minimum string length. /// The . - public static IPropertyRule MinimumLength(this IPropertyRule rule, int minLength) where TEntity : class + public static IPropertyRule MinimumLength(this IPropertyRule rule, int minLength) where TEntity : class => Chain(rule, new StringRule(minLength: _ => minLength)); /// @@ -92,6 +92,6 @@ public static IPropertyRule MinimumLength(this IProper /// The being extended. /// The maximum string length. /// The . - public static IPropertyRule MaximumLength(this IPropertyRule rule, int maxLength) where TEntity : class + public static IPropertyRule MaximumLength(this IPropertyRule rule, int maxLength) where TEntity : class => Chain(rule, new StringRule(maxLength: _ => maxLength)); -} \ No newline at end of file +} diff --git a/src/CoreEx.Validation/ValidationExtensions.UnwrapRule.cs b/src/CoreEx.Validation/ValidationExtensions.UnwrapRule.cs new file mode 100644 index 000000000..ed07fffbf --- /dev/null +++ b/src/CoreEx.Validation/ValidationExtensions.UnwrapRule.cs @@ -0,0 +1,30 @@ +namespace CoreEx.Validation; + +public static partial class ValidationExtensions +{ + /// + /// Validates the wrapped value, skipping the inner chain when an optional wrapper is unspecified. + /// + /// The owning entity type. + /// The wrapped value type. + /// The wrapped property rule. + /// A rule for the wrapped value. + public static IPropertyRule Unwrap(this IPropertyRule> rule) where TEntity : class + => Unwrap, TValue>(rule); + + /// + /// Validates the wrapped value for an explicitly specified wrapper and value type. + /// + /// The owning entity type. + /// The wrapper type. + /// The wrapped value type. + /// The wrapped property rule. + /// A rule for the wrapped value. + /// Call this static helper with explicit type arguments for custom struct wrappers because generic variance cannot infer their wrapped value type. + public static IPropertyRule Unwrap(IPropertyRule rule) where TEntity : class where TWrapper : IWrappedValue + { + var unwrapRule = new UnwrapRule(); + rule.ThrowIfNull().Chain(unwrapRule); + return unwrapRule; + } +} diff --git a/src/CoreEx.Validation/ValidationExtensions.cs b/src/CoreEx.Validation/ValidationExtensions.cs index 3f2e38de8..2268efa36 100644 --- a/src/CoreEx.Validation/ValidationExtensions.cs +++ b/src/CoreEx.Validation/ValidationExtensions.cs @@ -27,7 +27,8 @@ public static IPropertyRule AddClause(IP /// The being extended. /// The next in the chain. /// The to support fluent-style method-chaining. - public static IPropertyRule Chain(IPropertyRule rule, IPropertyRule nextRule) where TEntity : class + /// Accepts nullable reference inputs while retaining the next rule's property type. Runtime null handling remains the responsibility of each rule. + public static IPropertyRule Chain(IPropertyRule rule, IPropertyRule nextRule) where TEntity : class { rule.ThrowIfNull().Chain(nextRule.ThrowIfNull()); return nextRule; diff --git a/src/CoreEx/AGENTS.md b/src/CoreEx/AGENTS.md index 64b1fa39a..2b75586aa 100644 --- a/src/CoreEx/AGENTS.md +++ b/src/CoreEx/AGENTS.md @@ -148,6 +148,8 @@ See [usage, inline examples, and semantics](./README.md#validating-http-response Cached runtime metadata lookups prefer the type's static `IRuntimeMetadata` provider and use reflection only for types without that interface. This applies to root/nested `GetJsonPath` and `GetJsonPathAndText` resolution, so custom/generated JSON names, friendly labels, and excluded properties are respected consistently regardless of which cache entry point runs first. Entity instances are never constructed for metadata discovery. +For partial-update request DTOs, `Optional` distinguishes an omitted property from a supplied `null` or default value. Keep it at the request boundary; map only specified values into ordinary contracts or domain objects. CoreEx JSON defaults omit unspecified wrappers, while custom serializer options must ignore default values too. Custom value wrappers (optional or always present) may opt into runtime cleaning through `Entities.Abstractions.IWrappedValueCleaner` using their own type as `TSelf`, returning a non-null wrapper of the same runtime type and preserving its invariants; optional wrappers must also preserve presence. Typed cleaning uses cached constrained dispatch to avoid wrapper boxing; object/interface boundaries and inner cleaning may still allocate. Implementing the read-only wrapper interfaces alone does not enable cleaning or other `Optional` tooling. See the [contract guidance](../../.github/instructions/coreex-contracts.instructions.md#optional-patch-properties). + Use the standard interfaces on your contracts so the framework's ETag, paging, and change-log handling works automatically. For entities authored in a project that references `CoreEx.CodeGen`, prefer the Roslyn source generator — declare a `partial` class with `[Contract]` and the generator emits the boilerplate (constructors, `CopyFrom`, `Clone`, equality, `CleanUp`, etc.). diff --git a/src/CoreEx/Abstractions/Internal.cs b/src/CoreEx/Abstractions/Internal.cs index 1fab70403..3e37bc78e 100644 --- a/src/CoreEx/Abstractions/Internal.cs +++ b/src/CoreEx/Abstractions/Internal.cs @@ -140,16 +140,16 @@ public static string GetValueFromConfigurationWhereApplicable(string valueOrKey, /// This uses a to perform as it is fastest means possible. Where there is a mismatch of types an will be thrown. internal static TTo Cast(TFrom value, bool trustMe = false) { + // Check identical types before null and pattern checks, which can box value types. + if (typeof(TFrom) == typeof(TTo)) + return Unsafe.As(ref value); + if (value is null) return default!; if (value is TTo t) return t; - // Must be identical types, uses Unsafe.As to avoid boxing of value types. - if (typeof(TFrom) == typeof(TTo)) - return Unsafe.As(ref value); - // Allow bypass of type check where caller is certain of compatibility (use with extreme caution). if (trustMe) return Unsafe.As(ref value); @@ -184,4 +184,4 @@ private static string GetGenericFormattedName(Type type) return $"{name}<{string.Join(',', type.GetGenericArguments().Select(GetFormattedName))}>"; } -} \ No newline at end of file +} diff --git a/src/CoreEx/Entities/Abstractions/IOptionalValue.cs b/src/CoreEx/Entities/Abstractions/IOptionalValue.cs new file mode 100644 index 000000000..63cd33f78 --- /dev/null +++ b/src/CoreEx/Entities/Abstractions/IOptionalValue.cs @@ -0,0 +1,13 @@ +namespace CoreEx.Entities.Abstractions; + +/// +/// Represents a value wrapper that can distinguish an unspecified value from a specified value. +/// +/// The wrapped value type. +public interface IOptionalValue : IWrappedValue +{ + /// + /// Indicates whether a value was specified. + /// + bool IsSpecified { get; } +} diff --git a/src/CoreEx/Entities/Abstractions/IWrappedValue.cs b/src/CoreEx/Entities/Abstractions/IWrappedValue.cs new file mode 100644 index 000000000..3dcee36f3 --- /dev/null +++ b/src/CoreEx/Entities/Abstractions/IWrappedValue.cs @@ -0,0 +1,13 @@ +namespace CoreEx.Entities.Abstractions; + +/// +/// Represents a value wrapper that exposes its underlying value. +/// +/// The wrapped value type. +public interface IWrappedValue +{ + /// + /// Gets the wrapped value. + /// + T Value { get; } +} diff --git a/src/CoreEx/Entities/Abstractions/IWrappedValueCleaner.cs b/src/CoreEx/Entities/Abstractions/IWrappedValueCleaner.cs new file mode 100644 index 000000000..8da7cebc0 --- /dev/null +++ b/src/CoreEx/Entities/Abstractions/IWrappedValueCleaner.cs @@ -0,0 +1,16 @@ +namespace CoreEx.Entities.Abstractions; + +/// +/// Enables a value wrapper to participate in runtime cleaning and return its own type. +/// +/// The implementing wrapper type. +public interface IWrappedValueCleaner where TSelf : IWrappedValueCleaner +{ + /// + /// Cleans the wrapped value and returns the reconstructed wrapper. + /// + /// The cleaning configuration. + /// The wrapper containing the cleaned value. + /// The result must be non-null and have the same runtime wrapper type, preserving the wrapper's invariants. Optional wrappers must also preserve whether the value was specified, including when cleaning produces a null inner value, and must not read an unspecified value. + TSelf Clean(CleanArgs args); +} diff --git a/src/CoreEx/Entities/Optional.cs b/src/CoreEx/Entities/Optional.cs new file mode 100644 index 000000000..183b6e22d --- /dev/null +++ b/src/CoreEx/Entities/Optional.cs @@ -0,0 +1,66 @@ +namespace CoreEx.Entities; + +/// +/// Represents an optionally specified value, including a specified value. +/// +/// The wrapped value type. +[JsonConverter(typeof(OptionalJsonConverterFactory))] +public readonly struct Optional : IOptionalValue, IEquatable>, IWrappedValueCleaner> +{ + private readonly T? _value; + + /// + /// Initializes a new instance. + /// + private Optional(T? value) + { + _value = value; + IsSpecified = true; + } + + /// + /// Gets whether a value was specified. + /// + public bool IsSpecified { get; } + + /// + /// Gets the specified value. + /// + /// The value is unspecified. + public T Value => IsSpecified ? _value! : throw new InvalidOperationException("The Optional value is unspecified."); + + /// + /// Creates an containing the specified . + /// + /// The value, which may be . + /// The specified optional value. + public static Optional Specified(T? value) => new(value); + + /// + /// Implicitly converts a value to a specified . + /// + /// The value, which may be . + public static implicit operator Optional(T? value) => Specified(value); + + /// + public bool Equals(Optional other) => IsSpecified == other.IsSpecified && (!IsSpecified || EqualityComparer.Default.Equals(_value, other._value)); + + /// + public override bool Equals(object? obj) => obj is Optional other && Equals(other); + + /// + public override int GetHashCode() => IsSpecified ? HashCode.Combine(true, _value) : 0; + + /// + /// Determines whether two optional values are equal. + /// + public static bool operator ==(Optional left, Optional right) => left.Equals(right); + + /// + /// Determines whether two optional values are not equal. + /// + public static bool operator !=(Optional left, Optional right) => !left.Equals(right); + + /// + Optional IWrappedValueCleaner>.Clean(CleanArgs args) => IsSpecified ? Specified(RuntimeMetadata.Clean(_value, args)) : this; +} diff --git a/src/CoreEx/Entities/README.md b/src/CoreEx/Entities/README.md index 4a5641371..9ae2f3fc1 100644 --- a/src/CoreEx/Entities/README.md +++ b/src/CoreEx/Entities/README.md @@ -21,6 +21,7 @@ The namespace also includes supporting utilities: `Cleaner` for normalizing stri - 📐 **Feature detection**: `FeatureSupport` and extension methods allow infrastructure code to detect which interfaces an entity implements and act accordingly (e.g. auto-populate ETags or change logs). - ✍️ **Write-intent descriptors**: `Writable` enum and `[Writable]` attribute annotate entity properties with create-only / update-only / never-writable semantics for OpenAPI documentation and tooling. - ✅ **Non-nullable intent**: `[NonNullable]` marks a property as mandatory/non-null for OpenAPI tooling even though it is declared nullable (`string?`, `int?`) in code for DTO flexibility. +- 📝 **Partial-update values**: `Optional` distinguishes an omitted request property from a supplied `null` or default value; use it on explicit patch DTOs, not persisted contracts. - 🔧 **Identifier generation**: `IdentifierGenerator` and `IIdentifierGenerator` provide DI-replaceable new `Guid`/`string` identifier creation (using `Guid.CreateVersion7()` on .NET 9+). ## Key types @@ -32,6 +33,7 @@ The namespace also includes supporting utilities: `Cleaner` for normalizing stri | **[`Cleaner`](./Cleaner.cs)** | Static utility applying configurable string trimming, transformation, casing, and `DateTime` normalization to entity values. | | **[`CompositeKey`](./CompositeKey.cs)** | Immutable struct representing a multi-part entity key with boxing-free generic `Create` overloads for up to four arguments. | | **[`CompositeKeyComparer`](./CompositeKeyComparer.cs)** | `IEqualityComparer` for use in dictionaries and collections keyed by `CompositeKey`. | +| **[`Optional`](./Optional.cs)** | Presence-aware value wrapper for partial-update request properties; `default` is unspecified, while `Specified(value)` preserves supplied `null` and default values. | | **[`DataMap`](./DataMap.cs)** | String-keyed dictionary whose keys are preserved as-is (no camelCase conversion) during JSON serialization; used for arbitrary data maps on entities. | | **[`ETag`](./ETag.cs)** | Static utility for comparing ETag strings and throwing `ConcurrencyException` on mismatch; also generates ETag hashes from entity values. | | **[`FeatureSupport`](./FeatureSupport.cs)** | Enum of flags indicating which entity contract interfaces a type implements, used by infrastructure to decide which fields to auto-populate. | @@ -52,6 +54,16 @@ The namespace also includes supporting utilities: `Cleaner` for normalizing stri | [`ICopyFrom`](./Extended/ICopyFrom.cs) | Interface enabling an entity to copy its state from another instance of itself. | | [`IDefault`](./Extended/IDefault.cs) | Interface enabling an entity to determine whether it is in its default (unset) state. | +## Optional request values + +Use `Optional` on a dedicated patch/input DTO when the request must distinguish a missing field from an explicitly supplied `null`, `0`, or `false`. A default `Optional` is unspecified; assignment or `Optional.Specified(value)` marks it present. Reading `.Value` while unspecified throws. + +The JSON converter uses the inner value's ordinary JSON shape. Missing properties remain unspecified, and explicit `null` is specified-null when `T` permits null. CoreEx's `JsonDefaults.SerializerOptions` omits unspecified wrappers through `WhenWritingDefault`; custom serializer options must also ignore default values when serializing an object property. + +Keep the wrapper at the request boundary. Validate supplied values with `CoreEx.Validation`'s `.Unwrap()` and map only properties where `IsSpecified` is true into ordinary contracts or domain objects. Do not add `Optional` to event, repository, or persistence models. + +Custom value wrappers, whether optional or always present, can independently opt into runtime cleaning by implementing `CoreEx.Entities.Abstractions.IWrappedValueCleaner`, where `TSelf` is the implementing wrapper type. This capability is separate from the read-only `IWrappedValue` and `IOptionalValue` contracts: `Clean(CleanArgs)` returns a reconstructed wrapper of the same runtime type while preserving its invariants. Optional implementations must also preserve presence even when the cleaned inner value is null, and must not read an unspecified value. Runtime cleaning writes the result back to writable properties; null results or a different wrapper type are rejected. Typed cleaning uses cached constrained dispatch to avoid boxing the wrapper receiver and result; reflection initializes the dispatcher once per type. Object/interface-typed boundaries still require runtime dispatch and can box struct wrappers, and cleaning the inner value may itself allocate. This does not automatically add JSON, source-generation or OpenAPI support for custom wrappers. + ## Related Namespaces - **[`CoreEx`](../README.md)** - Defines `ExecutionContext` and `AuthenticationUser` consumed by `ChangeLog` to populate audit fields. diff --git a/src/CoreEx/Json/JsonDefaults.cs b/src/CoreEx/Json/JsonDefaults.cs index 81472ae18..67ce6520f 100644 --- a/src/CoreEx/Json/JsonDefaults.cs +++ b/src/CoreEx/Json/JsonDefaults.cs @@ -38,7 +38,7 @@ public JsonDefaultConfiguration() WriteIndented = false, PropertyNamingPolicy = _jsonSubstituteNamingPolicy, DictionaryKeyPolicy = null, - Converters = { new JsonStringEnumConverter(), new JsonReferenceDataConverter(), new JsonResultConverter(), new JsonDataMapConverterFactory() } + Converters = { new JsonStringEnumConverter(), new JsonReferenceDataConverter(), new JsonResultConverter(), new JsonDataMapConverterFactory(), new OptionalJsonConverterFactory() } }; } diff --git a/src/CoreEx/Json/OptionalJsonConverterFactory.cs b/src/CoreEx/Json/OptionalJsonConverterFactory.cs new file mode 100644 index 000000000..cbb85bfdd --- /dev/null +++ b/src/CoreEx/Json/OptionalJsonConverterFactory.cs @@ -0,0 +1,37 @@ +namespace CoreEx.Json; + +/// +/// Creates JSON converters for values. +/// +public sealed class OptionalJsonConverterFactory : JsonConverterFactory +{ + /// + public override bool CanConvert(Type typeToConvert) => typeToConvert.IsGenericType && typeToConvert.GetGenericTypeDefinition() == typeof(Optional<>); + + /// + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + => (JsonConverter)Activator.CreateInstance(typeof(OptionalJsonConverter<>).MakeGenericType(typeToConvert.GetGenericArguments()[0]))!; + + /// + /// Converts an optional value using its wrapped value's JSON representation. + /// + /// The wrapped value type. + private sealed class OptionalJsonConverter : JsonConverter> + { + /// + public override bool HandleNull => true; + + /// + public override Optional Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => Optional.Specified(JsonSerializer.Deserialize(ref reader, options)); + + /// + public override void Write(Utf8JsonWriter writer, Optional value, JsonSerializerOptions options) + { + if (!value.IsSpecified) + throw new JsonException("An unspecified Optional value cannot be serialized as a standalone value."); + + JsonSerializer.Serialize(writer, value.Value, options); + } + } +} diff --git a/src/CoreEx/Metadata/PropertyRuntimeMetadata.cs b/src/CoreEx/Metadata/PropertyRuntimeMetadata.cs index 09eb6c673..4f8617aa5 100644 --- a/src/CoreEx/Metadata/PropertyRuntimeMetadata.cs +++ b/src/CoreEx/Metadata/PropertyRuntimeMetadata.cs @@ -20,6 +20,7 @@ public readonly struct PropertyRuntimeMetadata(string name, // Where the property type does not itself carry runtime metadata (i.e. is not an IContract-like graph type), Clean and CleanAndDefault have the same outcome (see CleanOption remarks). private static readonly bool _isRuntimeMetadataProperty = typeof(IRuntimeMetadataCore).IsAssignableFrom(typeof(TProperty)); + private static readonly bool _isWrappedValueCleaner = RuntimeMetadata.IsWrappedValueCleaner(typeof(TProperty)); private readonly Func _getValue = getValue.ThrowIfNull(); private readonly Action? _setValue = setValue; @@ -96,6 +97,9 @@ public void Clean(TEntity entity, CleanArgs args) return; var val = Cleaner.Clean(GetValue(entity), args); + if (_isWrappedValueCleaner && !IsReadOnly && val is TProperty wrappedValue) + SetValue(entity, wrappedValue); + if (IsReadOnly) return; diff --git a/src/CoreEx/Metadata/RuntimeMetadata.Clean.WrappedValue.cs b/src/CoreEx/Metadata/RuntimeMetadata.Clean.WrappedValue.cs new file mode 100644 index 000000000..974f6512b --- /dev/null +++ b/src/CoreEx/Metadata/RuntimeMetadata.Clean.WrappedValue.cs @@ -0,0 +1,49 @@ +namespace CoreEx.Metadata; + +public static partial class RuntimeMetadata +{ + private static readonly ConcurrentDictionary> _wrappedValueCleaners = new(); + private static readonly ConcurrentDictionary _wrappedValueCleanerTypes = new(); + + /// + /// Indicates whether a type implements its self-typed wrapper cleaning contract. + /// + internal static bool IsWrappedValueCleaner(Type type) => _wrappedValueCleanerTypes.GetOrAdd(type, + static t => t.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IWrappedValueCleaner<>) && i.GenericTypeArguments[0] == t)); + + /// + /// Cleans a wrapper through a constrained call without boxing its receiver or result. + /// + [return: NotNullIfNotNull(nameof(value))] + private static TWrapper? CleanWrapper(TWrapper? value, CleanArgs args) where TWrapper : IWrappedValueCleaner + { + if (!typeof(TWrapper).IsValueType && value is null) + return value; + + // Null references returned above; the remaining value-type path cannot be null. + var wrapper = value!; + var cleaned = wrapper.Clean(args); + if (!typeof(TWrapper).IsValueType && (cleaned is null || cleaned.GetType() != wrapper.GetType())) + throw new InvalidOperationException($"The wrapped-value cleaner for '{wrapper.GetType()}' must return a non-null value of the same runtime wrapper type."); + + return cleaned; + } + + /// + /// Adapts a wrapper whose type was erased by an object or interface boundary. + /// + private static object CleanErasedWrapper(object value, CleanArgs args) where TWrapper : IWrappedValueCleaner => CleanWrapper((TWrapper)value, args); + + /// + /// Caches typed wrapper dispatch once per declared value type. + /// + private static class WrappedValueCleaner + { + /// + /// Gets the constrained cleaning delegate when the type supports wrapper cleaning. + /// + internal static readonly Func? Clean = IsWrappedValueCleaner(typeof(T)) + ? typeof(RuntimeMetadata).GetMethod(nameof(CleanWrapper), BindingFlags.NonPublic | BindingFlags.Static)!.MakeGenericMethod(typeof(T)).CreateDelegate>() + : null; + } +} diff --git a/src/CoreEx/Metadata/RuntimeMetadata.Clean.cs b/src/CoreEx/Metadata/RuntimeMetadata.Clean.cs index 1cd26210f..0d9032480 100644 --- a/src/CoreEx/Metadata/RuntimeMetadata.Clean.cs +++ b/src/CoreEx/Metadata/RuntimeMetadata.Clean.cs @@ -16,6 +16,9 @@ public static partial class RuntimeMetadata /// this is unable to be cleaned/replaced. An empty array, collection, or dictionary will be set to . public static T? Clean(T? value, CleanArgs args = default) { + if (WrappedValueCleaner.Clean is { } cleanWrapper) + return cleanWrapper(value!, args); + if (value is string str) return Internal.Cast(Cleaner.Clean(str, Cleaner.DefaultStringTrim, Cleaner.DefaultStringTransform, Cleaner.DefaultStringCase)!); @@ -25,6 +28,16 @@ public static partial class RuntimeMetadata if (value is DateTime dt) return Internal.Cast(Cleaner.Clean(dt, Cleaner.DefaultDateTimeTransform)); + var runtimeType = value.GetType(); + if (runtimeType != typeof(T) && IsWrappedValueCleaner(runtimeType)) + { + var clean = _wrappedValueCleaners.GetOrAdd(runtimeType, static type => + typeof(RuntimeMetadata).GetMethod(nameof(CleanErasedWrapper), BindingFlags.NonPublic | BindingFlags.Static)! + .MakeGenericMethod(type).CreateDelegate>()); + + return Internal.Cast(clean(value, args)); + } + // All reference-type branches below can form cycles — allocate the visited set once per thread and reuse it. var set = _visitingForClean ??= new HashSet(ReferenceEqualityComparer.Instance); var isRoot = set.Count == 0; diff --git a/src/CoreEx/README.md b/src/CoreEx/README.md index 238cf11f1..541bfc153 100644 --- a/src/CoreEx/README.md +++ b/src/CoreEx/README.md @@ -24,6 +24,7 @@ All other CoreEx packages (`CoreEx.Events`, `CoreEx.Database`, `CoreEx.AspNetCor - 🔐 **Execution context**: An `AsyncLocal`-scoped `ExecutionContext` that carries user identity (`AuthenticationUser`), tenant ID, operation type, request timestamp, and arbitrary attributes throughout the lifetime of a request. - 🚂 **Railway-oriented results**: `Result` and `Result` value types that represent success or a typed error without throwing, enabling functional pipeline-style error propagation for expected business outcomes. - 📦 **Entity contracts**: Interfaces and types for identifiers (`IIdentifier`), ETags (`IETag`), audit change logs (`IChangeLog`), composite keys (`CompositeKey`), and entity cleaning/transformation — used as the common contract shape across all CoreEx layers. +- 📝 **Partial-update values**: `Optional` preserves omitted-versus-supplied intent for explicit request/patch DTOs without changing ordinary contracts or persistence models. - 🗺️ **Explicit mapping**: Uni- and Bi-directional mapper base classes and a `Mapper` utility for propagating standard contract properties (identity, ETags, change logs, tenant, partition) between source and destination types without a reflection-heavy mapping library. - 🔄 **JSON utilities**: RFC 7396 JSON Merge Patch (`JsonMergePatch`), include/exclude JSON property filtering (`JsonFilter`), and custom converters for common CoreEx types. - 💉 **DI lifetime attributes**: `[ScopedService]`, `[SingletonService]`, and `[TransientService]` attributes that mark implementation types for automatic discovery and registration via `AddDynamicServicesUsing`. @@ -51,6 +52,12 @@ All other CoreEx packages (`CoreEx.Events`, `CoreEx.Database`, `CoreEx.AspNetCor See the [Extended Exceptions](#extended-exceptions) section below for the full list of semantic exception types. +## Optional patch values + +For partial-update request DTOs, `Optional` distinguishes an omitted property from a supplied `null` or default value. `default(Optional)` is unspecified; implicit conversion from `T` and `Optional.Specified(value)` create a specified value. Its JSON converter reads and writes the inner value shape, while CoreEx's `WhenWritingDefault` serializer setting omits unspecified properties. OpenAPI exposes the inner type and nullability while leaving the property non-required. + +Keep `Optional` on the request boundary: validate with `.Unwrap()` and copy only specified values into the ordinary contract/domain object. It is not a persistence or event-model type. See [contract patch guidance](../../.github/instructions/coreex-contracts.instructions.md#optional-patch-properties) and [optional validation](../CoreEx.Validation/README.md#optional-patch-validation). + ## Validating HTTP response values HTTP success and JSON deserialization do not guarantee a valid dependency response. Use diff --git a/tests/CoreEx.AspNetCore.Test.Api/Controllers/OtherController.cs b/tests/CoreEx.AspNetCore.Test.Api/Controllers/OtherController.cs index 873f1c4eb..b62cfed74 100644 --- a/tests/CoreEx.AspNetCore.Test.Api/Controllers/OtherController.cs +++ b/tests/CoreEx.AspNetCore.Test.Api/Controllers/OtherController.cs @@ -1,5 +1,6 @@ using CoreEx.AspNetCore.Mvc; using CoreEx.AspNetCore.Test.Api.Entities; +using CoreEx.Http; using Microsoft.AspNetCore.Mvc; namespace CoreEx.AspNetCore.Test.Api.Controllers; @@ -12,6 +13,11 @@ public class OtherController(WebApi webApi) : ControllerBase [ProducesResponseType(204)] public IActionResult OptionalBody() => NoContent(); + [HttpPatch("optional-patch")] + [Accepts(HttpNames.MergePatchJsonMediaTypeName)] + [ProducesResponseType(204)] + public IActionResult OptionalPatch() => NoContent(); + [HttpPost("non-nullable-map")] [Accepts>] [ProducesResponseType(204)] diff --git a/tests/CoreEx.AspNetCore.Test.Api/Entities/OptionalPatchRequest.cs b/tests/CoreEx.AspNetCore.Test.Api/Entities/OptionalPatchRequest.cs new file mode 100644 index 000000000..a9c4c283a --- /dev/null +++ b/tests/CoreEx.AspNetCore.Test.Api/Entities/OptionalPatchRequest.cs @@ -0,0 +1,42 @@ +using CoreEx.Entities; +using CoreEx.RefData; + +namespace CoreEx.AspNetCore.Test.Api.Entities; + +/// +/// Represents a partial-update request used to verify optional contract and schema behavior. +/// +[Contract] +public partial class OptionalPatchRequest +{ + /// + /// Gets or sets the non-nullable title when supplied. + /// + public Optional Title { get; set; } + + /// + /// Gets or sets the nullable notes when supplied. + /// + public Optional Notes { get; set; } + + /// + /// Gets or sets the non-nullable count when supplied. + /// + public Optional Count { get; set; } + + /// + /// Gets or sets the nullable count when supplied. + /// + public Optional NullableCount { get; set; } + + /// + /// Gets or sets the tags when supplied. + /// + public Optional> Tags { get; set; } + + /// + /// Gets or sets the gender reference-data code when supplied. + /// + [ReferenceData(Text = false)] + public partial Optional GenderCode { get; set; } +} diff --git a/tests/CoreEx.AspNetCore.Test.Unit/OtherApiTests.cs b/tests/CoreEx.AspNetCore.Test.Unit/OtherApiTests.cs index 7d47c5f9c..0d55522ff 100644 --- a/tests/CoreEx.AspNetCore.Test.Unit/OtherApiTests.cs +++ b/tests/CoreEx.AspNetCore.Test.Unit/OtherApiTests.cs @@ -1,5 +1,7 @@ using CoreEx.AspNetCore.Idempotency; +using CoreEx.AspNetCore.Test.Api.Entities; using CoreEx.Caching; +using CoreEx.Entities; using CoreEx.Http; using Microsoft.AspNetCore.Mvc; using System.Net; @@ -160,6 +162,50 @@ public void Swagger_JSON_NonNullableAttribute_DataMapValue_NotNullable() props.GetProperty("optionalCode").GetProperty("nullable").GetBoolean().Should().BeTrue(); } + [Test] + public void Swagger_JSON_OptionalProperties_AreNotRequired_AndKeepInnerNullability() + { + var json = Test.Http() + .Run(HttpMethod.Get, "/swagger/v1/swagger.json") + .Assert(HttpStatusCode.OK) + .GetContent(); + + using var doc = JsonDocument.Parse(json!); + var schema = doc.RootElement.GetProperty("components").GetProperty("schemas").GetProperty("OptionalPatchRequest"); + var properties = schema.GetProperty("properties"); + + schema.TryGetProperty("required", out _).Should().BeFalse(); + properties.GetProperty("title").GetProperty("type").GetString().Should().Be("string"); + properties.GetProperty("title").GetProperty("nullable").GetBoolean().Should().BeFalse(); + properties.GetProperty("notes").GetProperty("nullable").GetBoolean().Should().BeTrue(); + properties.GetProperty("count").GetProperty("type").GetString().Should().Be("integer"); + properties.GetProperty("count").GetProperty("nullable").GetBoolean().Should().BeFalse(); + properties.GetProperty("nullableCount").GetProperty("type").EnumerateArray().Select(x => x.GetString()).Should().BeEquivalentTo("integer", "null"); + properties.GetProperty("nullableCount").GetProperty("nullable").GetBoolean().Should().BeTrue(); + properties.GetProperty("tags").GetProperty("type").GetString().Should().Be("array"); + properties.GetProperty("tags").GetProperty("items").GetProperty("type").GetString().Should().Be("string"); + properties.GetProperty("gender").GetProperty("type").EnumerateArray().Select(x => x.GetString()).Should().BeEquivalentTo("null", "string"); + properties.GetProperty("gender").GetProperty("nullable").GetBoolean().Should().BeTrue(); + properties.TryGetProperty("genderCode", out _).Should().BeFalse(); + } + + [Test] + public void OptionalReferenceDataNavigation_SharesCodePresence() + { + var request = new OptionalPatchRequest(); + + request.Gender.IsSpecified.Should().BeFalse(); + request.Gender = Optional.Specified(null); + request.GenderCode.IsSpecified.Should().BeTrue(); + request.GenderCode.Value.Should().BeNull(); + request.Gender = default; + request.GenderCode.IsSpecified.Should().BeFalse(); + + request.GenderCode = Optional.Specified(null); + request.Gender.IsSpecified.Should().BeTrue(); + request.Gender.Value.Should().BeNull(); + } + [TestCase("/health/live")] [TestCase("/health/startup")] [TestCase("/health/ready")] diff --git a/tests/CoreEx.Test.Unit/Entities/OptionalTests.cs b/tests/CoreEx.Test.Unit/Entities/OptionalTests.cs new file mode 100644 index 000000000..9c5f47f0b --- /dev/null +++ b/tests/CoreEx.Test.Unit/Entities/OptionalTests.cs @@ -0,0 +1,105 @@ +namespace CoreEx.Test.Unit.Entities; + +[TestFixture] +public class OptionalTests +{ + [Test] + public void Default_IsUnspecified_AndValueThrows() + { + var value = default(Optional); + + value.IsSpecified.Should().BeFalse(); + Action act = () => _ = value.Value; + act.Should().Throw(); + } + + [Test] + public void ImplicitConversion_MarksDefaultAndNullValuesSpecified() + { + Optional number = 0; + Optional text = null; + + number.IsSpecified.Should().BeTrue(); + number.Value.Should().Be(0); + text.IsSpecified.Should().BeTrue(); + text.Value.Should().BeNull(); + } + + [Test] + public void Equality_IncludesSpecifiedState() + { + Optional unspecified = default; + Optional specifiedNull = Optional.Specified(null); + + unspecified.Should().NotBe(specifiedNull); + specifiedNull.Should().Be(Optional.Specified(null)); + Optional.Specified(0).Should().NotBe(default(Optional)); + } + + [Test] + public void Clean_CleansSpecifiedInnerValue_AndLeavesUnspecifiedUntouched() + { + var entity = new OptionalContainer { Value = Optional.Specified(" text ") }; + RuntimeMetadata.Clean(entity); + var unspecified = Cleaner.Clean(default(Optional)); + + entity.Value.IsSpecified.Should().BeTrue(); + entity.Value.Value.Should().Be(" text"); + unspecified.IsSpecified.Should().BeFalse(); + } + + public class OptionalContainer + { + public Optional Value { get; set; } + } + + [TestCase(false, null)] + [TestCase(true, null)] + [TestCase(true, " text ")] + public void CustomCleaner_ParticipatesInDirectAndPropertyCleaning(bool specified, string? text) + { + var value = new CustomOptional(specified, text); + var direct = RuntimeMetadata.Clean(value); + var entity = new CustomContainer { Value = value }; + RuntimeMetadata.Clean(entity); + + direct.IsSpecified.Should().Be(specified); + entity.Value.IsSpecified.Should().Be(specified); + if (specified) + { + direct.Value.Should().Be(text is null ? null : " text"); + entity.Value.Value.Should().Be(direct.Value); + } + } + + [TestCase(false)] + [TestCase(true)] + public void CustomCleaner_RejectsInvalidReconstruction(bool returnNull) + { + Action clean = () => RuntimeMetadata.Clean(new InvalidOptionalCleaner(returnNull)); + clean.Should().Throw() + .WithMessage("*must return a non-null value of the same runtime wrapper type."); + } + + public readonly record struct CustomOptional(bool IsSpecified, string? InnerValue) + : CoreEx.Entities.Abstractions.IOptionalValue, CoreEx.Entities.Abstractions.IWrappedValueCleaner + { + public string? Value => IsSpecified ? InnerValue : throw new InvalidOperationException("An unspecified value must not be read."); + + CustomOptional CoreEx.Entities.Abstractions.IWrappedValueCleaner.Clean(CleanArgs args) + => IsSpecified ? new CustomOptional(true, RuntimeMetadata.Clean(Value, args)) : this; + } + + public class CustomContainer + { + public CustomOptional Value { get; set; } + } + + private class InvalidOptionalCleaner(bool returnNull) : CoreEx.Entities.Abstractions.IWrappedValueCleaner + { + // Deliberately violate the non-null return contract to verify runtime enforcement. + public InvalidOptionalCleaner Clean(CleanArgs args) => returnNull ? null! : new DifferentWrapper(); + } + + private sealed class DifferentWrapper() : InvalidOptionalCleaner(false); +} diff --git a/tests/CoreEx.Test.Unit/Entities/WrappedValueCleanerTests.cs b/tests/CoreEx.Test.Unit/Entities/WrappedValueCleanerTests.cs new file mode 100644 index 000000000..c97532ecc --- /dev/null +++ b/tests/CoreEx.Test.Unit/Entities/WrappedValueCleanerTests.cs @@ -0,0 +1,94 @@ +namespace CoreEx.Test.Unit.Entities; + +[TestFixture] +public class WrappedValueCleanerTests +{ + [TestCase(false)] + [TestCase(true)] + public void TypedOptionalCleaning_DoesNotBoxWrappers(bool specified) + { + var value = specified ? Optional.Specified("text") : default; + for (var i = 0; i < 1000; i++) + value = Cleaner.Clean(value); + + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 10000; i++) + value = Cleaner.Clean(value); + + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + allocated.Should().Be(0); + value.IsSpecified.Should().Be(specified); + if (specified) + value.Value.Should().Be("text"); + } + + [Test] + public void TypedCustomStructCleaning_DoesNotBoxWrappers() + { + var value = new TextValue("text"); + for (var i = 0; i < 1000; i++) + value = RuntimeMetadata.Clean(value); + + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 10000; i++) + value = RuntimeMetadata.Clean(value); + + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + allocated.Should().Be(0); + value.Value.Should().Be("text"); + } + + [Test] + public void ErasedWrapperCleaning_PreservesRuntimeDispatch() + { + object value = new TextValue("text "); + CoreEx.Entities.Abstractions.IWrappedValue reference = new TextReference("text "); + var cleaned = RuntimeMetadata.Clean(value); + var cleanedReference = RuntimeMetadata.Clean(reference); + + cleaned.Should().BeOfType().Which.Value.Should().Be("text"); + cleanedReference.Should().BeOfType().Which.Value.Should().Be("text"); + RuntimeMetadata.Clean(null).Should().BeNull(); + } + + [TestCase(null)] + [TestCase(" text ")] + public void NonOptionalWrappers_CleanDirectlyAndWriteBackToProperties(string? text) + { + var entity = new Container { StructValue = new(text), ReferenceValue = new(text) }; + var originalReference = entity.ReferenceValue; + var directStruct = Cleaner.Clean(entity.StructValue); + var directReference = Cleaner.Clean(entity.ReferenceValue); + + RuntimeMetadata.Clean(entity); + + var expected = Cleaner.Clean(text); + directStruct.Value.Should().Be(expected); + directReference.Should().NotBeNull(); + directReference!.Value.Should().Be(expected); + entity.StructValue.Value.Should().Be(expected); + entity.ReferenceValue.Value.Should().Be(expected); + entity.ReferenceValue.Should().NotBeSameAs(originalReference); + originalReference.Value.Should().Be(text); + } + + public readonly record struct TextValue(string? Value) + : CoreEx.Entities.Abstractions.IWrappedValue, CoreEx.Entities.Abstractions.IWrappedValueCleaner + { + TextValue CoreEx.Entities.Abstractions.IWrappedValueCleaner.Clean(CleanArgs args) + => new TextValue(RuntimeMetadata.Clean(Value, args)); + } + + public sealed record TextReference(string? Value) + : CoreEx.Entities.Abstractions.IWrappedValue, CoreEx.Entities.Abstractions.IWrappedValueCleaner + { + TextReference CoreEx.Entities.Abstractions.IWrappedValueCleaner.Clean(CleanArgs args) + => new TextReference(RuntimeMetadata.Clean(Value, args)); + } + + public class Container + { + public TextValue StructValue { get; set; } + public TextReference ReferenceValue { get; set; } = new(null); + } +} diff --git a/tests/CoreEx.Test.Unit/Json/OptionalJsonConverterTests.cs b/tests/CoreEx.Test.Unit/Json/OptionalJsonConverterTests.cs new file mode 100644 index 000000000..fbf2d92c7 --- /dev/null +++ b/tests/CoreEx.Test.Unit/Json/OptionalJsonConverterTests.cs @@ -0,0 +1,48 @@ +namespace CoreEx.Test.Unit.Json; + +[TestFixture] +public class OptionalJsonConverterTests +{ + [Test] + public void ContainingObject_OmitsUnspecifiedAndPreservesSpecifiedNullAndDefault() + { + var options = new CoreEx.Json.JsonDefaults.JsonDefaultConfiguration().SerializerOptions; + var target = new OptionalDocument + { + Name = Optional.Specified(null), + Count = 0 + }; + + JsonSerializer.Serialize(target, options).Should().Be("""{"name":null,"count":0}"""); + } + + [Test] + public void ContainingObject_DeserializationDistinguishesMissingNullAndDefault() + { + var options = new CoreEx.Json.JsonDefaults.JsonDefaultConfiguration().SerializerOptions; + var target = JsonSerializer.Deserialize("""{"name":null,"count":0}""", options); + + target.Should().NotBeNull(); + target!.Name.IsSpecified.Should().BeTrue(); + target.Name.Value.Should().BeNull(); + target.Count.IsSpecified.Should().BeTrue(); + target.Count.Value.Should().Be(0); + target.Description.IsSpecified.Should().BeFalse(); + } + + [Test] + public void Converter_UsesInnerScalarRepresentation() + { + var options = new CoreEx.Json.JsonDefaults.JsonDefaultConfiguration().SerializerOptions; + + JsonSerializer.Serialize(Optional.Specified(12), options).Should().Be("12"); + JsonSerializer.Deserialize>("null", options).Should().Be(Optional.Specified(null)); + } + + public class OptionalDocument + { + public Optional Name { get; set; } + public Optional Count { get; set; } + public Optional Description { get; set; } + } +} diff --git a/tests/CoreEx.Validation.Test.Unit/Helper.cs b/tests/CoreEx.Validation.Test.Unit/Helper.cs index 4b6896c9d..5cf7f85b8 100644 --- a/tests/CoreEx.Validation.Test.Unit/Helper.cs +++ b/tests/CoreEx.Validation.Test.Unit/Helper.cs @@ -2,6 +2,7 @@ using CoreEx.Invokers; using CoreEx.Validation.Abstractions; using System.Text.Json; +using System.Text.Json.Serialization; namespace CoreEx.Validation.Test.Unit; @@ -14,7 +15,7 @@ private static JsonSerializerOptions GetJsonSerializerOptions() if (_jsonSerializerOptions is not null) return _jsonSerializerOptions; - _jsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true }; + _jsonSerializerOptions = new JsonSerializerOptions { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault, WriteIndented = true }; _jsonSerializerOptions.Converters.Add(new IgnoreTypeConverter()); _jsonSerializerOptions.Converters.Add(new IgnoreTypeConverter()); _jsonSerializerOptions.Converters.Add(new IgnoreTypeConverter()); diff --git a/tests/CoreEx.Validation.Test.Unit/Rules/ChainRuleTests.cs b/tests/CoreEx.Validation.Test.Unit/Rules/ChainRuleTests.cs new file mode 100644 index 000000000..6eaeb1909 --- /dev/null +++ b/tests/CoreEx.Validation.Test.Unit/Rules/ChainRuleTests.cs @@ -0,0 +1,44 @@ +namespace CoreEx.Validation.Test.Unit.Rules; + +[TestFixture] +public class ChainRuleTests +{ + [Test] + public void Chain_PreservesNextRuleTypeForNullableAndNonNullableInputs() + { + var validator = Validator.Create() + .HasProperty(x => x.Text, rule => + { + CoreEx.Validation.Rules.IPropertyRule next = ValidationExtensions.Chain(rule, new CoreEx.Validation.Rules.StringRule(maxLength: _ => 3)); + next.Between("aaa", "zzz"); + }) + .HasProperty(x => x.NullableText, rule => + { + CoreEx.Validation.Rules.IPropertyRule next = ValidationExtensions.Chain(rule, new CoreEx.Validation.Rules.StringRule(maxLength: _ => 3)); + next.Between("aaa", "zzz"); + }) + .HasProperty(x => x.Number, rule => + { + CoreEx.Validation.Rules.IPropertyRule next = ValidationExtensions.Chain(rule, new CoreEx.Validation.Rules.MandatoryRule(_ => false, _ => false)); + next.Between(0, 3); + }) + .HasProperty(x => x.NullableNumber, rule => + { + CoreEx.Validation.Rules.IPropertyRule next = ValidationExtensions.Chain(rule, new CoreEx.Validation.Rules.MandatoryRule(_ => false, _ => false)); + next.Between(0, 3); + }); + + validator.ValidateAsSuccess(new Request()); + validator.ValidateAsSuccess(new Request { NullableText = "abc", NullableNumber = 0 }); + validator.ValidateAsError(new Request { NullableText = "long" }, "nullableText", "Nullable text must not exceed 3 character(s) in length."); + validator.ValidateAsError(new Request { NullableNumber = 4 }, "nullableNumber", "Nullable number must be between '0' and '3'."); + } + + public class Request + { + public string Text { get; set; } = "abc"; + public string? NullableText { get; set; } + public int Number { get; set; } + public int? NullableNumber { get; set; } = 2; + } +} diff --git a/tests/CoreEx.Validation.Test.Unit/Rules/OptionalRuleTests.cs b/tests/CoreEx.Validation.Test.Unit/Rules/OptionalRuleTests.cs new file mode 100644 index 000000000..da896ec32 --- /dev/null +++ b/tests/CoreEx.Validation.Test.Unit/Rules/OptionalRuleTests.cs @@ -0,0 +1,222 @@ +using CoreEx.Entities; + +namespace CoreEx.Validation.Test.Unit.Rules; + +[TestFixture] +public class OptionalRuleTests +{ + [Test] + public void Unwrap_SkipsUnspecifiedValues_AndValidatesSpecifiedValues() + { + var validator = Validator.Create() + .HasProperty(x => x.Notes, rule => rule.Unwrap().String(3)); + + validator.ValidateAsSuccess(new PatchRequest()); + validator.ValidateAsSuccess(new PatchRequest { Notes = "abc" }); + validator.ValidateAsError(new PatchRequest { Notes = "long" }, "notes", "Notes must not exceed 3 character(s) in length."); + } + + [Test] + public void Unwrap_ValidatesNullableStringsWithoutAnnotationWarnings() + { + var validator = Validator.Create() + .HasProperty(x => x.NullableNotes, rule => rule.Unwrap().String(3)); + + validator.ValidateAsSuccess(new PatchRequest()); + validator.ValidateAsSuccess(new PatchRequest { NullableNotes = Optional.Specified(null) }); + validator.ValidateAsError(new PatchRequest { NullableNotes = "long" }, "nullableNotes", "Nullable notes must not exceed 3 character(s) in length."); + } + + [Test] + public void Unwrap_StringOverloadsAndSubsequentRules_AcceptNullableStrings() + { + var regex = new System.Text.RegularExpressions.Regex("^[a-z]+$", System.Text.RegularExpressions.RegexOptions.None, TimeSpan.FromSeconds(1)); + var common = Validator.CreateCommon(rule => rule.String(3)); + var validator = Validator.Create() + .HasProperty(x => x.NullableNotes, rule => rule.Unwrap().Common(common) + .String(3).String(1, 3).String(_ => 1, _ => 3, _ => regex) + .String(regex).Matches(regex).Length(3).MinimumLength(1).MaximumLength(3) + .Between("aaa", "zzz").Common(common)); + + validator.ValidateAsSuccess(new PatchRequest()); + validator.ValidateAsSuccess(new PatchRequest { NullableNotes = Optional.Specified(null) }); + validator.ValidateAsSuccess(new PatchRequest { NullableNotes = "abc" }); + validator.ValidateAsError(new PatchRequest { NullableNotes = "long" }, "nullableNotes", "Nullable notes must not exceed 3 character(s) in length."); + } + + [Test] + public void Unwrap_CommonAndBetween_HandleNullableAndNonNullableNumbers() + { + var common = Validator.CreateCommon(rule => rule.Between(0, 3)); + var numberValidator = Validator.Create() + .HasProperty(x => x.Count, rule => rule.Unwrap().Common(common).Between(0, 3)); + var nullableValidator = Validator.Create() + .HasProperty(x => x.NullableCount, rule => rule.Unwrap().Common(common).Between(0, 3)); + + numberValidator.ValidateAsSuccess(new PatchRequest()); + numberValidator.ValidateAsSuccess(new PatchRequest { Count = 0 }); + numberValidator.ValidateAsSuccess(new PatchRequest { Count = 2 }); + numberValidator.ValidateAsError(new PatchRequest { Count = 4 }, "count", "Count must be between '0' and '3'."); + nullableValidator.ValidateAsSuccess(new PatchRequest()); + nullableValidator.ValidateAsSuccess(new PatchRequest { NullableCount = Optional.Specified(null) }); + nullableValidator.ValidateAsSuccess(new PatchRequest { NullableCount = 0 }); + nullableValidator.ValidateAsSuccess(new PatchRequest { NullableCount = 2 }); + nullableValidator.ValidateAsError(new PatchRequest { NullableCount = 4 }, "nullableCount", "Nullable count must be between '0' and '3'."); + } + + [Test] + public void Unwrap_NullableNumberRange_PreservesNullAndZero() + { + var validator = Validator.Create() + .HasProperty(x => x.NullableCount, rule => rule.Unwrap().Between(0, 3)); + + validator.ValidateAsSuccess(new PatchRequest()); + validator.ValidateAsSuccess(new PatchRequest { NullableCount = Optional.Specified(null) }); + validator.ValidateAsSuccess(new PatchRequest { NullableCount = 0 }); + validator.ValidateAsError(new PatchRequest { NullableCount = 4 }, "nullableCount", "Nullable count must be between '0' and '3'."); + } + + [Test] + public void Specified_AcceptsSuppliedDefaultsForAllFourTypes() + { + var validator = Validator.Create() + .HasProperty(x => x.Notes, rule => rule.Specified()) + .HasProperty(x => x.NullableNotes, rule => rule.Specified()) + .HasProperty(x => x.Count, rule => rule.Specified()) + .HasProperty(x => x.NullableCount, rule => rule.Specified()); + + validator.ValidateAsSuccess(new PatchRequest + { + Notes = "", + NullableNotes = Optional.Specified(null), + Count = 0, + NullableCount = Optional.Specified(null) + }); + } + + [Test] + public void Unwrap_CommonMandatory_PreservesNullableNumericDefaultSemantics() + { + var common = Validator.CreateCommon(rule => rule.Mandatory()); + var validator = Validator.Create() + .HasProperty(x => x.NullableCount, rule => rule.Unwrap().Common(common)); + + validator.ValidateAsSuccess(new PatchRequest()); + validator.ValidateAsError(new PatchRequest { NullableCount = Optional.Specified(null) }, "nullableCount", "Nullable count is required."); + validator.ValidateAsError(new PatchRequest { NullableCount = 0 }, "nullableCount", "Nullable count is required."); + validator.ValidateAsSuccess(new PatchRequest { NullableCount = 2 }); + } + + [Test] + public void Unwrap_NullableNumberAfterRange_PreservesDefaultSemantics() + { + var validator = Validator.Create() + .HasProperty(x => x.NullableCount, rule => rule.Unwrap().Between(0, 3).Mandatory()); + var notNull = Validator.Create() + .HasProperty(x => x.NullableCount, rule => rule.Unwrap().NotNull()); + + validator.ValidateAsError(new PatchRequest { NullableCount = 0 }, "nullableCount", "Nullable count is required."); + validator.ValidateAsError(new PatchRequest { NullableCount = Optional.Specified(null) }, "nullableCount", "Nullable count is required."); + validator.ValidateAsSuccess(new PatchRequest { NullableCount = 2 }); + notNull.ValidateAsSuccess(new PatchRequest { NullableCount = 0 }); + notNull.ValidateAsError(new PatchRequest { NullableCount = Optional.Specified(null) }, "nullableCount", "Nullable count is required."); + } + + [Test] + public void SpecifiedAndUnwrap_PreservePresenceAndMandatorySemanticsForAllFourTypes() + { + var text = Validator.Create().HasProperty(x => x.Notes, rule => rule.Specified().Unwrap().Mandatory().String(3)); + var nullableText = Validator.Create().HasProperty(x => x.NullableNotes, rule => rule.Specified().Unwrap().Mandatory().String(3)); + var number = Validator.Create().HasProperty(x => x.Count, rule => rule.Specified().Unwrap().Mandatory()); + var nullableNumber = Validator.Create().HasProperty(x => x.NullableCount, rule => rule.Specified().Unwrap().Mandatory()); + + text.ValidateAsError(new PatchRequest(), "notes", "Notes is required."); + nullableText.ValidateAsError(new PatchRequest(), "nullableNotes", "Nullable notes is required."); + number.ValidateAsError(new PatchRequest(), "count", "Count is required."); + nullableNumber.ValidateAsError(new PatchRequest(), "nullableCount", "Nullable count is required."); + text.ValidateAsSuccess(new PatchRequest { Notes = "abc" }); + text.ValidateAsError(new PatchRequest { Notes = "" }, "notes", "Notes is required."); + nullableText.ValidateAsSuccess(new PatchRequest { NullableNotes = "abc" }); + nullableText.ValidateAsError(new PatchRequest { NullableNotes = Optional.Specified(null) }, "nullableNotes", "Nullable notes is required."); + number.ValidateAsError(new PatchRequest { Count = 0 }, "count", "Count is required."); + number.ValidateAsSuccess(new PatchRequest { Count = 2 }); + nullableNumber.ValidateAsError(new PatchRequest { NullableCount = Optional.Specified(null) }, "nullableCount", "Nullable count is required."); + nullableNumber.ValidateAsError(new PatchRequest { NullableCount = 0 }, "nullableCount", "Nullable count is required."); + nullableNumber.ValidateAsSuccess(new PatchRequest { NullableCount = 2 }); + } + + [Test] + public void Unwrap_MandatoryValidatesSpecifiedNullAndDefaultValues() + { + var textValidator = Validator.Create() + .HasProperty(x => x.NullableNotes, rule => rule.Unwrap().Mandatory()); + var numberValidator = Validator.Create() + .HasProperty(x => x.Count, rule => rule.Unwrap().Mandatory()); + + textValidator.ValidateAsSuccess(new PatchRequest()); + textValidator.ValidateAsError(new PatchRequest { NullableNotes = Optional.Specified(null) }, "nullableNotes", "Nullable notes is required."); + numberValidator.ValidateAsSuccess(new PatchRequest()); + numberValidator.ValidateAsError(new PatchRequest { Count = 0 }, "count", "Count is required."); + } + + [Test] + public void Specified_RequiresPresenceWithoutRejectingSpecifiedNull() + { + var validator = Validator.Create() + .HasProperty(x => x.NullableNotes, rule => rule.Specified()); + + validator.ValidateAsError(new PatchRequest(), "nullableNotes", "Nullable notes is required."); + validator.ValidateAsSuccess(new PatchRequest { NullableNotes = Optional.Specified(null) }); + } + + [Test] + public void Specified_ChainsInnerValidation() + { + var validator = Validator.Create() + .HasProperty(x => x.Notes, rule => rule.Specified().Unwrap().String(3)); + + validator.ValidateAsError(new PatchRequest(), "notes", "Notes is required."); + validator.ValidateAsSuccess(new PatchRequest { Notes = "abc" }); + validator.ValidateAsError(new PatchRequest { Notes = "long" }, "notes", "Notes must not exceed 3 character(s) in length."); + } + + [Test] + public void Specified_SupportsCustomMessagesAndConditionalClauses() + { + var validator = Validator.Create() + .HasProperty(x => x.Notes, rule => rule.Specified().WithMessage("Supply notes.").WhenEntity(x => x.Count.IsSpecified)); + + validator.ValidateAsSuccess(new PatchRequest()); + validator.ValidateAsError(new PatchRequest { Count = 0 }, "notes", "Supply notes."); + validator.ValidateAsSuccess(new PatchRequest { Count = 0, Notes = "abc" }); + } + + [Test] + public void Unwrap_PreservesOuterPropertyTextAndPath() + { + var validator = Validator.Create() + .HasProperty(x => x.Notes, rule => rule.WithText("Additional notes").Unwrap().String(3)); + + validator.ValidateAsError(new PatchRequest { Notes = "long" }, "notes", "Additional notes must not exceed 3 character(s) in length."); + } + + [Test] + public void Unwrap_ValidatesOptionalCollections() + { + var validator = Validator.Create() + .HasProperty(x => x.Numbers, rule => rule.Unwrap().Collection(maxCount: 2)); + + validator.ValidateAsSuccess(new PatchRequest()); + validator.ValidateAsSuccess(new PatchRequest { Numbers = new List { 1, 2 } }); + validator.ValidateAsError(new PatchRequest { Numbers = new List { 1, 2, 3 } }, "numbers", "Numbers must not exceed 2 item(s)."); + } + + public class PatchRequest + { + public Optional Notes { get; set; } + public Optional NullableNotes { get; set; } + public Optional Count { get; set; } + public Optional NullableCount { get; set; } + public Optional> Numbers { get; set; } + } +} diff --git a/tests/CoreEx.Validation.Test.Unit/Rules/UnwrapRuleTests.cs b/tests/CoreEx.Validation.Test.Unit/Rules/UnwrapRuleTests.cs new file mode 100644 index 000000000..b281700dd --- /dev/null +++ b/tests/CoreEx.Validation.Test.Unit/Rules/UnwrapRuleTests.cs @@ -0,0 +1,91 @@ +namespace CoreEx.Validation.Test.Unit.Rules; + +[TestFixture] +public class UnwrapRuleTests +{ + [Test] + public void WrapperExtensions_ExposeOnlyWrapperSpecificReceivers() + { + var methods = typeof(ValidationExtensions).GetMethods() + .Where(method => method.Name is "Unwrap" or "Specified") + .ToArray(); + + methods.Should().HaveCount(6); + foreach (var method in methods) + { + var propertyType = method.GetParameters()[0].ParameterType.GetGenericArguments()[1]; + var isExtension = method.IsDefined(typeof(System.Runtime.CompilerServices.ExtensionAttribute), false); + isExtension.Should().Be(!propertyType.IsGenericParameter); + } + } + + [Test] + public void Unwrap_InfersCustomReferenceWrapperValueType() + { + var validator = Validator.Create() + .HasProperty(x => x.Text, rule => rule.WithText("Wrapped text").Unwrap().String(3)); + + validator.ValidateAsSuccess(new Request { Text = new TextWrapper("abc") }); + validator.ValidateAsError(new Request { Text = new TextWrapper("long") }, "text", "Wrapped text must not exceed 3 character(s) in length."); + } + + [Test] + public void Unwrap_SupportsCustomStructWrappers() + { + var validator = Validator.Create() + .HasProperty(x => x.Number, rule => ValidationExtensions.Unwrap(rule).Between(1, 3)); + + validator.ValidateAsSuccess(new Request { Number = new NumberWrapper(2) }); + validator.ValidateAsError(new Request { Number = new NumberWrapper(4) }, "number", "Number must be between '1' and '3'."); + } + + [Test] + public void Unwrap_CustomWrapperCannotBeRebuiltByNormalization() + { + var validator = Validator.Create() + .HasProperty(x => x.Text, rule => rule.Unwrap().CompareValues(["ABC"], StringComparer.OrdinalIgnoreCase, overrideValueWhereMatched: true)); + var request = new Request { Text = new TextWrapper("abc") }; + + Assert.ThrowsAsync(async () => await validator.ValidateAsync(request)) + .Message.Should().Be("The property 'Text' is read-only and cannot be overridden."); + request.Text.Value.Should().Be("abc"); + } + + [Test] + public void Unwrap_CustomOptionalChecksPresenceBeforeReadingValue() + { + var validator = Validator.Create() + .HasProperty(x => x.OptionalText, rule => rule.Unwrap().Mandatory()); + + validator.ValidateAsSuccess(new Request()); + validator.ValidateAsSuccess(new Request { OptionalText = new OptionalTextWrapper(true, "abc") }); + validator.ValidateAsError(new Request { OptionalText = new OptionalTextWrapper(true, null) }, "optionalText", "Optional text is required."); + } + + [Test] + public void Specified_ValidatesCustomOptionalPresenceWithoutReadingValue() + { + var validator = Validator.Create() + .HasProperty(x => x.OptionalText, rule => rule.Specified()); + + validator.ValidateAsError(new Request(), "optionalText", "Optional text is required."); + validator.ValidateAsSuccess(new Request { OptionalText = new OptionalTextWrapper(true, null) }); + } + + public class Request + { + public TextWrapper Text { get; set; } = new("abc"); + public NumberWrapper Number { get; set; } + public OptionalTextWrapper OptionalText { get; set; } = new(false, null); + } + + public sealed record TextWrapper(string Value) : CoreEx.Entities.Abstractions.IWrappedValue; + + public readonly record struct NumberWrapper(int Value) : CoreEx.Entities.Abstractions.IWrappedValue; + + public sealed class OptionalTextWrapper(bool isSpecified, string? value) : CoreEx.Entities.Abstractions.IOptionalValue + { + public bool IsSpecified => isSpecified; + string? CoreEx.Entities.Abstractions.IWrappedValue.Value => isSpecified ? value : throw new InvalidOperationException("An unspecified value must not be read."); + } +} diff --git a/tests/CoreEx.Validation.Test.Unit/Rules/WrappedContextTests.cs b/tests/CoreEx.Validation.Test.Unit/Rules/WrappedContextTests.cs new file mode 100644 index 000000000..76fe5c4f9 --- /dev/null +++ b/tests/CoreEx.Validation.Test.Unit/Rules/WrappedContextTests.cs @@ -0,0 +1,349 @@ +namespace CoreEx.Validation.Test.Unit.Rules; + +[TestFixture] +public class WrappedContextTests +{ + [Test] + public void Unwrap_ErrorMessageUsesInnerValueAndOuterLabel() + { + var validator = Validator.Create() + .HasProperty(x => x.Text, rule => rule.WithText("Additional notes").Unwrap().String(3).WithMessage("{0}: {1} is too long.")); + + validator.ValidateAsError(new Request { Text = "long" }, "text", "Additional notes: 'long' is too long."); + } + + [Test] + public void Unwrap_OverrideWritesBackAndRefreshesTheChain() + { + var validator = Validator.Create() + .HasProperty(x => x.Text, rule => rule.Unwrap().String(3) + .CompareValues(["ABC"], StringComparer.OrdinalIgnoreCase, overrideValueWhereMatched: true) + .CompareValues(["ABC"])); + var request = new Request { Text = "abc" }; + + validator.ValidateAsSuccess(new Request()); + validator.ValidateAsSuccess(request); + request.Text.IsSpecified.Should().BeTrue(); + request.Text.Value.Should().Be("ABC"); + } + + [Test] + public void Unwrap_ReadOnlyPropertyCannotBeOverridden() + { + var validator = Validator.Create() + .HasProperty(x => x.ReadOnlyText, rule => rule.Unwrap().CompareValues(["ABC"], StringComparer.OrdinalIgnoreCase, overrideValueWhereMatched: true)); + + Assert.ThrowsAsync(async () => await validator.ValidateAsync(new Request())) + .Message.Should().Be("The property 'ReadOnlyText' is read-only and cannot be overridden."); + } + + [TestCase(false)] + [TestCase(true)] + public void Unwrap_EntityUsesInnerValueAndNestedErrorPath(bool configure) + { + var childValidator = Validator.Create().HasProperty(x => x.Name, rule => rule.Mandatory()); + var validator = Validator.Create(); + if (configure) + validator.HasProperty(x => x.Child, rule => rule.Unwrap().Entity(with => with.WithValidator(childValidator))); + else + validator.HasProperty(x => x.Child, rule => rule.Unwrap().Entity(childValidator)); + + validator.ValidateAsSuccess(new Request()); + validator.ValidateAsSuccess(new Request { Child = CoreEx.Entities.Optional.Specified(null) }); + validator.ValidateAsSuccess(new Request { Child = new Child { Name = "abc" } }); + validator.ValidateAsError(new Request { Child = new Child() }, "child.name", "Name is required."); + } + + [Test] + public void Unwrap_CollectionAndDictionaryRetainNestedErrorPaths() + { + var childValidator = Validator.Create().HasProperty(x => x.Name, rule => rule.Mandatory()); + var collectionValidator = Validator.Create() + .HasProperty(x => x.Children, rule => rule.Unwrap().Collection(with => with.WithItemValidator(childValidator))); + var dictionaryValidator = Validator.Create() + .HasProperty(x => x.ChildrenByName, rule => rule.Unwrap().Dictionary(with => with.WithValueValidator(childValidator))); + + collectionValidator.ValidateAsSuccess(new Request()); + collectionValidator.ValidateAsSuccess(new Request { Children = new List { new() { Name = "abc" } } }); + collectionValidator.ValidateAsError(new Request { Children = new List { new() } }, "children[0].name", "Name is required."); + dictionaryValidator.ValidateAsSuccess(new Request()); + dictionaryValidator.ValidateAsSuccess(new Request { ChildrenByName = new Dictionary { ["Home"] = new() { Name = "abc" } } }); + dictionaryValidator.ValidateAsError(new Request { ChildrenByName = new Dictionary { ["Home"] = new() } }, "childrenByName.Home.name", "Name is required."); + } + + [Test] + public void Unwrap_NullableStringNormalizationPreservesSpecifiedNull() + { + var validator = Validator.Create() + .HasProperty(x => x.NullableText, rule => rule.Unwrap() + .CompareValues(["ABC"], StringComparer.OrdinalIgnoreCase, overrideValueWhereMatched: true).CompareValues(["ABC"])); + var request = new Request { NullableText = "abc" }; + var nullRequest = new Request { NullableText = CoreEx.Entities.Optional.Specified(null) }; + + validator.ValidateAsSuccess(request); + request.NullableText.Value.Should().Be("ABC"); + validator.ValidateAsSuccess(nullRequest); + nullRequest.NullableText.IsSpecified.Should().BeTrue(); + nullRequest.NullableText.Value.Should().BeNull(); + } + + [TestCase("CompareValues", "ZZZ")] + [TestCase("CompareValuesDelegate", "ZZZ")] + [TestCase("Between", "ZZZ")] + [TestCase("BetweenDelegate", "ZZZ")] + [TestCase("InclusiveBetween", "ZZZ")] + [TestCase("InclusiveBetweenDelegate", "ZZZ")] + [TestCase("ExclusiveBetween", "ZZZ")] + [TestCase("ExclusiveBetweenDelegate", "ZZZ")] + [TestCase("Compare", "ZZZ")] + [TestCase("CompareDelegate", "ZZZ")] + [TestCase("Equal", "ZZZ")] + [TestCase("EqualDelegate", "ZZZ")] + [TestCase("NotEqual", "ZZZ")] + [TestCase("NotEqualDelegate", "ZZZ")] + [TestCase("LessThan", "ZZZ")] + [TestCase("LessThanDelegate", "ZZZ")] + [TestCase("LessThanOrEqualTo", "ZZZ")] + [TestCase("LessThanOrEqualToDelegate", "ZZZ")] + [TestCase("GreaterThan", "000")] + [TestCase("GreaterThanDelegate", "000")] + [TestCase("GreaterThanOrEqualTo", "000")] + [TestCase("GreaterThanOrEqualToDelegate", "000")] + [TestCase("CompareProperty", "ZZZ")] + public async Task Unwrap_ComparisonsAcceptNullableStringsDirectly(string comparison, string invalid) + { + var validator = Validator.Create().HasProperty(x => x.NullableText, rule => ConfigureComparison(rule.Unwrap(), comparison)); + + validator.ValidateAsSuccess(new Request()); + validator.ValidateAsSuccess(new Request { NullableText = CoreEx.Entities.Optional.Specified(null) }); + validator.ValidateAsSuccess(new Request { NullableText = "ABC", Code = "ABC" }); + var result = await validator.ValidateAsync(new Request { NullableText = invalid, Code = "ABC" }); + result.HasErrors.Should().BeTrue(); + result.Messages.Should().ContainSingle().Which.Property.Should().Be("nullableText"); + } + + private static CoreEx.Validation.Rules.IPropertyRule ConfigureComparison(CoreEx.Validation.Rules.IPropertyRule rule, string comparison) => comparison switch + { + "CompareValues" => rule.CompareValues(["ABC"]), + "CompareValuesDelegate" => rule.CompareValues(_ => ["ABC"]), + "Between" => rule.Between("AAA", "CCC"), + "BetweenDelegate" => rule.Between(_ => "AAA", _ => "CCC"), + "InclusiveBetween" => rule.InclusiveBetween("AAA", "CCC"), + "InclusiveBetweenDelegate" => rule.InclusiveBetween(_ => "AAA", _ => "CCC"), + "ExclusiveBetween" => rule.ExclusiveBetween("AAA", "CCC"), + "ExclusiveBetweenDelegate" => rule.ExclusiveBetween(_ => "AAA", _ => "CCC"), + "Compare" => rule.Compare(CompareOperator.Equal, "ABC"), + "CompareDelegate" => rule.Compare(CompareOperator.Equal, _ => "ABC"), + "Equal" => rule.Equal("ABC"), + "EqualDelegate" => rule.Equal(_ => "ABC"), + "NotEqual" => rule.NotEqual("ZZZ"), + "NotEqualDelegate" => rule.NotEqual(_ => "ZZZ"), + "LessThan" => rule.LessThan("CCC"), + "LessThanDelegate" => rule.LessThan(_ => "CCC"), + "LessThanOrEqualTo" => rule.LessThanOrEqualTo("CCC"), + "LessThanOrEqualToDelegate" => rule.LessThanOrEqualTo(_ => "CCC"), + "GreaterThan" => rule.GreaterThan("AAA"), + "GreaterThanDelegate" => rule.GreaterThan(_ => "AAA"), + "GreaterThanOrEqualTo" => rule.GreaterThanOrEqualTo("AAA"), + "GreaterThanOrEqualToDelegate" => rule.GreaterThanOrEqualTo(_ => "AAA"), + "CompareProperty" => rule.CompareProperty(CompareOperator.Equal, x => x.Code), + _ => throw new ArgumentOutOfRangeException(nameof(comparison)) + }; + + [Test] + public void Common_PreservesOuterLabelAndValueFormatting() + { + var common = Validator.CreateCommon(rule => rule.GreaterThan(10).WithMessage("{0}: {1} is too small.")); + var validator = Validator.Create() + .HasProperty(x => x.Amount, rule => rule.WithText("Order amount").WithFormat("F2", System.Globalization.CultureInfo.InvariantCulture, null).Common(common)); + + validator.ValidateAsError(new Request { Amount = 2m }, "amount", "Order amount: 2.00 is too small."); + } + + [Test] + public void Unwrap_CommonPreservesOuterLabel() + { + var common = Validator.CreateCommon(rule => rule.String(3)); + var validator = Validator.Create() + .HasProperty(x => x.Text, rule => rule.WithText("Additional notes").Unwrap().Common(common)); + + validator.ValidateAsError(new Request { Text = "long" }, "text", "Additional notes must not exceed 3 character(s) in length."); + } + + [Test] + public void Common_OverrideUpdatesOrdinaryAndWrappedProperties() + { + var common = Validator.CreateCommon(rule => rule.CompareValues(["ABC"], StringComparer.OrdinalIgnoreCase, overrideValueWhereMatched: true).CompareValues(["ABC"])); + var validator = Validator.Create() + .HasProperty(x => x.Code, rule => rule.Common(common).CompareValues(["ABC"])) + .HasProperty(x => x.Text, rule => rule.Unwrap().Common(common).CompareValues(["ABC"])); + var request = new Request { Code = "abc", Text = "abc" }; + + validator.ValidateAsSuccess(request); + request.Code.Should().Be("ABC"); + request.Text.Value.Should().Be("ABC"); + } + + [Test] + public void Common_OverrideRespectsReadOnlyProperties() + { + var common = Validator.CreateCommon(rule => rule.CompareValues(["ABC"], StringComparer.OrdinalIgnoreCase, overrideValueWhereMatched: true)); + var validator = Validator.Create() + .HasProperty(x => x.ReadOnlyText, rule => rule.Unwrap().Common(common)); + + Assert.ThrowsAsync(async () => await validator.ValidateAsync(new Request())) + .Message.Should().Be("The property 'ReadOnlyText' is read-only and cannot be overridden."); + } + + [Test] + public void Common_NullableNumberDoesNotBecomeZero() + { + var common = Validator.CreateCommon(rule => rule.Between(1, 3).Equal(2)); + var ordinary = Validator.Create().HasProperty(x => x.NullableCount, rule => rule.Common(common)); + var wrapped = Validator.Create().HasProperty(x => x.WrappedNullableCount, rule => rule.Unwrap().Common(common)); + + ordinary.ValidateAsSuccess(new Request()); + ordinary.ValidateAsSuccess(new Request { NullableCount = 2 }); + ordinary.ValidateAsError(new Request { NullableCount = 0 }, "nullableCount", "Nullable count must be between '1' and '3'."); + wrapped.ValidateAsSuccess(new Request { WrappedNullableCount = CoreEx.Entities.Optional.Specified(null) }); + wrapped.ValidateAsSuccess(new Request { WrappedNullableCount = 2 }); + wrapped.ValidateAsError(new Request { WrappedNullableCount = 0 }, "wrappedNullableCount", "Wrapped nullable count must be between '1' and '3'."); + } + + [Test] + public void Common_NotNullDistinguishesNullFromZero() + { + var common = Validator.CreateCommon(rule => rule.Between(0, 3).NotNull().WithMessage("{0}: {1} must be supplied.")); + var ordinary = Validator.Create().HasProperty(x => x.NullableCount, rule => rule.Common(common)); + var wrapped = Validator.Create().HasProperty(x => x.WrappedNullableCount, rule => rule.Unwrap().Common(common)); + + ordinary.ValidateAsError(new Request(), "nullableCount", "Nullable count: (null) must be supplied."); + ordinary.ValidateAsSuccess(new Request { NullableCount = 0 }); + wrapped.ValidateAsSuccess(new Request()); + wrapped.ValidateAsError(new Request { WrappedNullableCount = CoreEx.Entities.Optional.Specified(null) }, "wrappedNullableCount", "Wrapped nullable count: (null) must be supplied."); + wrapped.ValidateAsSuccess(new Request { WrappedNullableCount = 0 }); + } + + [TestCase(false, null)] + [TestCase(false, 0)] + [TestCase(false, 2)] + [TestCase(true, null)] + [TestCase(true, 0)] + [TestCase(true, 2)] + public void Common_TypedMetadataReadsDoNotAllocateAndPreserveNullState(bool wrapped, int? value) + { + var invoked = false; + var inner = Validator.CreateCommon(rule => rule.Between(0, 3)) + .AdditionalAsync((context, _) => + { + invoked = true; + var metadata = ((CoreEx.Validation.Abstractions.IPropertyContext)context).Metadata; + metadata.Type.Should().Be(typeof(int)); + context.IsValueNull.Should().Be(!value.HasValue); + context.Value.Should().Be(value.GetValueOrDefault()); + + var entity = context.Entity; + var total = 0; + for (var i = 0; i < 1000; i++) + total += metadata.GetValue(entity); + + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 10000; i++) + total += metadata.GetValue(entity); + + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + allocated.Should().Be(0, "typed metadata reads must not box each integer value"); + total.Should().Be(11000 * value.GetValueOrDefault()); + return Task.CompletedTask; + }); + var common = Validator.CreateCommon(rule => rule.Common(inner).Between(0, 3)); + var validator = Validator.Create(); + if (wrapped) + validator.HasProperty(x => x.WrappedNullableCount, rule => rule.Unwrap().Common(common)); + else + validator.HasProperty(x => x.NullableCount, rule => rule.Common(common)); + + validator.ValidateAsSuccess(new Request { NullableCount = value, WrappedNullableCount = CoreEx.Entities.Optional.Specified(value) }); + invoked.Should().BeTrue(); + } + + [TestCase(false)] + [TestCase(true)] + public void NumericOverride_RefreshesOrdinaryAndWrappedNullableValues(bool useCommon) + { + var common = Validator.CreateCommon(rule => rule.CompareValues([1], new LastDigitComparer(), overrideValueWhereMatched: true).Equal(1)); + var validator = Validator.Create(); + if (useCommon) + { + validator.HasProperty(x => x.Count, rule => rule.Common(common).Equal(1)) + .HasProperty(x => x.NullableCount, rule => rule.Common(common).Equal(1)) + .HasProperty(x => x.WrappedCount, rule => rule.Unwrap().Common(common).Equal(1)) + .HasProperty(x => x.WrappedNullableCount, rule => rule.Unwrap().Common(common).Equal(1)); + } + else + { + validator.HasProperty(x => x.Count, rule => rule.CompareValues([1], new LastDigitComparer(), overrideValueWhereMatched: true).Equal(1)) + .HasProperty(x => x.NullableCount, rule => rule.CompareValues([1], new LastDigitComparer(), overrideValueWhereMatched: true).Equal(1)) + .HasProperty(x => x.WrappedCount, rule => rule.Unwrap().CompareValues([1], new LastDigitComparer(), overrideValueWhereMatched: true).Equal(1)) + .HasProperty(x => x.WrappedNullableCount, rule => rule.Unwrap().CompareValues([1], new LastDigitComparer(), overrideValueWhereMatched: true).Equal(1)); + } + + var request = new Request { Count = 11, NullableCount = 11, WrappedCount = 11, WrappedNullableCount = 11 }; + validator.ValidateAsSuccess(request); + request.Count.Should().Be(1); + request.NullableCount.Should().Be(1); + request.WrappedCount.Value.Should().Be(1); + request.WrappedNullableCount.Value.Should().Be(1); + } + + [Test] + public void Common_SecondaryValidationObservesNormalizedValue() + { + var common = Validator.CreateCommon(rule => rule.CompareValues(["ABC"], StringComparer.OrdinalIgnoreCase, overrideValueWhereMatched: true)) + .AdditionalAsync((context, _) => + { + context.Value.Should().Be("ABC"); + context.Override("DEF"); + return Task.CompletedTask; + }) + .AdditionalAsync((context, _) => + { + context.Value.Should().Be("DEF"); + return Task.CompletedTask; + }); + + var validator = Validator.Create() + .HasProperty(x => x.Text, rule => rule.Unwrap().Common(common).CompareValues(["DEF"])); + var request = new Request { Text = "abc" }; + + validator.ValidateAsSuccess(request); + request.Text.Value.Should().Be("DEF"); + } + + private sealed class LastDigitComparer : IEqualityComparer + { + public bool Equals(int x, int y) => x % 10 == y % 10; + public int GetHashCode(int obj) => obj % 10; + } + + public class Request + { + public string? Code { get; set; } + public int Count { get; set; } + public int? NullableCount { get; set; } + public CoreEx.Entities.Optional WrappedCount { get; set; } + public CoreEx.Entities.Optional WrappedNullableCount { get; set; } + public CoreEx.Entities.Optional Text { get; set; } + public CoreEx.Entities.Optional NullableText { get; set; } + public CoreEx.Entities.Optional ReadOnlyText => "abc"; + public CoreEx.Entities.Optional Child { get; set; } + public CoreEx.Entities.Optional> Children { get; set; } + public CoreEx.Entities.Optional> ChildrenByName { get; set; } + public decimal Amount { get; set; } + } + + public class Child + { + public string? Name { get; set; } + } +}