Keep unsupplied fields indexed after a merge

AzureSearchEmulator.UnitTests/MergeKeepsFieldsIndexedTests.cs

@@ -0,0 +1,199 @@
+using System.Text.Json.Nodes;
+using AzureSearchEmulator.Indexing;
+using AzureSearchEmulator.Models;
+using AzureSearchEmulator.SearchData;
+using AzureSearchEmulator.Searching;
+using Lucene.Net.Index;
+using Lucene.Net.Store;
+using Xunit;
+
+namespace AzureSearchEmulator.UnitTests;
+
+/// <summary>
+/// Covers the fields a merge does not supply. They must stay searchable, filterable and
+/// facetable, not only retrievable.
+/// </summary>
+/// <remarks>
+/// A merge used to write back the stored fields of the old document as they were. Stored fields
+/// carry no indexed terms or doc values, so after any merge the document still came back from a
+/// lookup by key but no longer matched a filter or a search on its other fields.
+/// </remarks>
+public class MergeKeepsFieldsIndexedTests : IDisposable
+{
+    private readonly RAMDirectory _directory = new();
+    private readonly SearchIndex _index;
+    private readonly LuceneNetIndexWriterFactory _writerFactory;
+    private readonly LuceneNetSearchIndexer _indexer;
+    private readonly LuceneNetIndexSearcher _searcher;
+
+    public MergeKeepsFieldsIndexedTests()
+    {
+        _index = new SearchIndex
+        {
+            Name = "mergeable",
+            Fields =
+            [
+                new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true },
+                new SearchField { Name = "Source", Type = "Edm.String", Searchable = false, Filterable = true },
+                new SearchField { Name = "Content", Type = "Edm.String", Searchable = true },
+                new SearchField { Name = "Groups", Type = "Collection(Edm.String)", Searchable = false, Filterable = true },
+                new SearchField { Name = "Rating", Type = "Edm.Int32", Filterable = true, Facetable = true },
+                new SearchField { Name = "Published", Type = "Edm.Boolean", Filterable = true },
+                new SearchField { Name = "Updated", Type = "Edm.DateTimeOffset", Filterable = true },
+                new SearchField { Name = "Location", Type = "Edm.GeographyPoint", Filterable = true },
+                new SearchField
+                {
+                    Name = "Address",
+                    Type = "Edm.ComplexType",
+                    Fields = [new SearchField { Name = "City", Type = "Edm.String", Searchable = false, Filterable = true }],
+                },
+                new SearchField
+                {
+                    Name = "Files",
+                    Type = "Collection(Edm.ComplexType)",
+                    Fields = [new SearchField { Name = "Path", Type = "Edm.String", Searchable = false, Filterable = true }],
+                },
+                new SearchField { Name = "Title", Type = "Edm.String", Searchable = true, Filterable = true },
+            ]
+        };
+
+        var factory = new SharedDirectoryFactory(_directory);
+        _writerFactory = new LuceneNetIndexWriterFactory(factory);
+        _indexer = new LuceneNetSearchIndexer(_writerFactory, factory);
+        _searcher = new LuceneNetIndexSearcher(factory);
+    }
+
+    public void Dispose()
+    {
+        // Writer before directory: it holds an open handle and commits on dispose.
+        _writerFactory.Dispose();
+        _directory.Dispose();
+    }
+
+    // Parsed from text, as a request body is: a date built in C# as a string JsonValue cannot be
+    // read back as a DateTimeOffset.
+    private static JsonObject Doc(string id) => JsonNode.Parse($$"""
+        {
+          "Id": "{{id}}",
+          "Source": "blob-{{id}}",
+          "Content": "quarterly board minutes",
+          "Groups": ["finance", "legal"],
+          "Rating": 4,
+          "Published": true,
+          "Updated": "2025-03-01T00:00:00Z",
+          "Location": { "type": "Point", "coordinates": [-1.55, 53.8] },
+          "Address": { "City": "Leeds" },
+          "Files": [{ "Path": "Entities/Company 4" }],
+          "Title": "Old title"
+        }
+        """)!.AsObject();
+
+    private static JsonObject TitleChange(string id) => new() { ["Id"] = id, ["Title"] = "New title" };
+
+    private void Index(IndexDocumentAction action)
+    {
+        var result = Assert.Single(_indexer.IndexDocuments(_index, [action]).Value);
+        Assert.True(result.Status, result.ErrorMessage);
+    }
+
+    private async Task<int?> Count(SearchRequest request)
+    {
+        request.Count = true;
+        return (await _searcher.Search(_index, request)).Count;
+    }
+
+    public static TheoryData<string> FiltersOnUntouchedFields =>
+    [
+        "Id eq '1' and Source eq 'blob-1'",
+        "Groups/any(g: search.in(g, 'finance'))",
+        "Rating eq 4",
+        "Published eq true",
+        "Updated eq 2025-03-01T00:00:00Z",
+        "geo.distance(Location, geography'POINT(-1.55 53.8)') le 1",
+        "Address/City eq 'Leeds'",
+        "Files/any(f: f/Path eq 'Entities/Company 4')",
+    ];
+
+    [Theory]
+    [MemberData(nameof(FiltersOnUntouchedFields))]
+    public async Task Merge_LeavesUntouchedFieldsFilterable(string filter)
+    {
+        Index(new UploadIndexDocumentAction(Doc("1")));
+        Index(new MergeIndexDocumentAction(TitleChange("1")));
+
+        Assert.Equal(1, await Count(new SearchRequest { Filter = filter }));
+    }
+
+    [Theory]
+    [MemberData(nameof(FiltersOnUntouchedFields))]
+    public async Task MergeOrUpload_OnExistingDocument_LeavesUntouchedFieldsFilterable(string filter)
+    {
+        Index(new UploadIndexDocumentAction(Doc("1")));
+        Index(new MergeOrUploadIndexDocumentAction(TitleChange("1")));
+
+        Assert.Equal(1, await Count(new SearchRequest { Filter = filter }));
+    }
+
+    [Fact]
+    public async Task Merge_LeavesUntouchedFieldsSearchable()
+    {
+        Index(new UploadIndexDocumentAction(Doc("1")));
+        Index(new MergeIndexDocumentAction(TitleChange("1")));
+
+        Assert.Equal(1, await Count(new SearchRequest { Search = "minutes", SearchFields = "Content" }));
+    }
+
+    [Fact]
+    public async Task Merge_LeavesUntouchedFieldsFacetable()
+    {
+        Index(new UploadIndexDocumentAction(Doc("1")));
+        Index(new MergeIndexDocumentAction(TitleChange("1")));
+
+        var response = await _searcher.Search(_index, new SearchRequest { Facets = ["Rating"] });
+
+        var bucket = Assert.Single(response.Facets!["Rating"]);
+        Assert.Equal(1, bucket.Count);
+    }
+
+    [Fact]
+    public async Task Merge_ReplacesTheSuppliedFieldInTheIndex()
+    {
+        Index(new UploadIndexDocumentAction(Doc("1")));
+        Index(new MergeIndexDocumentAction(TitleChange("1")));
+
+        Assert.Equal(0, await Count(new SearchRequest { Filter = "Title eq 'Old title'" }));
+        Assert.Equal(1, await Count(new SearchRequest { Filter = "Title eq 'New title'" }));
+        Assert.Equal(1, await Count(new SearchRequest { Search = "new", SearchFields = "Title" }));
+    }
+
+    /// <summary>
+    /// A second merge reads back the document the first one wrote, so it must also find every
+    /// field stored.
+    /// </summary>
+    [Fact]
+    public async Task RepeatedMerges_KeepEveryFieldRetrievable()
+    {
+        Index(new UploadIndexDocumentAction(Doc("1")));
+        Index(new MergeIndexDocumentAction(TitleChange("1")));
+        Index(new MergeIndexDocumentAction(new JsonObject { ["Id"] = "1", ["Rating"] = 5 }));
+
+        var doc = await _searcher.GetDoc(_index, "1");
+
+        Assert.NotNull(doc);
+        Assert.Equal(Doc("1").Select(p => p.Key).Order(), doc!.Select(p => p.Key).Order());
+        Assert.Equal("blob-1", doc["Source"]?.GetValue<string>());
+        Assert.Equal("New title", doc["Title"]?.GetValue<string>());
+        Assert.Equal(5, doc["Rating"]?.GetValue<int>());
+        Assert.Equal("Leeds", doc["Address"]?["City"]?.GetValue<string>());
+        Assert.Equal(1, await Count(new SearchRequest { Filter = "Rating eq 5 and Source eq 'blob-1'" }));
+    }
+
+    private class SharedDirectoryFactory(Lucene.Net.Store.Directory directory) : ILuceneDirectoryFactory, ILuceneIndexReaderFactory
+    {
+        public Lucene.Net.Store.Directory GetDirectory(string indexName) => directory;
+        public void ClearCachedDirectory(string indexName) { }
+        public IndexReader GetIndexReader(string indexName) => DirectoryReader.Open(directory);
+        public IndexReader RefreshReader(string indexName) => GetIndexReader(indexName);
+        public void ClearCachedReader(string indexName) { }
+    }
+}

AzureSearchEmulator.UnitTests/MergeNullClearsFieldTests.cs

@@ -106,8 +106,7 @@ public async Task Merge_OmittingAField_LeavesItAlone()
 
     /// <summary>
     /// A cleared string must also lose the unanalyzed <c>__azs_raw__</c> copy that filtering
-    /// reads, or the document keeps matching a filter for a value it no longer has — the
-    /// failure that motivated clearing every name a field writes rather than just its own.
+    /// reads, or the document keeps matching a filter for a value it no longer has.
     /// </summary>
     [Fact]
     public async Task Merge_WithExplicitNull_AlsoClearsTheFilterableCopy()

AzureSearchEmulator.UnitTests/VectorSearchTests.cs

@@ -409,7 +409,7 @@ public async Task Metric_ChangesTheRanking()
     /// A document merged on an unrelated field stays findable by vector search.
     /// </summary>
     /// <remarks>
-    /// The end-to-end half of the bug that <c>MergeDocument</c>'s doc-values rebuild fixes. The
+    /// The end-to-end half of the bug where a merge dropped the packed doc-values copy. The
     /// storage test in phase 1 asserts the packed copy survives; this asserts what that survival
     /// is <em>for</em>. Without it the document would keep retrieving correctly while silently
     /// dropping out of every vector query — the failure is invisible from the retrieval side,

AzureSearchEmulator/Indexing/MergeIndexDocumentAction.cs

@@ -5,6 +5,6 @@ namespace AzureSearchEmulator.Indexing;
 
 public class MergeIndexDocumentAction(JsonObject item) : UpsertIndexDocumentActionBase(item)
 {
-    protected override void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable<IIndexableField> docFields)
-        => MergeDocument(context, keyTerm, docFields, ClearedFields, false);
+    protected override void IndexDocument(IndexingContext context, Term keyTerm)
+        => MergeDocument(context, keyTerm, false);
 }

AzureSearchEmulator/Indexing/MergeOrUploadIndexDocumentAction.cs

@@ -5,6 +5,6 @@ namespace AzureSearchEmulator.Indexing;
 
 public class MergeOrUploadIndexDocumentAction(JsonObject item) : UpsertIndexDocumentActionBase(item)
 {
-    protected override void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable<IIndexableField> docFields)
-        => MergeDocument(context, keyTerm, docFields, ClearedFields, true);
+    protected override void IndexDocument(IndexingContext context, Term keyTerm)
+        => MergeDocument(context, keyTerm, true);
 }

AzureSearchEmulator/Indexing/SearchFieldExtensions.cs

@@ -52,6 +52,62 @@ private static IEnumerable<IIndexableField> CreateFields(
         return CreateSingleValueFields(field, value, path, index);
     }
 
+    /// <summary>
+    /// Reads the field's value back from a document's stored fields, or null when it has none.
+    /// </summary>
+    /// <remarks>
+    /// Does not check <see cref="SearchField.Retrievable"/>: a merge rebuilds the document from
+    /// whatever is stored, and a vector's stored copy is written even when it is not retrievable.
+    /// A complex value comes back whole, including sub-fields that are not retrievable.
+    /// </remarks>
+    public static JsonNode? GetStoredValue(this SearchField field, Document doc)
+    {
+        if (field.IsComplex())
+        {
+            // Read from the stored JSON copy rather than the flattened leaves, which cannot say
+            // which values belonged to which element of a Collection(Edm.ComplexType).
+            var storedComplexJson = doc.Get(ComplexTypeSupport.GetComplexStorageFieldName(field.Name));
+            return storedComplexJson == null ? null : JsonNode.Parse(storedComplexJson);
+        }
+
+        if (field.IsCollection())
+        {
+            var storedJson = doc.Get(GetCollectionStorageFieldName(field.Name));
+            return storedJson == null ? null : JsonNode.Parse(storedJson);
+        }
+
+        if (field.Type == GeoSupport.GeographyPointType)
+        {
+            // Points are stored as a separate lat/lon pair rather than under the
+            // field's own name, so they need their own lookup.
+            var latField = doc.GetField(GeoSupport.GetLatFieldName(field.Name));
+            var lonField = doc.GetField(GeoSupport.GetLonFieldName(field.Name));
+
+            return latField?.GetDoubleValue() is double storedLat
+                && lonField?.GetDoubleValue() is double storedLon
+                    ? GeoSupport.CreateGeoJsonPoint(storedLon, storedLat)
+                    : null;
+        }
+
+        var docField = doc.GetField(field.Name);
+
+        if (docField == null)
+        {
+            return null;
+        }
+
+        return field.Type switch
+        {
+            "Edm.String" => docField.GetStringValue(),
+            "Edm.Int32" => docField.GetInt32Value(),
+            "Edm.Int64" => docField.GetInt64Value(),
+            "Edm.Double" => docField.GetDoubleValue(),
+            "Edm.Boolean" => docField.GetInt32Value() is int i ? i != 0 : null,
+            "Edm.DateTimeOffset" => docField.GetInt64Value() is long ms ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : null,
+            _ => throw new InvalidOperationException($"Unsupported field type {field.Type}")
+        };
+    }
+
     private static IEnumerable<IIndexableField> CreateSingleValueFields(
         SearchField field,
         JsonNode value,

AzureSearchEmulator/Indexing/UploadIndexDocumentAction.cs

@@ -5,6 +5,6 @@ namespace AzureSearchEmulator.Indexing;
 
 public class UploadIndexDocumentAction(JsonObject item) : UpsertIndexDocumentActionBase(item)
 {
-    protected override void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable<IIndexableField> docFields)
-        => context.Writer.UpdateDocument(keyTerm, docFields);
+    protected override void IndexDocument(IndexingContext context, Term keyTerm)
+        => context.Writer.UpdateDocument(keyTerm, GetDocFields(context.Index, Item));
 }