From 0efc63b72f2cf9d0939745f0674c6fbbd05cd296 Mon Sep 17 00:00:00 2001 From: Daniel Green <757038+danielgreen@users.noreply.github.com> Date: Wed, 21 Oct 2026 12:20:48 +0100 Subject: [PATCH 1/2] Move reading a field's stored value into GetStoredValue ConvertSearchDoc turned each field's stored Lucene fields back into JSON inline. That code moves, unchanged, into SearchFieldExtensions.GetStoredValue, which returns the raw stored value. ConvertSearchDoc still applies retrievable, $select and the non-retrievable sub-fields of a complex value, as before. No change in behaviour. The switch's Edm.ComplexType case goes: it was unreachable, since complex fields return earlier. This lets the merge action read a stored document back with the same code. --- .../Indexing/SearchFieldExtensions.cs | 55 +++++++++++++++++ .../Searching/LuceneNetIndexSearcher.cs | 59 ++----------------- 2 files changed, 60 insertions(+), 54 deletions(-) diff --git a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs index ba1f781..fc6ad0c 100644 --- a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs +++ b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs @@ -52,6 +52,61 @@ private static IEnumerable CreateFields( return CreateSingleValueFields(field, value, path, index); } + /// + /// Reads the field's value back from a document's stored fields, or null when it has none. + /// + /// + /// Does not check , and a complex value comes back whole, + /// including sub-fields that are not retrievable: the caller decides what to return. + /// + public static JsonNode? GetStoredValue(this SearchField field, Document doc) + { + if (field.IsComplex()) + { + // Read from the stored JSON copy rather than the flattened leaves, which cannot say + // which values belonged to which element of a Collection(Edm.ComplexType). + var storedComplexJson = doc.Get(ComplexTypeSupport.GetComplexStorageFieldName(field.Name)); + return storedComplexJson == null ? null : JsonNode.Parse(storedComplexJson); + } + + if (field.IsCollection()) + { + var storedJson = doc.Get(GetCollectionStorageFieldName(field.Name)); + return storedJson == null ? null : JsonNode.Parse(storedJson); + } + + if (field.Type == GeoSupport.GeographyPointType) + { + // Points are stored as a separate lat/lon pair rather than under the + // field's own name, so they need their own lookup. + var latField = doc.GetField(GeoSupport.GetLatFieldName(field.Name)); + var lonField = doc.GetField(GeoSupport.GetLonFieldName(field.Name)); + + return latField?.GetDoubleValue() is double storedLat + && lonField?.GetDoubleValue() is double storedLon + ? GeoSupport.CreateGeoJsonPoint(storedLon, storedLat) + : null; + } + + var docField = doc.GetField(field.Name); + + if (docField == null) + { + return null; + } + + return field.Type switch + { + "Edm.String" => docField.GetStringValue(), + "Edm.Int32" => docField.GetInt32Value(), + "Edm.Int64" => docField.GetInt64Value(), + "Edm.Double" => docField.GetDoubleValue(), + "Edm.Boolean" => docField.GetInt32Value() is int i ? i != 0 : null, + "Edm.DateTimeOffset" => docField.GetInt64Value() is long ms ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : null, + _ => throw new InvalidOperationException($"Unsupported field type {field.Type}") + }; + } + private static IEnumerable CreateSingleValueFields( SearchField field, JsonNode value, diff --git a/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs b/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs index 2d049f5..53e7249 100644 --- a/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs +++ b/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs @@ -959,65 +959,16 @@ private static JsonObject ConvertSearchDoc( continue; } - if (field.IsComplex()) - { - // Complex values round-trip through the stored JSON sidecar rather than the - // flattened leaves, which cannot say which values belonged to which element - // of a Collection(Edm.ComplexType). - var storedComplexJson = doc.Get(ComplexTypeSupport.GetComplexStorageFieldName(field.Name)); - if (storedComplexJson != null) - { - result[field.Name] = FilterToRetrievableSubFields( - field, - JsonNode.Parse(storedComplexJson), - selection?.GetSubSelection(field.Name)); - } - continue; - } + var value = field.GetStoredValue(doc); - if (field.IsCollection()) + if (value == null) { - var storedJson = doc.Get(SearchFieldExtensions.GetCollectionStorageFieldName(field.Name)); - if (storedJson != null) - { - result[field.Name] = JsonNode.Parse(storedJson); - } continue; } - if (field.Type == GeoSupport.GeographyPointType) - { - // Points are stored as a separate lat/lon pair rather than under the - // field's own name, so they need their own lookup. - var latField = doc.GetField(GeoSupport.GetLatFieldName(field.Name)); - var lonField = doc.GetField(GeoSupport.GetLonFieldName(field.Name)); - - if (latField?.GetDoubleValue() is double storedLat - && lonField?.GetDoubleValue() is double storedLon) - { - result[field.Name] = GeoSupport.CreateGeoJsonPoint(storedLon, storedLat); - } - - continue; - } - - var docField = doc.GetField(field.Name); - - if (docField != null) - { - result[field.Name] = field.Type switch - { - "Edm.String" => docField.GetStringValue(), - "Edm.Int32" => docField.GetInt32Value(), - "Edm.Int64" => docField.GetInt64Value(), - "Edm.Double" => docField.GetDoubleValue(), - "Edm.Boolean" => docField.GetInt32Value() is int i ? i != 0 : null, - "Edm.DateTimeOffset" => docField.GetInt64Value() is long ms ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : null, - // Edm.GeographyPoint is handled above, before this switch. - "Edm.ComplexType" => throw new NotImplementedException(), - _ => throw new InvalidOperationException($"Unsupported field type {field.Type}") - }; - } + result[field.Name] = field.IsComplex() + ? FilterToRetrievableSubFields(field, value, selection?.GetSubSelection(field.Name)) + : value; } return result; From d08c77e130439848a8930393c8c47c2a591957f3 Mon Sep 17 00:00:00 2001 From: Daniel Green <757038+danielgreen@users.noreply.github.com> Date: Wed, 21 Oct 2026 12:20:49 +0100 Subject: [PATCH 2/2] Keep unsupplied fields indexed after a merge After a merge or mergeOrUpload on a document that already existed, the document kept its values but stopped matching filters, facets and vector queries on the fields the merge did not supply. For example, after merging only Title, "Id eq '1' and Source eq 'blob-1'" no longer found the document. The cause: the merge wrote back the stored fields returned by IndexSearcher.Doc, which carry no indexed terms or doc values. A merge now reads the stored document back as JSON with GetStoredValue, applies the item's fields on top, and indexes the result as an upload does. An explicit null still clears a field, because null values are not indexed. This replaces the earlier partial repairs, which the new merge covers: ClearedFields, GetClearedFields and GetLuceneFieldNames (#71: remove every Lucene copy of a field set to null) and RestoreVectorDocValues (#46: rebuild a vector's packed copy). It also keeps the facet, geo and complex-element doc values that RestoreVectorDocValues' comment noted were still lost. Limit: a field that is not retrievable has no stored copy (except a vector), so a merge that does not supply it loses its value. --- .../MergeKeepsFieldsIndexedTests.cs | 199 ++++++++++++++++++ .../MergeNullClearsFieldTests.cs | 3 +- .../VectorSearchTests.cs | 2 +- .../Indexing/MergeIndexDocumentAction.cs | 4 +- .../MergeOrUploadIndexDocumentAction.cs | 4 +- .../Indexing/SearchFieldExtensions.cs | 5 +- .../Indexing/UploadIndexDocumentAction.cs | 4 +- .../Indexing/UpsertIndexDocumentActionBase.cs | 197 +++-------------- 8 files changed, 241 insertions(+), 177 deletions(-) create mode 100644 AzureSearchEmulator.UnitTests/MergeKeepsFieldsIndexedTests.cs diff --git a/AzureSearchEmulator.UnitTests/MergeKeepsFieldsIndexedTests.cs b/AzureSearchEmulator.UnitTests/MergeKeepsFieldsIndexedTests.cs new file mode 100644 index 0000000..e5cf109 --- /dev/null +++ b/AzureSearchEmulator.UnitTests/MergeKeepsFieldsIndexedTests.cs @@ -0,0 +1,199 @@ +using System.Text.Json.Nodes; +using AzureSearchEmulator.Indexing; +using AzureSearchEmulator.Models; +using AzureSearchEmulator.SearchData; +using AzureSearchEmulator.Searching; +using Lucene.Net.Index; +using Lucene.Net.Store; +using Xunit; + +namespace AzureSearchEmulator.UnitTests; + +/// +/// Covers the fields a merge does not supply. They must stay searchable, filterable and +/// facetable, not only retrievable. +/// +/// +/// A merge used to write back the stored fields of the old document as they were. Stored fields +/// carry no indexed terms or doc values, so after any merge the document still came back from a +/// lookup by key but no longer matched a filter or a search on its other fields. +/// +public class MergeKeepsFieldsIndexedTests : IDisposable +{ + private readonly RAMDirectory _directory = new(); + private readonly SearchIndex _index; + private readonly LuceneNetIndexWriterFactory _writerFactory; + private readonly LuceneNetSearchIndexer _indexer; + private readonly LuceneNetIndexSearcher _searcher; + + public MergeKeepsFieldsIndexedTests() + { + _index = new SearchIndex + { + Name = "mergeable", + Fields = + [ + new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true }, + new SearchField { Name = "Source", Type = "Edm.String", Searchable = false, Filterable = true }, + new SearchField { Name = "Content", Type = "Edm.String", Searchable = true }, + new SearchField { Name = "Groups", Type = "Collection(Edm.String)", Searchable = false, Filterable = true }, + new SearchField { Name = "Rating", Type = "Edm.Int32", Filterable = true, Facetable = true }, + new SearchField { Name = "Published", Type = "Edm.Boolean", Filterable = true }, + new SearchField { Name = "Updated", Type = "Edm.DateTimeOffset", Filterable = true }, + new SearchField { Name = "Location", Type = "Edm.GeographyPoint", Filterable = true }, + new SearchField + { + Name = "Address", + Type = "Edm.ComplexType", + Fields = [new SearchField { Name = "City", Type = "Edm.String", Searchable = false, Filterable = true }], + }, + new SearchField + { + Name = "Files", + Type = "Collection(Edm.ComplexType)", + Fields = [new SearchField { Name = "Path", Type = "Edm.String", Searchable = false, Filterable = true }], + }, + new SearchField { Name = "Title", Type = "Edm.String", Searchable = true, Filterable = true }, + ] + }; + + var factory = new SharedDirectoryFactory(_directory); + _writerFactory = new LuceneNetIndexWriterFactory(factory); + _indexer = new LuceneNetSearchIndexer(_writerFactory, factory); + _searcher = new LuceneNetIndexSearcher(factory); + } + + public void Dispose() + { + // Writer before directory: it holds an open handle and commits on dispose. + _writerFactory.Dispose(); + _directory.Dispose(); + } + + // Parsed from text, as a request body is: a date built in C# as a string JsonValue cannot be + // read back as a DateTimeOffset. + private static JsonObject Doc(string id) => JsonNode.Parse($$""" + { + "Id": "{{id}}", + "Source": "blob-{{id}}", + "Content": "quarterly board minutes", + "Groups": ["finance", "legal"], + "Rating": 4, + "Published": true, + "Updated": "2025-03-01T00:00:00Z", + "Location": { "type": "Point", "coordinates": [-1.55, 53.8] }, + "Address": { "City": "Leeds" }, + "Files": [{ "Path": "Entities/Company 4" }], + "Title": "Old title" + } + """)!.AsObject(); + + private static JsonObject TitleChange(string id) => new() { ["Id"] = id, ["Title"] = "New title" }; + + private void Index(IndexDocumentAction action) + { + var result = Assert.Single(_indexer.IndexDocuments(_index, [action]).Value); + Assert.True(result.Status, result.ErrorMessage); + } + + private async Task Count(SearchRequest request) + { + request.Count = true; + return (await _searcher.Search(_index, request)).Count; + } + + public static TheoryData FiltersOnUntouchedFields => + [ + "Id eq '1' and Source eq 'blob-1'", + "Groups/any(g: search.in(g, 'finance'))", + "Rating eq 4", + "Published eq true", + "Updated eq 2025-03-01T00:00:00Z", + "geo.distance(Location, geography'POINT(-1.55 53.8)') le 1", + "Address/City eq 'Leeds'", + "Files/any(f: f/Path eq 'Entities/Company 4')", + ]; + + [Theory] + [MemberData(nameof(FiltersOnUntouchedFields))] + public async Task Merge_LeavesUntouchedFieldsFilterable(string filter) + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + + Assert.Equal(1, await Count(new SearchRequest { Filter = filter })); + } + + [Theory] + [MemberData(nameof(FiltersOnUntouchedFields))] + public async Task MergeOrUpload_OnExistingDocument_LeavesUntouchedFieldsFilterable(string filter) + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeOrUploadIndexDocumentAction(TitleChange("1"))); + + Assert.Equal(1, await Count(new SearchRequest { Filter = filter })); + } + + [Fact] + public async Task Merge_LeavesUntouchedFieldsSearchable() + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + + Assert.Equal(1, await Count(new SearchRequest { Search = "minutes", SearchFields = "Content" })); + } + + [Fact] + public async Task Merge_LeavesUntouchedFieldsFacetable() + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + + var response = await _searcher.Search(_index, new SearchRequest { Facets = ["Rating"] }); + + var bucket = Assert.Single(response.Facets!["Rating"]); + Assert.Equal(1, bucket.Count); + } + + [Fact] + public async Task Merge_ReplacesTheSuppliedFieldInTheIndex() + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + + Assert.Equal(0, await Count(new SearchRequest { Filter = "Title eq 'Old title'" })); + Assert.Equal(1, await Count(new SearchRequest { Filter = "Title eq 'New title'" })); + Assert.Equal(1, await Count(new SearchRequest { Search = "new", SearchFields = "Title" })); + } + + /// + /// A second merge reads back the document the first one wrote, so it must also find every + /// field stored. + /// + [Fact] + public async Task RepeatedMerges_KeepEveryFieldRetrievable() + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + Index(new MergeIndexDocumentAction(new JsonObject { ["Id"] = "1", ["Rating"] = 5 })); + + var doc = await _searcher.GetDoc(_index, "1"); + + Assert.NotNull(doc); + Assert.Equal(Doc("1").Select(p => p.Key).Order(), doc!.Select(p => p.Key).Order()); + Assert.Equal("blob-1", doc["Source"]?.GetValue()); + Assert.Equal("New title", doc["Title"]?.GetValue()); + Assert.Equal(5, doc["Rating"]?.GetValue()); + Assert.Equal("Leeds", doc["Address"]?["City"]?.GetValue()); + Assert.Equal(1, await Count(new SearchRequest { Filter = "Rating eq 5 and Source eq 'blob-1'" })); + } + + private class SharedDirectoryFactory(Lucene.Net.Store.Directory directory) : ILuceneDirectoryFactory, ILuceneIndexReaderFactory + { + public Lucene.Net.Store.Directory GetDirectory(string indexName) => directory; + public void ClearCachedDirectory(string indexName) { } + public IndexReader GetIndexReader(string indexName) => DirectoryReader.Open(directory); + public IndexReader RefreshReader(string indexName) => GetIndexReader(indexName); + public void ClearCachedReader(string indexName) { } + } +} diff --git a/AzureSearchEmulator.UnitTests/MergeNullClearsFieldTests.cs b/AzureSearchEmulator.UnitTests/MergeNullClearsFieldTests.cs index b7b8dc5..1fcd044 100644 --- a/AzureSearchEmulator.UnitTests/MergeNullClearsFieldTests.cs +++ b/AzureSearchEmulator.UnitTests/MergeNullClearsFieldTests.cs @@ -106,8 +106,7 @@ public async Task Merge_OmittingAField_LeavesItAlone() /// /// A cleared string must also lose the unanalyzed __azs_raw__ copy that filtering - /// reads, or the document keeps matching a filter for a value it no longer has — the - /// failure that motivated clearing every name a field writes rather than just its own. + /// reads, or the document keeps matching a filter for a value it no longer has. /// [Fact] public async Task Merge_WithExplicitNull_AlsoClearsTheFilterableCopy() diff --git a/AzureSearchEmulator.UnitTests/VectorSearchTests.cs b/AzureSearchEmulator.UnitTests/VectorSearchTests.cs index 8c5a3bb..6840ca4 100644 --- a/AzureSearchEmulator.UnitTests/VectorSearchTests.cs +++ b/AzureSearchEmulator.UnitTests/VectorSearchTests.cs @@ -409,7 +409,7 @@ public async Task Metric_ChangesTheRanking() /// A document merged on an unrelated field stays findable by vector search. /// /// - /// The end-to-end half of the bug that MergeDocument's doc-values rebuild fixes. The + /// The end-to-end half of the bug where a merge dropped the packed doc-values copy. The /// storage test in phase 1 asserts the packed copy survives; this asserts what that survival /// is for. Without it the document would keep retrieving correctly while silently /// dropping out of every vector query — the failure is invisible from the retrieval side, diff --git a/AzureSearchEmulator/Indexing/MergeIndexDocumentAction.cs b/AzureSearchEmulator/Indexing/MergeIndexDocumentAction.cs index 97422ee..adda8aa 100644 --- a/AzureSearchEmulator/Indexing/MergeIndexDocumentAction.cs +++ b/AzureSearchEmulator/Indexing/MergeIndexDocumentAction.cs @@ -5,6 +5,6 @@ namespace AzureSearchEmulator.Indexing; public class MergeIndexDocumentAction(JsonObject item) : UpsertIndexDocumentActionBase(item) { - protected override void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable docFields) - => MergeDocument(context, keyTerm, docFields, ClearedFields, false); + protected override void IndexDocument(IndexingContext context, Term keyTerm) + => MergeDocument(context, keyTerm, false); } diff --git a/AzureSearchEmulator/Indexing/MergeOrUploadIndexDocumentAction.cs b/AzureSearchEmulator/Indexing/MergeOrUploadIndexDocumentAction.cs index 70392af..f6755bd 100644 --- a/AzureSearchEmulator/Indexing/MergeOrUploadIndexDocumentAction.cs +++ b/AzureSearchEmulator/Indexing/MergeOrUploadIndexDocumentAction.cs @@ -5,6 +5,6 @@ namespace AzureSearchEmulator.Indexing; public class MergeOrUploadIndexDocumentAction(JsonObject item) : UpsertIndexDocumentActionBase(item) { - protected override void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable docFields) - => MergeDocument(context, keyTerm, docFields, ClearedFields, true); + protected override void IndexDocument(IndexingContext context, Term keyTerm) + => MergeDocument(context, keyTerm, true); } diff --git a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs index fc6ad0c..312cee2 100644 --- a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs +++ b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs @@ -56,8 +56,9 @@ private static IEnumerable CreateFields( /// Reads the field's value back from a document's stored fields, or null when it has none. /// /// - /// Does not check , and a complex value comes back whole, - /// including sub-fields that are not retrievable: the caller decides what to return. + /// Does not check : a merge rebuilds the document from + /// whatever is stored, and a vector's stored copy is written even when it is not retrievable. + /// A complex value comes back whole, including sub-fields that are not retrievable. /// public static JsonNode? GetStoredValue(this SearchField field, Document doc) { diff --git a/AzureSearchEmulator/Indexing/UploadIndexDocumentAction.cs b/AzureSearchEmulator/Indexing/UploadIndexDocumentAction.cs index 21da125..139fd9d 100644 --- a/AzureSearchEmulator/Indexing/UploadIndexDocumentAction.cs +++ b/AzureSearchEmulator/Indexing/UploadIndexDocumentAction.cs @@ -5,6 +5,6 @@ namespace AzureSearchEmulator.Indexing; public class UploadIndexDocumentAction(JsonObject item) : UpsertIndexDocumentActionBase(item) { - protected override void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable docFields) - => context.Writer.UpdateDocument(keyTerm, docFields); + protected override void IndexDocument(IndexingContext context, Term keyTerm) + => context.Writer.UpdateDocument(keyTerm, GetDocFields(context.Index, Item)); } diff --git a/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs b/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs index 0bba3c0..693cbca 100644 --- a/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs +++ b/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs @@ -1,26 +1,12 @@ using System.Text.Json.Nodes; using AzureSearchEmulator.Models; -using AzureSearchEmulator.Searching; -using Lucene.Net.Documents; using Lucene.Net.Index; using Lucene.Net.Search; -using Lucene.Net.Util; namespace AzureSearchEmulator.Indexing; public abstract class UpsertIndexDocumentActionBase(JsonObject item) : IndexDocumentAction(item) { - /// - /// The index fields this batch item set to an explicit null, populated before - /// runs so a merge can clear them (issue #71). - /// - /// - /// Carried on the action rather than added to 's signature - /// because only the merge actions have any use for it; upload replaces the whole document, - /// where a null means "no value" and there is nothing to clear. - /// - protected IReadOnlyCollection ClearedFields { get; private set; } = []; - public override IndexingResult PerformIndexingAsync(IndexingContext context) { // GetKeyTerm throws when the batch item omits the key field. It has to run inside @@ -40,9 +26,7 @@ public override IndexingResult PerformIndexingAsync(IndexingContext context) try { - var fields = GetDocFields(context.Index); - ClearedFields = GetClearedFields(context.Index).ToList(); - IndexDocument(context, keyTerm, fields); + IndexDocument(context, keyTerm); return new IndexingResult(keyTerm.Text, true, 200); } catch (DocumentNotFoundException) @@ -55,53 +39,38 @@ public override IndexingResult PerformIndexingAsync(IndexingContext context) } } - protected IEnumerable GetDocFields(SearchIndex index) + protected static IEnumerable GetDocFields(SearchIndex index, JsonObject item) { return from f in index.Fields - join v in Item on f.Name equals v.Key + join v in item on f.Name equals v.Key where v.Value != null from indexField in f.CreateFields(v.Value!, index) // [!]: null checked by where clause select indexField; } /// - /// The fields the batch item named with an explicit JSON null, which a merge clears - /// (issue #71). + /// Merges the batch item into the stored document and indexes the result as an upload would. /// /// /// - /// Azure documents setting a field to null on a merge as the way to remove its value, and - /// distinguishes that from omitting the field, which leaves the existing value alone. - /// cannot carry the distinction: a null produces no Lucene field, - /// so a merge driven only by that list has nothing to tell it which names to remove and - /// silently keeps the old value. + /// The whole document is indexed again because returns + /// stored fields only. Writing those back as they are would drop every indexed and doc-values + /// copy of the fields the item does not supply, so filters, searches and vector queries would + /// stop matching them. + /// + /// + /// The item's fields replace the stored ones whole, as Azure does: a complex field or a + /// collection is not merged element by element. An explicit null in the item clears the field, + /// because skips null values. /// /// - /// Only the field names are needed, so the value itself is never converted — which is what - /// makes this safe for types that would reject a null, such as a geography point or a - /// vector whose length must match dimensions. + /// A field that is not retrievable has no stored copy (except a vector), so a merge that does + /// not supply it loses its value. /// /// - protected IEnumerable GetClearedFields(SearchIndex index) - { - return from f in index.Fields - join v in Item on f.Name equals v.Key - where v.Value is null - select f; - } - - protected static void MergeDocument(IndexingContext context, Term keyTerm, IEnumerable docFields, bool uploadIfMissing) - => MergeDocument(context, keyTerm, docFields, [], uploadIfMissing); - - protected static void MergeDocument( - IndexingContext context, - Term keyTerm, - IEnumerable docFields, - IEnumerable clearedFields, - bool uploadIfMissing) + protected void MergeDocument(IndexingContext context, Term keyTerm, bool uploadIfMissing) { - var reader = context.Reader.Value; - var searcher = new IndexSearcher(reader); + var searcher = new IndexSearcher(context.Reader.Value); var docs = searcher.Search(new TermQuery(keyTerm), 1); @@ -110,132 +79,28 @@ protected static void MergeDocument( throw new DocumentNotFoundException(); } - var doc = docs.TotalHits == 0 ? new Document() : searcher.Doc(docs.ScoreDocs[0].Doc); + var merged = new JsonObject(); - var materialized = docFields.ToList(); - foreach (var name in materialized.Select(f => f.Name).Distinct()) + if (docs.TotalHits > 0) { - doc.RemoveFields(name); - } - - // Runs after the replacements are removed and before they are added, so a field that is - // both cleared and supplied — which the JSON object cannot express, but a caller could - // reach through a complex field's sub-fields — keeps the supplied value. - foreach (var name in clearedFields.SelectMany(GetLuceneFieldNames)) - { - doc.RemoveFields(name); - } + var stored = searcher.Doc(docs.ScoreDocs[0].Doc); - foreach (var docField in materialized) - { - doc.Add(docField); - } - - RestoreVectorDocValues(context.Index, doc); - - context.Writer.UpdateDocument(keyTerm, doc.Fields); - } - - /// - /// Every Lucene field name a single index field can write, so that clearing it removes the - /// whole set rather than the retrievable copy alone (issue #71). - /// - /// - /// - /// One index field routinely occupies several Lucene names: a searchable string that is also - /// filterable writes a second, unanalyzed copy under __azs_raw__; a facetable field - /// adds doc values; a geography point writes a latitude, a longitude and a doc-values entry; - /// a vector writes a stored sidecar and a packed copy. Removing only the field's own name - /// would leave those behind, and a document still carrying the raw copy of a cleared string - /// keeps matching filters for a value it no longer has. - /// - /// - /// The names are produced by asking each helper rather than by reconstructing the prefixes - /// here, so a convention added later is picked up by changing its own helper's call site - /// alone. Names are emitted unconditionally instead of being predicted from the field's - /// attributes: ignores a name the document does not - /// carry, which makes over-listing free and under-listing the only real failure. - /// - /// - /// A complex field holds no value of its own, so clearing one means clearing every leaf - /// beneath it, plus the storage and doc-values entries the complex field itself writes. - /// - /// - private static IEnumerable GetLuceneFieldNames(SearchField field) - { - if (field.IsComplex()) - { - yield return ComplexTypeSupport.GetComplexStorageFieldName(field.Name); - yield return ComplexTypeSupport.GetComplexElementsDocValuesFieldName(field.Name); + foreach (var field in context.Index.Fields) + { + if (field.GetStoredValue(stored) is { } value) + { + merged[field.Name] = value; + } + } } - foreach (var (path, _) in ComplexTypeSupport.EnumerateLeafFields(field)) + foreach (var (name, value) in Item) { - yield return path; - yield return SearchFieldExtensions.GetRawStringFieldName(path); - yield return SearchFieldExtensions.GetCollectionStorageFieldName(path); - yield return FacetSupport.GetFacetDocValuesFieldName(path); - yield return GeoSupport.GetLatFieldName(path); - yield return GeoSupport.GetLonFieldName(path); - yield return GeoSupport.GetPointDocValuesFieldName(path); - yield return VectorSearchSupport.GetVectorDocValuesFieldName(path); + merged[name] = value?.DeepClone(); } - } - /// - /// Rebuilds the packed doc values of every vector field the merged document still holds a - /// stored value for (issue #46). - /// - /// - /// - /// returns stored fields and nothing else, so the - /// document reconstructed above has lost every doc-values field it had. For a vector that - /// matters more than it looks: the stored JSON sidecar is what retrieval reads, so a merged - /// document keeps coming back correctly, while the packed copy a query scans is gone — the - /// document silently stops being findable by vector search while still looking intact. - /// - /// - /// Rebuilding from the sidecar is possible precisely because the sidecar survives, which is - /// why it is the authoritative copy. A vector whose sidecar is absent is skipped rather than - /// zero-filled: the field simply has no value for this document, and inventing one would put - /// it in results it does not belong in. - /// - /// - /// Deliberately scoped to vector fields. The same reconstruction drops facet, geo and - /// complex-element doc values too, which is a pre-existing fault worth fixing on its own - /// terms rather than quietly here, where it would be neither tested nor reviewed as such. - /// - /// - private static void RestoreVectorDocValues(SearchIndex index, Document doc) - { - foreach (var (path, field) in ComplexTypeSupport.EnumerateLeafFields(index)) - { - if (!field.IsVectorField()) - { - continue; - } - - var docValuesName = VectorSearchSupport.GetVectorDocValuesFieldName(path); - - // Present already when this merge supplied the vector itself, in which case the - // fields built for it are the ones to keep. - if (doc.GetField(docValuesName) != null) - { - continue; - } - - var storedJson = doc.Get(SearchFieldExtensions.GetCollectionStorageFieldName(path)); - - if (storedJson == null || JsonNode.Parse(storedJson) is not JsonArray array) - { - continue; - } - - var vector = VectorSearchSupport.ParseVector(path, array, field.Dimensions); - - doc.Add(new BinaryDocValuesField(docValuesName, new BytesRef(VectorSearchSupport.PackVector(vector)))); - } + context.Writer.UpdateDocument(keyTerm, GetDocFields(context.Index, merged)); } - protected abstract void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable docFields); + protected abstract void IndexDocument(IndexingContext context, Term keyTerm); }