diff --git a/AzureSearchEmulator.UnitTests/MergeKeepsFieldsIndexedTests.cs b/AzureSearchEmulator.UnitTests/MergeKeepsFieldsIndexedTests.cs new file mode 100644 index 0000000..e5cf109 --- /dev/null +++ b/AzureSearchEmulator.UnitTests/MergeKeepsFieldsIndexedTests.cs @@ -0,0 +1,199 @@ +using System.Text.Json.Nodes; +using AzureSearchEmulator.Indexing; +using AzureSearchEmulator.Models; +using AzureSearchEmulator.SearchData; +using AzureSearchEmulator.Searching; +using Lucene.Net.Index; +using Lucene.Net.Store; +using Xunit; + +namespace AzureSearchEmulator.UnitTests; + +/// +/// Covers the fields a merge does not supply. They must stay searchable, filterable and +/// facetable, not only retrievable. +/// +/// +/// A merge used to write back the stored fields of the old document as they were. Stored fields +/// carry no indexed terms or doc values, so after any merge the document still came back from a +/// lookup by key but no longer matched a filter or a search on its other fields. +/// +public class MergeKeepsFieldsIndexedTests : IDisposable +{ + private readonly RAMDirectory _directory = new(); + private readonly SearchIndex _index; + private readonly LuceneNetIndexWriterFactory _writerFactory; + private readonly LuceneNetSearchIndexer _indexer; + private readonly LuceneNetIndexSearcher _searcher; + + public MergeKeepsFieldsIndexedTests() + { + _index = new SearchIndex + { + Name = "mergeable", + Fields = + [ + new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true }, + new SearchField { Name = "Source", Type = "Edm.String", Searchable = false, Filterable = true }, + new SearchField { Name = "Content", Type = "Edm.String", Searchable = true }, + new SearchField { Name = "Groups", Type = "Collection(Edm.String)", Searchable = false, Filterable = true }, + new SearchField { Name = "Rating", Type = "Edm.Int32", Filterable = true, Facetable = true }, + new SearchField { Name = "Published", Type = "Edm.Boolean", Filterable = true }, + new SearchField { Name = "Updated", Type = "Edm.DateTimeOffset", Filterable = true }, + new SearchField { Name = "Location", Type = "Edm.GeographyPoint", Filterable = true }, + new SearchField + { + Name = "Address", + Type = "Edm.ComplexType", + Fields = [new SearchField { Name = "City", Type = "Edm.String", Searchable = false, Filterable = true }], + }, + new SearchField + { + Name = "Files", + Type = "Collection(Edm.ComplexType)", + Fields = [new SearchField { Name = "Path", Type = "Edm.String", Searchable = false, Filterable = true }], + }, + new SearchField { Name = "Title", Type = "Edm.String", Searchable = true, Filterable = true }, + ] + }; + + var factory = new SharedDirectoryFactory(_directory); + _writerFactory = new LuceneNetIndexWriterFactory(factory); + _indexer = new LuceneNetSearchIndexer(_writerFactory, factory); + _searcher = new LuceneNetIndexSearcher(factory); + } + + public void Dispose() + { + // Writer before directory: it holds an open handle and commits on dispose. + _writerFactory.Dispose(); + _directory.Dispose(); + } + + // Parsed from text, as a request body is: a date built in C# as a string JsonValue cannot be + // read back as a DateTimeOffset. + private static JsonObject Doc(string id) => JsonNode.Parse($$""" + { + "Id": "{{id}}", + "Source": "blob-{{id}}", + "Content": "quarterly board minutes", + "Groups": ["finance", "legal"], + "Rating": 4, + "Published": true, + "Updated": "2025-03-01T00:00:00Z", + "Location": { "type": "Point", "coordinates": [-1.55, 53.8] }, + "Address": { "City": "Leeds" }, + "Files": [{ "Path": "Entities/Company 4" }], + "Title": "Old title" + } + """)!.AsObject(); + + private static JsonObject TitleChange(string id) => new() { ["Id"] = id, ["Title"] = "New title" }; + + private void Index(IndexDocumentAction action) + { + var result = Assert.Single(_indexer.IndexDocuments(_index, [action]).Value); + Assert.True(result.Status, result.ErrorMessage); + } + + private async Task Count(SearchRequest request) + { + request.Count = true; + return (await _searcher.Search(_index, request)).Count; + } + + public static TheoryData FiltersOnUntouchedFields => + [ + "Id eq '1' and Source eq 'blob-1'", + "Groups/any(g: search.in(g, 'finance'))", + "Rating eq 4", + "Published eq true", + "Updated eq 2025-03-01T00:00:00Z", + "geo.distance(Location, geography'POINT(-1.55 53.8)') le 1", + "Address/City eq 'Leeds'", + "Files/any(f: f/Path eq 'Entities/Company 4')", + ]; + + [Theory] + [MemberData(nameof(FiltersOnUntouchedFields))] + public async Task Merge_LeavesUntouchedFieldsFilterable(string filter) + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + + Assert.Equal(1, await Count(new SearchRequest { Filter = filter })); + } + + [Theory] + [MemberData(nameof(FiltersOnUntouchedFields))] + public async Task MergeOrUpload_OnExistingDocument_LeavesUntouchedFieldsFilterable(string filter) + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeOrUploadIndexDocumentAction(TitleChange("1"))); + + Assert.Equal(1, await Count(new SearchRequest { Filter = filter })); + } + + [Fact] + public async Task Merge_LeavesUntouchedFieldsSearchable() + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + + Assert.Equal(1, await Count(new SearchRequest { Search = "minutes", SearchFields = "Content" })); + } + + [Fact] + public async Task Merge_LeavesUntouchedFieldsFacetable() + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + + var response = await _searcher.Search(_index, new SearchRequest { Facets = ["Rating"] }); + + var bucket = Assert.Single(response.Facets!["Rating"]); + Assert.Equal(1, bucket.Count); + } + + [Fact] + public async Task Merge_ReplacesTheSuppliedFieldInTheIndex() + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + + Assert.Equal(0, await Count(new SearchRequest { Filter = "Title eq 'Old title'" })); + Assert.Equal(1, await Count(new SearchRequest { Filter = "Title eq 'New title'" })); + Assert.Equal(1, await Count(new SearchRequest { Search = "new", SearchFields = "Title" })); + } + + /// + /// A second merge reads back the document the first one wrote, so it must also find every + /// field stored. + /// + [Fact] + public async Task RepeatedMerges_KeepEveryFieldRetrievable() + { + Index(new UploadIndexDocumentAction(Doc("1"))); + Index(new MergeIndexDocumentAction(TitleChange("1"))); + Index(new MergeIndexDocumentAction(new JsonObject { ["Id"] = "1", ["Rating"] = 5 })); + + var doc = await _searcher.GetDoc(_index, "1"); + + Assert.NotNull(doc); + Assert.Equal(Doc("1").Select(p => p.Key).Order(), doc!.Select(p => p.Key).Order()); + Assert.Equal("blob-1", doc["Source"]?.GetValue()); + Assert.Equal("New title", doc["Title"]?.GetValue()); + Assert.Equal(5, doc["Rating"]?.GetValue()); + Assert.Equal("Leeds", doc["Address"]?["City"]?.GetValue()); + Assert.Equal(1, await Count(new SearchRequest { Filter = "Rating eq 5 and Source eq 'blob-1'" })); + } + + private class SharedDirectoryFactory(Lucene.Net.Store.Directory directory) : ILuceneDirectoryFactory, ILuceneIndexReaderFactory + { + public Lucene.Net.Store.Directory GetDirectory(string indexName) => directory; + public void ClearCachedDirectory(string indexName) { } + public IndexReader GetIndexReader(string indexName) => DirectoryReader.Open(directory); + public IndexReader RefreshReader(string indexName) => GetIndexReader(indexName); + public void ClearCachedReader(string indexName) { } + } +} diff --git a/AzureSearchEmulator.UnitTests/MergeNullClearsFieldTests.cs b/AzureSearchEmulator.UnitTests/MergeNullClearsFieldTests.cs index b7b8dc5..1fcd044 100644 --- a/AzureSearchEmulator.UnitTests/MergeNullClearsFieldTests.cs +++ b/AzureSearchEmulator.UnitTests/MergeNullClearsFieldTests.cs @@ -106,8 +106,7 @@ public async Task Merge_OmittingAField_LeavesItAlone() /// /// A cleared string must also lose the unanalyzed __azs_raw__ copy that filtering - /// reads, or the document keeps matching a filter for a value it no longer has — the - /// failure that motivated clearing every name a field writes rather than just its own. + /// reads, or the document keeps matching a filter for a value it no longer has. /// [Fact] public async Task Merge_WithExplicitNull_AlsoClearsTheFilterableCopy() diff --git a/AzureSearchEmulator.UnitTests/VectorSearchTests.cs b/AzureSearchEmulator.UnitTests/VectorSearchTests.cs index 8c5a3bb..6840ca4 100644 --- a/AzureSearchEmulator.UnitTests/VectorSearchTests.cs +++ b/AzureSearchEmulator.UnitTests/VectorSearchTests.cs @@ -409,7 +409,7 @@ public async Task Metric_ChangesTheRanking() /// A document merged on an unrelated field stays findable by vector search. /// /// - /// The end-to-end half of the bug that MergeDocument's doc-values rebuild fixes. The + /// The end-to-end half of the bug where a merge dropped the packed doc-values copy. The /// storage test in phase 1 asserts the packed copy survives; this asserts what that survival /// is for. Without it the document would keep retrieving correctly while silently /// dropping out of every vector query — the failure is invisible from the retrieval side, diff --git a/AzureSearchEmulator/Indexing/MergeIndexDocumentAction.cs b/AzureSearchEmulator/Indexing/MergeIndexDocumentAction.cs index 97422ee..adda8aa 100644 --- a/AzureSearchEmulator/Indexing/MergeIndexDocumentAction.cs +++ b/AzureSearchEmulator/Indexing/MergeIndexDocumentAction.cs @@ -5,6 +5,6 @@ namespace AzureSearchEmulator.Indexing; public class MergeIndexDocumentAction(JsonObject item) : UpsertIndexDocumentActionBase(item) { - protected override void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable docFields) - => MergeDocument(context, keyTerm, docFields, ClearedFields, false); + protected override void IndexDocument(IndexingContext context, Term keyTerm) + => MergeDocument(context, keyTerm, false); } diff --git a/AzureSearchEmulator/Indexing/MergeOrUploadIndexDocumentAction.cs b/AzureSearchEmulator/Indexing/MergeOrUploadIndexDocumentAction.cs index 70392af..f6755bd 100644 --- a/AzureSearchEmulator/Indexing/MergeOrUploadIndexDocumentAction.cs +++ b/AzureSearchEmulator/Indexing/MergeOrUploadIndexDocumentAction.cs @@ -5,6 +5,6 @@ namespace AzureSearchEmulator.Indexing; public class MergeOrUploadIndexDocumentAction(JsonObject item) : UpsertIndexDocumentActionBase(item) { - protected override void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable docFields) - => MergeDocument(context, keyTerm, docFields, ClearedFields, true); + protected override void IndexDocument(IndexingContext context, Term keyTerm) + => MergeDocument(context, keyTerm, true); } diff --git a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs index ba1f781..312cee2 100644 --- a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs +++ b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs @@ -52,6 +52,62 @@ private static IEnumerable CreateFields( return CreateSingleValueFields(field, value, path, index); } + /// + /// Reads the field's value back from a document's stored fields, or null when it has none. + /// + /// + /// Does not check : a merge rebuilds the document from + /// whatever is stored, and a vector's stored copy is written even when it is not retrievable. + /// A complex value comes back whole, including sub-fields that are not retrievable. + /// + public static JsonNode? GetStoredValue(this SearchField field, Document doc) + { + if (field.IsComplex()) + { + // Read from the stored JSON copy rather than the flattened leaves, which cannot say + // which values belonged to which element of a Collection(Edm.ComplexType). + var storedComplexJson = doc.Get(ComplexTypeSupport.GetComplexStorageFieldName(field.Name)); + return storedComplexJson == null ? null : JsonNode.Parse(storedComplexJson); + } + + if (field.IsCollection()) + { + var storedJson = doc.Get(GetCollectionStorageFieldName(field.Name)); + return storedJson == null ? null : JsonNode.Parse(storedJson); + } + + if (field.Type == GeoSupport.GeographyPointType) + { + // Points are stored as a separate lat/lon pair rather than under the + // field's own name, so they need their own lookup. + var latField = doc.GetField(GeoSupport.GetLatFieldName(field.Name)); + var lonField = doc.GetField(GeoSupport.GetLonFieldName(field.Name)); + + return latField?.GetDoubleValue() is double storedLat + && lonField?.GetDoubleValue() is double storedLon + ? GeoSupport.CreateGeoJsonPoint(storedLon, storedLat) + : null; + } + + var docField = doc.GetField(field.Name); + + if (docField == null) + { + return null; + } + + return field.Type switch + { + "Edm.String" => docField.GetStringValue(), + "Edm.Int32" => docField.GetInt32Value(), + "Edm.Int64" => docField.GetInt64Value(), + "Edm.Double" => docField.GetDoubleValue(), + "Edm.Boolean" => docField.GetInt32Value() is int i ? i != 0 : null, + "Edm.DateTimeOffset" => docField.GetInt64Value() is long ms ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : null, + _ => throw new InvalidOperationException($"Unsupported field type {field.Type}") + }; + } + private static IEnumerable CreateSingleValueFields( SearchField field, JsonNode value, diff --git a/AzureSearchEmulator/Indexing/UploadIndexDocumentAction.cs b/AzureSearchEmulator/Indexing/UploadIndexDocumentAction.cs index 21da125..139fd9d 100644 --- a/AzureSearchEmulator/Indexing/UploadIndexDocumentAction.cs +++ b/AzureSearchEmulator/Indexing/UploadIndexDocumentAction.cs @@ -5,6 +5,6 @@ namespace AzureSearchEmulator.Indexing; public class UploadIndexDocumentAction(JsonObject item) : UpsertIndexDocumentActionBase(item) { - protected override void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable docFields) - => context.Writer.UpdateDocument(keyTerm, docFields); + protected override void IndexDocument(IndexingContext context, Term keyTerm) + => context.Writer.UpdateDocument(keyTerm, GetDocFields(context.Index, Item)); } diff --git a/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs b/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs index 0bba3c0..693cbca 100644 --- a/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs +++ b/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs @@ -1,26 +1,12 @@ using System.Text.Json.Nodes; using AzureSearchEmulator.Models; -using AzureSearchEmulator.Searching; -using Lucene.Net.Documents; using Lucene.Net.Index; using Lucene.Net.Search; -using Lucene.Net.Util; namespace AzureSearchEmulator.Indexing; public abstract class UpsertIndexDocumentActionBase(JsonObject item) : IndexDocumentAction(item) { - /// - /// The index fields this batch item set to an explicit null, populated before - /// runs so a merge can clear them (issue #71). - /// - /// - /// Carried on the action rather than added to 's signature - /// because only the merge actions have any use for it; upload replaces the whole document, - /// where a null means "no value" and there is nothing to clear. - /// - protected IReadOnlyCollection ClearedFields { get; private set; } = []; - public override IndexingResult PerformIndexingAsync(IndexingContext context) { // GetKeyTerm throws when the batch item omits the key field. It has to run inside @@ -40,9 +26,7 @@ public override IndexingResult PerformIndexingAsync(IndexingContext context) try { - var fields = GetDocFields(context.Index); - ClearedFields = GetClearedFields(context.Index).ToList(); - IndexDocument(context, keyTerm, fields); + IndexDocument(context, keyTerm); return new IndexingResult(keyTerm.Text, true, 200); } catch (DocumentNotFoundException) @@ -55,53 +39,38 @@ public override IndexingResult PerformIndexingAsync(IndexingContext context) } } - protected IEnumerable GetDocFields(SearchIndex index) + protected static IEnumerable GetDocFields(SearchIndex index, JsonObject item) { return from f in index.Fields - join v in Item on f.Name equals v.Key + join v in item on f.Name equals v.Key where v.Value != null from indexField in f.CreateFields(v.Value!, index) // [!]: null checked by where clause select indexField; } /// - /// The fields the batch item named with an explicit JSON null, which a merge clears - /// (issue #71). + /// Merges the batch item into the stored document and indexes the result as an upload would. /// /// /// - /// Azure documents setting a field to null on a merge as the way to remove its value, and - /// distinguishes that from omitting the field, which leaves the existing value alone. - /// cannot carry the distinction: a null produces no Lucene field, - /// so a merge driven only by that list has nothing to tell it which names to remove and - /// silently keeps the old value. + /// The whole document is indexed again because returns + /// stored fields only. Writing those back as they are would drop every indexed and doc-values + /// copy of the fields the item does not supply, so filters, searches and vector queries would + /// stop matching them. + /// + /// + /// The item's fields replace the stored ones whole, as Azure does: a complex field or a + /// collection is not merged element by element. An explicit null in the item clears the field, + /// because skips null values. /// /// - /// Only the field names are needed, so the value itself is never converted — which is what - /// makes this safe for types that would reject a null, such as a geography point or a - /// vector whose length must match dimensions. + /// A field that is not retrievable has no stored copy (except a vector), so a merge that does + /// not supply it loses its value. /// /// - protected IEnumerable GetClearedFields(SearchIndex index) - { - return from f in index.Fields - join v in Item on f.Name equals v.Key - where v.Value is null - select f; - } - - protected static void MergeDocument(IndexingContext context, Term keyTerm, IEnumerable docFields, bool uploadIfMissing) - => MergeDocument(context, keyTerm, docFields, [], uploadIfMissing); - - protected static void MergeDocument( - IndexingContext context, - Term keyTerm, - IEnumerable docFields, - IEnumerable clearedFields, - bool uploadIfMissing) + protected void MergeDocument(IndexingContext context, Term keyTerm, bool uploadIfMissing) { - var reader = context.Reader.Value; - var searcher = new IndexSearcher(reader); + var searcher = new IndexSearcher(context.Reader.Value); var docs = searcher.Search(new TermQuery(keyTerm), 1); @@ -110,132 +79,28 @@ protected static void MergeDocument( throw new DocumentNotFoundException(); } - var doc = docs.TotalHits == 0 ? new Document() : searcher.Doc(docs.ScoreDocs[0].Doc); + var merged = new JsonObject(); - var materialized = docFields.ToList(); - foreach (var name in materialized.Select(f => f.Name).Distinct()) + if (docs.TotalHits > 0) { - doc.RemoveFields(name); - } - - // Runs after the replacements are removed and before they are added, so a field that is - // both cleared and supplied — which the JSON object cannot express, but a caller could - // reach through a complex field's sub-fields — keeps the supplied value. - foreach (var name in clearedFields.SelectMany(GetLuceneFieldNames)) - { - doc.RemoveFields(name); - } + var stored = searcher.Doc(docs.ScoreDocs[0].Doc); - foreach (var docField in materialized) - { - doc.Add(docField); - } - - RestoreVectorDocValues(context.Index, doc); - - context.Writer.UpdateDocument(keyTerm, doc.Fields); - } - - /// - /// Every Lucene field name a single index field can write, so that clearing it removes the - /// whole set rather than the retrievable copy alone (issue #71). - /// - /// - /// - /// One index field routinely occupies several Lucene names: a searchable string that is also - /// filterable writes a second, unanalyzed copy under __azs_raw__; a facetable field - /// adds doc values; a geography point writes a latitude, a longitude and a doc-values entry; - /// a vector writes a stored sidecar and a packed copy. Removing only the field's own name - /// would leave those behind, and a document still carrying the raw copy of a cleared string - /// keeps matching filters for a value it no longer has. - /// - /// - /// The names are produced by asking each helper rather than by reconstructing the prefixes - /// here, so a convention added later is picked up by changing its own helper's call site - /// alone. Names are emitted unconditionally instead of being predicted from the field's - /// attributes: ignores a name the document does not - /// carry, which makes over-listing free and under-listing the only real failure. - /// - /// - /// A complex field holds no value of its own, so clearing one means clearing every leaf - /// beneath it, plus the storage and doc-values entries the complex field itself writes. - /// - /// - private static IEnumerable GetLuceneFieldNames(SearchField field) - { - if (field.IsComplex()) - { - yield return ComplexTypeSupport.GetComplexStorageFieldName(field.Name); - yield return ComplexTypeSupport.GetComplexElementsDocValuesFieldName(field.Name); + foreach (var field in context.Index.Fields) + { + if (field.GetStoredValue(stored) is { } value) + { + merged[field.Name] = value; + } + } } - foreach (var (path, _) in ComplexTypeSupport.EnumerateLeafFields(field)) + foreach (var (name, value) in Item) { - yield return path; - yield return SearchFieldExtensions.GetRawStringFieldName(path); - yield return SearchFieldExtensions.GetCollectionStorageFieldName(path); - yield return FacetSupport.GetFacetDocValuesFieldName(path); - yield return GeoSupport.GetLatFieldName(path); - yield return GeoSupport.GetLonFieldName(path); - yield return GeoSupport.GetPointDocValuesFieldName(path); - yield return VectorSearchSupport.GetVectorDocValuesFieldName(path); + merged[name] = value?.DeepClone(); } - } - /// - /// Rebuilds the packed doc values of every vector field the merged document still holds a - /// stored value for (issue #46). - /// - /// - /// - /// returns stored fields and nothing else, so the - /// document reconstructed above has lost every doc-values field it had. For a vector that - /// matters more than it looks: the stored JSON sidecar is what retrieval reads, so a merged - /// document keeps coming back correctly, while the packed copy a query scans is gone — the - /// document silently stops being findable by vector search while still looking intact. - /// - /// - /// Rebuilding from the sidecar is possible precisely because the sidecar survives, which is - /// why it is the authoritative copy. A vector whose sidecar is absent is skipped rather than - /// zero-filled: the field simply has no value for this document, and inventing one would put - /// it in results it does not belong in. - /// - /// - /// Deliberately scoped to vector fields. The same reconstruction drops facet, geo and - /// complex-element doc values too, which is a pre-existing fault worth fixing on its own - /// terms rather than quietly here, where it would be neither tested nor reviewed as such. - /// - /// - private static void RestoreVectorDocValues(SearchIndex index, Document doc) - { - foreach (var (path, field) in ComplexTypeSupport.EnumerateLeafFields(index)) - { - if (!field.IsVectorField()) - { - continue; - } - - var docValuesName = VectorSearchSupport.GetVectorDocValuesFieldName(path); - - // Present already when this merge supplied the vector itself, in which case the - // fields built for it are the ones to keep. - if (doc.GetField(docValuesName) != null) - { - continue; - } - - var storedJson = doc.Get(SearchFieldExtensions.GetCollectionStorageFieldName(path)); - - if (storedJson == null || JsonNode.Parse(storedJson) is not JsonArray array) - { - continue; - } - - var vector = VectorSearchSupport.ParseVector(path, array, field.Dimensions); - - doc.Add(new BinaryDocValuesField(docValuesName, new BytesRef(VectorSearchSupport.PackVector(vector)))); - } + context.Writer.UpdateDocument(keyTerm, GetDocFields(context.Index, merged)); } - protected abstract void IndexDocument(IndexingContext context, Term keyTerm, IEnumerable docFields); + protected abstract void IndexDocument(IndexingContext context, Term keyTerm); } diff --git a/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs b/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs index 2d049f5..53e7249 100644 --- a/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs +++ b/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs @@ -959,65 +959,16 @@ private static JsonObject ConvertSearchDoc( continue; } - if (field.IsComplex()) - { - // Complex values round-trip through the stored JSON sidecar rather than the - // flattened leaves, which cannot say which values belonged to which element - // of a Collection(Edm.ComplexType). - var storedComplexJson = doc.Get(ComplexTypeSupport.GetComplexStorageFieldName(field.Name)); - if (storedComplexJson != null) - { - result[field.Name] = FilterToRetrievableSubFields( - field, - JsonNode.Parse(storedComplexJson), - selection?.GetSubSelection(field.Name)); - } - continue; - } + var value = field.GetStoredValue(doc); - if (field.IsCollection()) + if (value == null) { - var storedJson = doc.Get(SearchFieldExtensions.GetCollectionStorageFieldName(field.Name)); - if (storedJson != null) - { - result[field.Name] = JsonNode.Parse(storedJson); - } continue; } - if (field.Type == GeoSupport.GeographyPointType) - { - // Points are stored as a separate lat/lon pair rather than under the - // field's own name, so they need their own lookup. - var latField = doc.GetField(GeoSupport.GetLatFieldName(field.Name)); - var lonField = doc.GetField(GeoSupport.GetLonFieldName(field.Name)); - - if (latField?.GetDoubleValue() is double storedLat - && lonField?.GetDoubleValue() is double storedLon) - { - result[field.Name] = GeoSupport.CreateGeoJsonPoint(storedLon, storedLat); - } - - continue; - } - - var docField = doc.GetField(field.Name); - - if (docField != null) - { - result[field.Name] = field.Type switch - { - "Edm.String" => docField.GetStringValue(), - "Edm.Int32" => docField.GetInt32Value(), - "Edm.Int64" => docField.GetInt64Value(), - "Edm.Double" => docField.GetDoubleValue(), - "Edm.Boolean" => docField.GetInt32Value() is int i ? i != 0 : null, - "Edm.DateTimeOffset" => docField.GetInt64Value() is long ms ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : null, - // Edm.GeographyPoint is handled above, before this switch. - "Edm.ComplexType" => throw new NotImplementedException(), - _ => throw new InvalidOperationException($"Unsupported field type {field.Type}") - }; - } + result[field.Name] = field.IsComplex() + ? FilterToRetrievableSubFields(field, value, selection?.GetSubSelection(field.Name)) + : value; } return result;