From 7c008ea3c5dbde5830f326ead8caff832b6dc3c4 Mon Sep 17 00:00:00 2001 From: Matthew Lorimor Date: Thu, 5 Nov 2026 19:44:18 -0600 Subject: [PATCH 1/2] Test that a MementoFilter read holds each table once, read in pieces (red) (#327) Two tests, as #313 has for InfiniFilter. TestAReadHoldsEachTableOnce (TestMementoFilter) reads a filter of 2-bit fingerprints built for 2,048 keys and given 48,000, whose chain holds tables of 98,344 and 12,328 bytes, and holds what the read allocates, counted in a no-GC region, and what it is charged, to its payload and one copy of its tables plus 4 KiB: in the ordinary envelope, and in the large form, where the payload is not held, to the tables alone. Each is held from below too. Both tables are past the slack, which a premise checks. Red on 5752769 (fix/329-read-path-leftovers, on 05a5669), at the first check: the read allocated 333,024 bytes, 111,676 more than its payload and its tables That is the second copy of the tables (110,672) and 1,004 besides. The test prints every figure first, and on 5752769 they are: payload 110,676, tables 110,672 ordinary read 333,024, 111,676 past payload + tables ordinary charge 332,312, 110,964 past payload + tables large-form read 222,424, 111,752 past the tables large-form charge 221,712, 111,040 past the tables TestAMementoFiltersTableIsReadFromTheLargeFormInPiecesStraightIntoIts- Segment (TestLargeEnvelopeReading) holds the budget sample's table, found as the bytes its words are written as, to being read in pieces of the shipped 16 KiB through the stream's span overload, one after another, with nothing among them, and in pieces of eight words with the piece lowered, as the Infini test does. Red on 5752769: the table's 73,744 bytes at 58, with the piece at 16,384, were read in 9,218 reads, the first of 8 bytes through the span overload, and not in 5 pieces through the span overload which is the reader's loop of ReadUInt64, a word asked of the stream at a time. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KK3MspqAcczxT45xCfLgyx --- .../TestLargeEnvelopeReading.cs | 56 +++++++++++++++ .../TestMementoFilter.cs | 71 +++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/TestProbabilisticDataStructures/TestLargeEnvelopeReading.cs b/TestProbabilisticDataStructures/TestLargeEnvelopeReading.cs index c72a7393..f178137b 100644 --- a/TestProbabilisticDataStructures/TestLargeEnvelopeReading.cs +++ b/TestProbabilisticDataStructures/TestLargeEnvelopeReading.cs @@ -1190,6 +1190,62 @@ public void TestACountMinSketchsCountersAreReadFromTheLargeFormInPiecesStraightI } } + /// + /// A 's table, whose words have no length before them, is read + /// from the large form in pieces straight into its segment's table (#327), as an Infini + /// filter's is (#313): through the stream's span overload, every piece as long as a piece + /// but the last, one after another from the table's first byte to its last, and nothing + /// else read among them. Shown with the piece as it ships and lowered to eight words. + /// + /// + /// Before #327 the words were read a word at a time into an array of their own, eight + /// bytes asked of the stream and appended to the checksum for each, and then copied into + /// the table. The table is found as the bytes its words are written as, which nothing + /// else in the payload repeats; the budget sample is one table longer than a piece as it + /// ships, so it is read in more than one at either length. + /// + [TestMethod] + public void TestAMementoFiltersTableIsReadFromTheLargeFormInPiecesStraightIntoItsSegment() + { + var sample = TestReadBudget.Samples.Value[StructureId.MementoFilter]; + var large = TestReadBudget.InTheLargeForm(sample).Bytes; + var filter = (MementoFilter)sample.Read(sample.Bytes, ReadOptions.Unlimited); + Assert.HasCount(1, filter.Segments, "premise: the sample is not one table"); + + var words = filter.Segments[0].Data; + var table = new byte[words.Length * sizeof(ulong)]; + for (var w = 0; w < words.Length; w++) + { + BinaryPrimitives.WriteUInt64LittleEndian(table.AsSpan(w * sizeof(ulong)), words[w]); + } + + long at = large.AsSpan().IndexOf(table); + Assert.IsGreaterThanOrEqualTo(0L, at, "the table is not in the payload in the large form"); + Assert.AreEqual(at, (long)large.AsSpan().LastIndexOf(table), "the table appears in the payload twice"); + Assert.IsGreaterThan((long)ShippedBlockPiece, (long)table.Length, + $"the table's {table.Length:N0} bytes are no longer than a piece as it ships"); + var end = at + table.Length; + + foreach (var piece in new int?[] { null, Piece }) + { + var length = (long)(piece ?? ShippedBlockPiece); + var reads = SecondPass(sample, large, piece).Where(a => a.From < end && at < a.To).ToList(); + var expected = new List(); + for (var from = at; from < end; from += length) + { + expected.Add(new Asked(false, from, Math.Min(from + length, end), false)); + } + + Console.WriteLine( + $"[TABLE] MementoFilter: {table.Length:N0} bytes at {at:N0}, piece {length:N0}: " + + $"{reads.Count:N0} reads, the first {reads[0].Count:N0} bytes"); + CollectionAssert.AreEqual(expected, reads, + $"the table's {table.Length:N0} bytes at {at:N0}, with the piece at {length:N0}, were read in " + + $"{reads.Count:N0} reads, the first of {reads[0].Count:N0} bytes through the " + + $"{(reads[0].Array ? "array" : "span")} overload, and not in {expected.Count:N0} pieces through the span overload"); + } + } + /// /// An 's table, whose words have no length before them, is read /// from the large form in pieces straight into its segment's table (#313), as a count-min diff --git a/TestProbabilisticDataStructures/TestMementoFilter.cs b/TestProbabilisticDataStructures/TestMementoFilter.cs index 4a261c8d..df8fafc5 100644 --- a/TestProbabilisticDataStructures/TestMementoFilter.cs +++ b/TestProbabilisticDataStructures/TestMementoFilter.cs @@ -241,6 +241,77 @@ public void TestRemovingLeavesEverythingElseIntact() } } + /// + /// A read holds each table once and is charged for it once (#327), as an + /// read is (#313). In the ordinary envelope, what it allocates + /// and is charged beyond the payload it holds is the tables it builds, read into them and + /// not first into arrays of their own and copied; in the large form, where the payload is + /// not held, it is the tables alone. + /// + /// + /// Two-bit fingerprints are the narrowest, and an entry's runs out after the fewest + /// expansions, so a filter built for 2,048 keys and given 48,000 keeps a table behind the + /// first, of the size it had when they ran out. The slack, 4 KiB, is short of the first + /// table and of the second, which a premise checks, so a second copy of either fails it, + /// as one of every table does. Each figure is held from below too, to the payload and the tables, + /// so that a measurement of nothing cannot pass. The reads are counted in a no-GC region + /// (#293). + /// + [TestMethod] + public void TestAReadHoldsEachTableOnce() + { + const long Slack = 4 * 1024; + + var filter = new MementoFilter(64, 2, initialCapacity: 2048); + for (ulong i = 0; i < 48_000; i++) + { + filter.Add(i * 7919); + } + + long TableBytes(MementoSegment segment) => ReadBudget.ArrayOf((ulong)segment.Data.Length, sizeof(ulong)); + var tables = filter.Segments.Sum(TableBytes); + Assert.IsGreaterThan(1, filter.Segments.Count, "premise: the chain does not hold tables behind the first"); + Assert.IsGreaterThan(Slack, TableBytes(filter.Segments[1]), + $"premise: a copy of the second table could hide in the slack: tables of {string.Join(", ", filter.Segments.Select(TableBytes))} bytes"); + + var bytes = filter.ToByteArray(); + long payload = bytes.Length - PersistenceFormat.EnvelopeLength; + MementoFilter.ReadFrom(new MemoryStream(bytes, writable: false)); + var ordinary = NoGCRegion.Allocated(() => MementoFilter.ReadFrom(new MemoryStream(bytes, writable: false))); + var charged = CapacityModel.ChargeOf(StructureId.MementoFilter, bytes); + + var large = LargeForm.WriteLarge(filter); + Assert.IsTrue(LargeForm.IsLarge(large), "the filter was not written in the large form"); + MementoFilter.ReadFrom(new MemoryStream(large, writable: false)); + var streamed = NoGCRegion.Allocated(() => MementoFilter.ReadFrom(new MemoryStream(large, writable: false))); + var budget = new ReadBudget(long.MaxValue, StructureId.MementoFilter); + CollectionAssert.AreEqual(bytes, + MementoFilter.ReadFrom(new MemoryStream(large, writable: false), new ReadOptions(budget)).ToByteArray(), + "the large form does not read back as the filter"); + + Console.WriteLine( + $"[HOLDS ONCE] MementoFilter: tables of {string.Join(", ", filter.Segments.Select(TableBytes))} bytes, {tables:N0} in all, payload {payload:N0}; " + + $"ordinary read {ordinary:N0}, charged {charged:N0}; large-form read {streamed:N0}, charged {budget.Charged:N0}"); + + Assert.IsGreaterThanOrEqualTo(payload + tables, ordinary, + $"the read allocated {ordinary:N0} bytes, less than the {payload:N0}-byte payload it holds and the {tables:N0} bytes of its tables"); + Assert.IsLessThanOrEqualTo(payload + tables + Slack, ordinary, + $"the read allocated {ordinary:N0} bytes, {ordinary - payload - tables:N0} more than its payload and its tables"); + Assert.IsGreaterThanOrEqualTo(payload + tables, charged, + $"the read was charged {charged:N0} bytes, less than its payload and its tables"); + Assert.IsLessThanOrEqualTo(payload + tables + Slack, charged, + $"the read was charged {charged:N0} bytes, {charged - payload - tables:N0} more than its payload and its tables"); + + Assert.IsGreaterThanOrEqualTo(tables, streamed, + $"the read of the large form allocated {streamed:N0} bytes, less than the {tables:N0} bytes of its tables"); + Assert.IsLessThanOrEqualTo(tables + Slack, streamed, + $"the read of the large form allocated {streamed:N0} bytes, {streamed - tables:N0} more than its tables"); + Assert.IsGreaterThanOrEqualTo(tables, budget.Charged, + $"the read of the large form was charged {budget.Charged:N0} bytes, less than its tables"); + Assert.IsLessThanOrEqualTo(tables + Slack, budget.Charged, + $"the read of the large form was charged {budget.Charged:N0} bytes, {budget.Charged - tables:N0} more than its tables"); + } + /// /// The filter grows rather than filling up, and keys added before a growth /// survive it. From 291a56c663b3b28483cc314c6b2743cd68c443e8 Mon Sep 17 00:00:00 2001 From: Matthew Lorimor Date: Thu, 5 Nov 2026 19:50:13 -0600 Subject: [PATCH 2/2] Read each MementoFilter table straight into its segment, held once (#327) The reader read a table's words a word at a time (ReadUInt64) into an array of their own, then MementoSegment.Restore copied that array into the table it built, so a read held, and was charged for, every table twice, as InfiniFilter's did before #313. The segment is now built first and its words are read straight into its table with PayloadReader.ReadWords(Span), the overload #329 made of Infini's and the cuckoo filter's helpers, in pieces through a stack buffer of BlockPieceFor (16 KiB). Restore no longer takes the words. The all-zero check for a 64-bit slot's table, which the read loop kept as it went, now asks the table (ContainsAnyExcept(0)). The checks before the allocation stand where they were: CheckSegment, which holds the words to the shape and to one array (#326), and RequireHolds, which holds the payload to the words (#322), both run before the table is charged or allocated. SegmentCost is one table and the segment, not two tables, and the price uses it as the reader does, so price and charge still come to one sum (TestReadBudget holds them equal for every sample). The segment is charged as two small objects, 128 bytes: a MementoSegment and its KeepsakeCodec come to 120 bytes beyond the table, measured in a no-GC region at tables of 2^2, 2^8 and 2^12 slots, where one small object, 64, was charged beside the second table. Charged one table and one small object, every read would have been 56 bytes a table under what it allocates. TestASegmentIsChargedWhatRestoringItAllocates holds the charge to at least what Restore allocates and less than a small object past it, as #329's test does for Infini. Measured on the red test's filter (2-bit fingerprints, 48,000 keys, tables of 98,344 and 12,328 bytes), in a no-GC region: 5752769 this ordinary read 333,024 222,352 ordinary charge 332,312 221,768 large-form read 222,424 111,752 large-form charge 221,712 111,168 Past the payload in the ordinary envelope, and in all in the large form, what a read allocates and is charged is halved: 222,348 to 111,676 and 221,636 to 111,092, 222,424 to 111,752 and 221,712 to 111,168. The pieces test reads the sample's 73,744-byte table in 5 reads through the span overload (1,153 with the piece lowered), where it took 9,218. Documents moved by it, through the capacity checks: - README: the default budget now reads a range of 65,536 up to a table of 2^27 slots, 100,663,296 items (was 50,331,648); a range of 256 stays at 2^27. - CAPACITY.md: the note said a range of 65,536 reads back only 2^26, with no claim behind it. It now says a read holds each table twice, in the payload and in the table, and that the default budget reads back 2^27 slots at both ranges, each held by a claim: the charge lines' slopes are 16 bytes a word at both ranges (were 24), and the fitted lines reach 2^27 and not 2^28. - FORMAT.md: MementoFilter's row of read charges is merged with InfiniFilter's, which it now reads as. TestAMementoTableNoArrayHoldsIsRefusedByNameBeforeAnythingIsPricedOrAll- ocated (#326) held the widest table an array holds to a price of at least its words twice over, the old double charge, and failed: 2^31 slots of 63 bits priced 16,911,433,896 bytes. It now asks for the words once and less than twice over, so that a price that kept the second table fails it too. Mutation, scripts/mutate.sh, MUTATION_JOBS=2, at this commit's tree (a984cf6, before this table was added to its message), whole suite unless a test is named in brackets: read into a copy, copy it in (defect) killed (4) read a word at a time, no copy killed (1) segment: two tables (5752769) killed (5) segment: one small object killed (1) segment: three small objects killed (1) zero-table check inverted killed (5) reader: hold check dropped (#322) killed (3) price: a second table killed (6) reader: segment not charged killed (2) [holds once] read into a copy killed (1) [holds once] two tables killed (1) [pieces] read a word at a time killed (1) [segment] one small object killed (1) [segment] three small objects killed (1) "Read into a copy" is the defect under the new charge: the words read into an array of their own and copied into the segment's table. "A word at a time" is the old loop of ReadUInt64, into the table, so it allocates what the fix does; only the pieces test sees it. "Two tables" puts back the old SegmentCost, by which price and charge agree, so only what is measured sees it. The segment test's mutants show its two bounds bite, and one small object is the charge that would have undercharged by 56 bytes a table, which nothing else kills. The test could not be red before the fix, which changes Restore's signature and makes SegmentCost visible; these are its failures. "Hold check dropped" removes #322's RequireHolds from the reader, so a payload cut inside a table has the table allocated before its words are found missing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KK3MspqAcczxT45xCfLgyx --- CAPACITY.md | 7 ++-- CHANGELOG.md | 16 +++++++++ FORMAT.md | 3 +- ProbabilisticDataStructures/MementoFilter.cs | 33 +++++++++---------- ProbabilisticDataStructures/MementoSegment.cs | 17 +++------- README.md | 2 +- .../CapacityClaims.cs | 16 +++++++++ .../TestExpandingTableLimits.cs | 6 ++-- .../TestMementoFilter.cs | 31 +++++++++++++++++ 9 files changed, 94 insertions(+), 37 deletions(-) diff --git a/CAPACITY.md b/CAPACITY.md index c695538d..1b5fbdd7 100644 --- a/CAPACITY.md +++ b/CAPACITY.md @@ -401,9 +401,10 @@ of 2^27. The same table, with each slot also holding the memento that lets it answer a range query: `F + 1 + m + 3` bits for a range up to 2^m wide, so a wider range costs more per slot. The items counted are entries, one for each block of `range` consecutive keys, so -keys that fall in blocks already present add nothing. A range of 256 reads back a table of -2^27 slots, and a range of 65,536 only 2^26. A slot is held in a 64-bit word, so the -constructor refuses a fingerprint and a range that make it wider. +keys that fall in blocks already present add nothing. A read holds each table twice, in the +payload and in the table its words are read straight into, so the default budget reads back +a table of 2^27 slots at a range of 256 and at a range of 65,536. A slot is held in a 64-bit +word, so the constructor refuses a fingerprint and a range that make it wider. ## Retains k diff --git a/CHANGELOG.md b/CHANGELOG.md index 201a6333..34f0162d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -777,6 +777,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 widest table an array holds, 2^31 slots of 63 bits, is still taken and priced for its words. No writer wrote such a table: the constructor and every doubling stop short of it. +- **A `MementoFilter` read holds each table once, and is charged for it once.** Its reader + read a table's words a word at a time into an array of their own and then copied them + into the table the segment builds, so a read held, and was charged for, every table + twice, as an `InfiniFilter` read did before #313. The words are now read straight into + the table, in pieces through a 16 KiB buffer on the stack, as an `InfiniFilter`'s are, + and the price and the charges are one table each and the segment, so they still come to + the same sum. The segment and its codec are charged as two small objects, 128 bytes, + where they were charged as one beside the second table: measured in a no-GC region they + are 120 bytes, more than the one. Measured in a no-GC region, a filter of 2-bit + fingerprints whose chain holds 110,672 bytes of tables allocated 333,024 bytes to read + and now allocates 222,352, and in the large form, where the payload is not held, 222,424 + and now 111,752. So the default budget now reads a filter of 8-bit fingerprints at a + range of 65,536 up to a table of 2^27 slots, 100,663,296 items, where it stopped at 2^26; + at a range of 256 it still stops at 2^27. A payload claiming a table no array holds is + still refused for that before anything is priced or allocated. (#327) + - **An `InfiniFilter` read holds each table once, and is charged for it once.** Its reader read a table's words into an array of their own and then copied them into the table the segment builds, so a read held, and was charged for, every table twice: a table of 2^32 diff --git a/FORMAT.md b/FORMAT.md index d7e93ed6..b14b1bed 100644 --- a/FORMAT.md +++ b/FORMAT.md @@ -608,8 +608,7 @@ gigabytes, and loads only with a budget raised to match. What each read is charg | `CountMinSketch`, `CountSketch`, `PrivateCountMinSketch`, `SublimeCountMinSketch` | every row, before it is read; Sublime also its packed rows | | `TopK`, `HeavyKeeper` | a heap sized by the elements held, not by `k`; HeavyKeeper also its bucket arrays | | `ThetaSketch`, `TupleSketch`, `MinHashSignature`, `Grafite`, `SetSketch`, `InvertibleBloomLookupTable`, `DDSketch`, `HyperLogLogPlus` (sparse) | arrays sized by the values the payload carries | -| `InfiniFilter` | each table, its words read straight into the segment | -| `MementoFilter` | each table twice: as read, and as the segment built from it | +| `InfiniFilter`, `MementoFilter` | each table, its words read straight into the segment | | `DpswSketch` | its checkpoints, rebuilt from the substream size, and a list of segments per substream; each segment's sketch as above | ## Payloads diff --git a/ProbabilisticDataStructures/MementoFilter.cs b/ProbabilisticDataStructures/MementoFilter.cs index 7ba05c95..4797e55b 100644 --- a/ProbabilisticDataStructures/MementoFilter.cs +++ b/ProbabilisticDataStructures/MementoFilter.cs @@ -839,21 +839,19 @@ private static MementoFilter ReadPayload( // has the same width, and if a payload says otherwise the lowest limit holds. filter.QuotientBitsLimit = Math.Min(filter.QuotientBitsLimit, QuotientBitsLimitFor(slotBits)); - // Read into one array, then copied into the table the segment builds. + // The segment's table is allocated, and the words, which have no length before + // them, are read straight into it. Read a word at a time into an array of their + // own and copied into the table, every table was held, and charged, twice + // (#327). The shape's check, above, holds the words to what one array holds. budget.Charge(SegmentCost(words)); - var data = new ulong[words]; - var anyBitSet = false; - for (var w = 0; w < words; w++) - { - data[w] = reader.ReadUInt64(); - anyBitSet |= data[w] != 0; - } + var segment = MementoSegment.Restore(quotientBits, fingerprintBits, mementoBits, count); + reader.ReadWords(segment.Data); // The one table with a history. A slot of exactly 64 bits was written with // a mask of zero, so such a table counted every key it was given and stored // none: its count is positive and its words are all zero, which no table // that holds an entry can be, since an entry sets its slot's metadata. - if (slotBits == MementoSegment.MaxSlotBits && count > 0 && !anyBitSet) + if (slotBits == MementoSegment.MaxSlotBits && count > 0 && !segment.Data.AsSpan().ContainsAnyExcept(0UL)) { throw new InvalidDataException( $"Table {i} counts {count} entries and every word of it is zero. " + @@ -868,8 +866,6 @@ private static MementoFilter ReadPayload( // since decoding a run walks it, and then what a run says about its own boxes, // once for each occupied run, so that a box the codec cannot decode is refused // now rather than by the first query that reads it. - var segment = MementoSegment.Restore( - quotientBits, fingerprintBits, mementoBits, count, data); segment.CheckMetadata(i); segment.CheckRuns(i); filter.Segments.Add(segment); @@ -903,9 +899,8 @@ private static void CheckHash(HashId recorded, Func, ulong>? } /// - /// Prices what allocates beyond the envelope: each table's - /// words, read into one array and copied into the table. See - /// . + /// Prices what allocates beyond the envelope: each table, which its + /// words are read straight into. See . /// internal static void Price(ref PayloadReader reader, ushort version) { @@ -929,9 +924,13 @@ internal static void Price(ref PayloadReader reader, ushort version) } } - /// One table's words, read into one array and copied into the table. - private static long SegmentCost(uint words) => ReadBudget.Sum( - ReadBudget.Times(2, ReadBudget.ArrayOf(words, sizeof(ulong))), + /// + /// One table, its words read straight into it, and the segment that holds it with its + /// codec, which come to more than one small object allows for, so are charged as two. + /// + internal static long SegmentCost(uint words) => ReadBudget.Sum( + ReadBudget.ArrayOf(words, sizeof(ulong)), + ReadBudget.SmallObject, ReadBudget.SmallObject); private static void CheckChain(int mementoBits, uint segmentCount) diff --git a/ProbabilisticDataStructures/MementoSegment.cs b/ProbabilisticDataStructures/MementoSegment.cs index 59f7f832..34034ddc 100644 --- a/ProbabilisticDataStructures/MementoSegment.cs +++ b/ProbabilisticDataStructures/MementoSegment.cs @@ -129,19 +129,12 @@ private void Shape(int quotientBits, int fingerprintBits, int mementoBits) } /// - /// Restores a segment from stored fields without rebuilding its table. + /// A segment of a stored shape and count, its table allocated and every word of it + /// nought, for the reader to read the stored words straight into (#327), and then to + /// check its metadata and its runs. /// - internal static MementoSegment Restore( - int quotientBits, int fingerprintBits, int mementoBits, uint count, - ulong[] data) - { - var segment = new MementoSegment(quotientBits, fingerprintBits, mementoBits) - { - Count = count, - }; - data.CopyTo(segment.Data, 0); - return segment; - } + internal static MementoSegment Restore(int quotientBits, int fingerprintBits, int mementoBits, uint count) => + new(quotientBits, fingerprintBits, mementoBits) { Count = count }; /// /// Empties the table where it is, leaving it as a new one of its shape would be: no diff --git a/README.md b/README.md index d81e67b7..1857a459 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ building at the scale it describes, which is the measurement as of commit 426348 | `InfiniFilter` | items added, 8-bit fingerprints | 6,442,450,944 | 201,326,592 | 805,306,368 | | `InfiniFilter` | items added, 16-bit fingerprints | 3,221,225,472 | 100,663,296 | 402,653,184 | | `MementoFilter` | items added, one per block, range 256, 8-bit fingerprints | 1,610,612,736 | 100,663,296 | 402,653,184 | -| `MementoFilter` | items added, one per block, range 65,536, 8-bit fingerprints | 1,610,612,736 | 50,331,648 | 402,653,184 | +| `MementoFilter` | items added, one per block, range 65,536, 8-bit fingerprints | 1,610,612,736 | 100,663,296 | 402,653,184 | #### Retains k diff --git a/TestProbabilisticDataStructures/CapacityClaims.cs b/TestProbabilisticDataStructures/CapacityClaims.cs index 6521dbd8..bdf2e6db 100644 --- a/TestProbabilisticDataStructures/CapacityClaims.cs +++ b/TestProbabilisticDataStructures/CapacityClaims.cs @@ -1313,6 +1313,22 @@ static bool ReadsUpTo(int f, int q) => return SlotBits(16) == 20 && SlotBits(8) == 12 && ReadsUpTo(8, 28) && ReadsUpTo(16, 27); }), new Claim("`F + 1 + m + 3` bits", () => ChargeLine.Known.ContainsKey("MementoFilter 256")), + // In bytes a word of the table, as Infini's: eight for the payload's copy of the + // word, and eight for the table it is read into, which it once was twice (#327). + new Claim("A read holds each table twice, in the payload and in the table its words are read straight into", () => + Near(Slope("MementoFilter 256"), 16, 0.01) && Near(Slope("MementoFilter 65536"), 16, 0.01)), + new Claim("so the default budget reads back a table of 2^27 slots at a range of 256 and at a range of 65,536", () => + { + static bool ReadsUpTo(ulong range, int q) + { + var slotBits = 8 + 1 + (int)Math.Ceiling(Math.Log2(range)) + 3; + var line = ChargeLine.Known[$"MementoFilter {range}"]; + return line.Surely((ulong)MementoSegment.Words(1u << q, slotBits)) + && !line.Possibly((ulong)MementoSegment.Words(1u << (q + 1), slotBits)); + } + + return ReadsUpTo(256, 27) && ReadsUpTo(65536, 27); + }), // ---- Inverse and the retaining sketches ----------------------------------------- new Claim("reads back at 8 bytes a slot, and a full one at 56", () => diff --git a/TestProbabilisticDataStructures/TestExpandingTableLimits.cs b/TestProbabilisticDataStructures/TestExpandingTableLimits.cs index 83627861..9a09896e 100644 --- a/TestProbabilisticDataStructures/TestExpandingTableLimits.cs +++ b/TestProbabilisticDataStructures/TestExpandingTableLimits.cs @@ -522,9 +522,11 @@ public void TestAMementoTableNoArrayHoldsIsRefusedByNameBeforeAnythingIsPricedOr { problems.Add($"2^31 slots of 63 bits: the price did not walk the table an array holds: {fitting.Stopped.Message}"); } - else if (fitting.Priced < 2 * fits.Words * sizeof(ulong)) + else if (fitting.Priced < fits.Words * sizeof(ulong) || fitting.Priced >= 2 * fits.Words * sizeof(ulong)) { - problems.Add($"2^31 slots of 63 bits: the price came to {fitting.Priced:N0} bytes, less than the table's words twice over"); + // Once: the words are read straight into the table (#327), where they were + // read into an array of their own and copied, and priced twice over. + problems.Add($"2^31 slots of 63 bits: the price came to {fitting.Priced:N0} bytes, not the table's words once"); } var (fitsThrown, fitsAllocated, _) = Read(fits, ReadOptions.Default); diff --git a/TestProbabilisticDataStructures/TestMementoFilter.cs b/TestProbabilisticDataStructures/TestMementoFilter.cs index df8fafc5..448fab6d 100644 --- a/TestProbabilisticDataStructures/TestMementoFilter.cs +++ b/TestProbabilisticDataStructures/TestMementoFilter.cs @@ -312,6 +312,37 @@ public void TestAReadHoldsEachTableOnce() $"the read of the large form was charged {budget.Charged:N0} bytes, {budget.Charged - tables:N0} more than its tables"); } + /// + /// A read charges each segment at least what restoring it allocates, and less than a + /// small object past that: its table, and the segment with its codec, which come to more + /// than one small object allows for. The restores are counted in a no-GC region. + /// + /// + /// Probed on 5752769, a segment and its codec came to 120 bytes beyond the table, at + /// tables of 2^2, 2^8 and 2^12 slots, and were charged + /// , 64, beside two tables. Charged one table (#327), + /// they would be 56 bytes a segment under what the read allocates. The shapes are the + /// narrowest table, where the segment is the largest share of what is allocated, the + /// widest fingerprint, and two between. The first restore of each is not counted, so that + /// what the runtime does once is not taken for the segment. + /// + [TestMethod] + public void TestASegmentIsChargedWhatRestoringItAllocates() + { + foreach (var (q, f, m) in new[] { (2, 2, 1), (2, 32, 8), (8, 8, 3), (12, 16, 3) }) + { + var words = (uint)MementoSegment.Restore(q, f, m, 1).Data.Length; + var charged = MementoFilter.SegmentCost(words); + var allocated = NoGCRegion.Allocated(() => MementoSegment.Restore(q, f, m, 1)); + Assert.IsLessThanOrEqualTo(charged, allocated, + $"a segment of 2^{q} slots, {f}-bit fingerprints and {m}-bit mementos allocated {allocated:N0} bytes " + + $"to restore, and a read is charged {charged:N0} for it"); + Assert.IsLessThan(ReadBudget.SmallObject, charged - allocated, + $"a segment of 2^{q} slots, {f}-bit fingerprints and {m}-bit mementos is charged {charged:N0} bytes, " + + $"{charged - allocated:N0} past the {allocated:N0} restoring it allocated"); + } + } + /// /// The filter grows rather than filling up, and keys added before a growth /// survive it.