From 0fee9538a46f1b81bb801b4b2bc5478fd1ad5b1c Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Wed, 2 Sep 2026 23:21:51 +0200 Subject: [PATCH 1/2] fix: an oversized first page no longer flushes an empty chunk current_subset is still empty on the first page, so a page over average_tokens_per_part flushed an empty chunk that both callers then passed to the model as a slice of the document. --- pageindex/page_index_classic.py | 5 ++++- tests/test_page_index.py | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/pageindex/page_index_classic.py b/pageindex/page_index_classic.py index 446e8893b..600c99b2c 100644 --- a/pageindex/page_index_classic.py +++ b/pageindex/page_index_classic.py @@ -528,7 +528,10 @@ def page_list_to_group_text(page_contents, token_lengths, max_tokens=20000, over average_tokens_per_part = math.ceil(((num_tokens / expected_parts_num) + max_tokens) / 2) for i, (page_content, page_tokens) in enumerate(zip(page_contents, token_lengths)): - if current_token_count + page_tokens > average_tokens_per_part: + # current_subset is still empty on the first page, so without this guard + # a first page over the average flushes an empty chunk -- which callers + # then hand to the model as a slice of the document. + if current_subset and current_token_count + page_tokens > average_tokens_per_part: subsets.append(''.join(current_subset)) # Start new subset from overlap if specified diff --git a/tests/test_page_index.py b/tests/test_page_index.py index da69a9a79..88036ef8d 100644 --- a/tests/test_page_index.py +++ b/tests/test_page_index.py @@ -3,6 +3,7 @@ from pageindex.page_index_classic import ( _secure_doc_text, + page_list_to_group_text, process_no_toc, process_toc_no_page_numbers, ) @@ -65,5 +66,43 @@ def test_secure_doc_text_neutralizes_document_delimiters(self): self.assertIn("", wrapped) +class PageListToGroupTextTest(unittest.TestCase): + PAGES = ["PAGE-A", "PAGE-B", "PAGE-C"] + + def test_oversized_first_page_does_not_emit_an_empty_chunk(self): + # A first page over average_tokens_per_part used to flush the still-empty + # accumulator, and callers hand chunk 0 straight to the model. + chunks = page_list_to_group_text( + self.PAGES, [60000, 500, 500], max_tokens=20000 + ) + + self.assertNotIn("", chunks) + + def test_every_page_survives_an_oversized_first_page(self): + chunks = page_list_to_group_text( + self.PAGES, [60000, 500, 500], max_tokens=20000 + ) + + joined = "".join(chunks) + for page in self.PAGES: + self.assertIn(page, joined) + + def test_oversized_middle_page_is_unaffected(self): + # Only the first page can hit the empty-accumulator case; later splits + # re-seed from the overlap page. + chunks = page_list_to_group_text( + self.PAGES, [100, 60000, 100], max_tokens=20000 + ) + + self.assertNotIn("", chunks) + + def test_document_under_the_limit_stays_one_chunk(self): + chunks = page_list_to_group_text( + self.PAGES, [100, 100, 100], max_tokens=20000 + ) + + self.assertEqual(chunks, ["PAGE-APAGE-BPAGE-C"]) + + if __name__ == "__main__": unittest.main() From 780b2edf3a5d507b3c6bd2c72588c99c4dfa96ba Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Fri, 4 Sep 2026 17:48:02 +0200 Subject: [PATCH 2/2] style: trim the chunking comments to the surrounding density --- pageindex/page_index_classic.py | 4 +--- tests/test_page_index.py | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pageindex/page_index_classic.py b/pageindex/page_index_classic.py index 600c99b2c..29d8117c4 100644 --- a/pageindex/page_index_classic.py +++ b/pageindex/page_index_classic.py @@ -528,9 +528,7 @@ def page_list_to_group_text(page_contents, token_lengths, max_tokens=20000, over average_tokens_per_part = math.ceil(((num_tokens / expected_parts_num) + max_tokens) / 2) for i, (page_content, page_tokens) in enumerate(zip(page_contents, token_lengths)): - # current_subset is still empty on the first page, so without this guard - # a first page over the average flushes an empty chunk -- which callers - # then hand to the model as a slice of the document. + # without the guard, an oversized first page flushes an empty chunk if current_subset and current_token_count + page_tokens > average_tokens_per_part: subsets.append(''.join(current_subset)) diff --git a/tests/test_page_index.py b/tests/test_page_index.py index 88036ef8d..4422d5c1c 100644 --- a/tests/test_page_index.py +++ b/tests/test_page_index.py @@ -70,8 +70,6 @@ class PageListToGroupTextTest(unittest.TestCase): PAGES = ["PAGE-A", "PAGE-B", "PAGE-C"] def test_oversized_first_page_does_not_emit_an_empty_chunk(self): - # A first page over average_tokens_per_part used to flush the still-empty - # accumulator, and callers hand chunk 0 straight to the model. chunks = page_list_to_group_text( self.PAGES, [60000, 500, 500], max_tokens=20000 ) @@ -88,8 +86,6 @@ def test_every_page_survives_an_oversized_first_page(self): self.assertIn(page, joined) def test_oversized_middle_page_is_unaffected(self): - # Only the first page can hit the empty-accumulator case; later splits - # re-seed from the overlap page. chunks = page_list_to_group_text( self.PAGES, [100, 60000, 100], max_tokens=20000 )