From d556155c9147242bf5293fe278f4c685d2a39273 Mon Sep 17 00:00:00 2001 From: Rossi-Luciano Date: Fri, 4 Sep 2026 12:24:45 -0300 Subject: [PATCH 1/2] fix: remove espaco extra ao redor de parenteses e ; nas citacoes do corpo extract_body_data extraia o texto de cada

via `para.xpath('.//text()...')` seguido de `' '.join(texts).split()`, que insere um espaco entre TODO par de fragmentos de texto, independente de ter espaco na fonte. Um paragrafo com citacao entre parenteses, ex.: "...pressure (Lang and Barling, 2012; Ripple et al., 2019) crop...", virava "...pressure ( Lang and Barling, 2012 ; Ripple et al., 2019 ) crop...": espaco extra logo apos "(", antes de ")" e antes de cada ";". get_text_from_node (xml_utils.py) ja fazia extracao correta preservando adjacencia real via .tail, inclusive com tratamento proprio pra xref, mas so era usado como fallback de excecao nessa funcao. Passa a ser o caminho principal, com um novo parametro skip_tags pra manter a exclusao de / do texto do paragrafo (preservando so o .tail que vem depois deles). _remove_double_spaces (chamada por get_text_from_node e por get_text_from_mixed_citation_node) passa a colapsar qualquer sequencia de espaco em branco (nao so espaco duplo literal) num unico espaco: o .tail de uma / pulada pode carregar a indentacao de quebra de linha do XML fonte (`\n `), que antes vazava crua pro paragrafo renderizado. Validado contra o corpus real de 26 artigos: 1765 paragrafos extraidos, so 15 ainda tem parenteses/espaco "suspeitos" e todos sao explicaveis (espaco literal na propria fonte XML, ou formula MathML achatada em texto, sem tratamento de formula ainda). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01X8r2LRJ3PGTT9vLaPtb373 --- packtools/sps/formats/pdf/pipeline/xml.py | 15 ++++---- packtools/sps/formats/pdf/utils/xml_utils.py | 34 ++++++++++++------ tests/sps/formats/pdf/pipeline/test_xml.py | 36 +++++++++++++++++++ tests/sps/formats/pdf/utils/test_xml_utils.py | 32 +++++++++++++++++ 4 files changed, 98 insertions(+), 19 deletions(-) diff --git a/packtools/sps/formats/pdf/pipeline/xml.py b/packtools/sps/formats/pdf/pipeline/xml.py index d0db5d2e7..a04738c7f 100644 --- a/packtools/sps/formats/pdf/pipeline/xml.py +++ b/packtools/sps/formats/pdf/pipeline/xml.py @@ -365,15 +365,14 @@ def extract_body_data(xml_tree, table_layout_overrides=None): if sec['title'] is not None: sec['title'] = ''.join(sec['title'].itertext()).strip() - # Collect textual paragraphs but exclude figure/table elements + # Collect textual paragraphs but exclude figure/table elements. Uses + # get_text_from_node (tail-preserving) rather than a bare + # `.xpath('.//text()...')` + `' '.join(...)`, which inserted an + # artificial space between every text-node fragment regardless of + # whether the source had one there (e.g. "(...)" came + # out as "( ... )", and "; " as "... ; ..."). for para in document_section.findall('p'): - try: - # Get text nodes that are not inside fig or table-wrap - texts = para.xpath('.//text()[not(ancestor::fig) and not(ancestor::table-wrap)]') - para_text = ' '.join(' '.join(texts).split()).strip() - except Exception: - # Fallback to generic text extraction - para_text = xml_utils.get_text_from_node(para) + para_text = xml_utils.get_text_from_node(para, skip_tags={'fig', 'table-wrap'}).strip() if para_text: sec['paragraphs'].append(para_text) diff --git a/packtools/sps/formats/pdf/utils/xml_utils.py b/packtools/sps/formats/pdf/utils/xml_utils.py index a250d945a..94438f970 100644 --- a/packtools/sps/formats/pdf/utils/xml_utils.py +++ b/packtools/sps/formats/pdf/utils/xml_utils.py @@ -1,20 +1,32 @@ -def get_text_from_node(node): +import re + + +def get_text_from_node(node, skip_tags=None): """ - Extracts text from an XML node, including its children. + Extracts text from an XML node, including its children, preserving the + adjacency of the source (no space is inserted between fragments unless + one was already there as literal text or a tail). Args: node (ElementTree): The XML node to extract text from. + skip_tags (set, optional): Child tag names to drop entirely from the + output; only their `.tail` (the text that follows them in the + source) is kept. Used to flatten a paragraph to readable text + while excluding embedded elements such as /. Returns: str: The text extracted from the given node. """ + skip_tags = skip_tags or set() texts_els = [] if node.text: texts_els.append(node.text) for child in node: - if child.tag == 'xref': + if child.tag in skip_tags: + pass + elif child.tag == 'xref': xref_text = child.text if child.text else '' for subchild in child: if subchild.tag in ('italic', 'bold'): @@ -26,9 +38,9 @@ def get_text_from_node(node): if child.text: texts_els.append(child.text) for subchild in child: - texts_els.append(get_text_from_node(subchild)) + texts_els.append(get_text_from_node(subchild, skip_tags=skip_tags)) else: - texts_els.append(get_text_from_node(child)) + texts_els.append(get_text_from_node(child, skip_tags=skip_tags)) if child.tail: texts_els.append(child.tail) @@ -113,14 +125,14 @@ def _add_period(text): def _remove_double_spaces(text): """ - Removes double spaces from the given text. + Collapses any run of whitespace (including tabs and newlines left over + from pretty-printed XML, e.g. the indentation tail of a skipped + /) into a single space. Args: - text (str): The text to remove double spaces from. + text (str): The text to normalize. Returns: - str: The text with double spaces removed. + str: The text with whitespace runs collapsed to single spaces. """ - while ' ' in text: - text = text.replace(' ', ' ') - return text + return re.sub(r'\s+', ' ', text) diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py index 774630aae..583c41bc8 100644 --- a/tests/sps/formats/pdf/pipeline/test_xml.py +++ b/tests/sps/formats/pdf/pipeline/test_xml.py @@ -417,6 +417,42 @@ def test_extract_body_data_with_table_references(self): result = xml_pipe.extract_body_data(xml) self.assertEqual(result, expected) + def test_paragraph_citations_have_no_stray_space_around_parentheses(self): + # Regression: a naive `.xpath('.//text()...')` + `' '.join(...)` + # inserted a space between every text-node fragment regardless of + # adjacency in the source, turning "(...; ... + # )" into "( ... ; ... )". + xml = etree.fromstring( + '

Introduction' + '

Pressure is increasing ' + '(Lang and Barling, 2012' + '; Ripple et al., 2019) ' + 'worldwide.

' + '
' + ) + result = xml_pipe.extract_body_data(xml) + self.assertEqual( + result[0]['paragraphs'], + ['Pressure is increasing (Lang and Barling, 2012; Ripple et al., 2019) worldwide.'], + ) + + def test_embedded_fig_tail_whitespace_is_collapsed_not_left_raw(self): + # A skipped 's tail can carry the source's pretty-printing + # indentation (a newline + spaces); it must collapse to one space + # rather than leak into the rendered paragraph. + xml = etree.fromstring( + '
Results' + '

See the figure below\n' + '\n ' + 'for details.

' + '
' + ) + result = xml_pipe.extract_body_data(xml) + self.assertEqual( + result[0]['paragraphs'], + ['See the figure below for details.'], + ) + class TestExtractCategory(unittest.TestCase): diff --git a/tests/sps/formats/pdf/utils/test_xml_utils.py b/tests/sps/formats/pdf/utils/test_xml_utils.py index e36c6ab9f..3cf633c57 100644 --- a/tests/sps/formats/pdf/utils/test_xml_utils.py +++ b/tests/sps/formats/pdf/utils/test_xml_utils.py @@ -55,6 +55,38 @@ def test_get_text_from_node_multiple_xref_italic(self): result = xml_utils.get_text_from_node(xmltree) self.assertEqual(expected, result) + def test_get_text_from_node_preserves_parenthesis_adjacency(self): + # No space should be inserted between "(" and the xref text, or + # between the xref text and ")", when none exists in the source. + xmltree = etree.fromstring( + '

seen (Author, 2020; ' + 'Other, 2021) here

' + ) + expected = 'seen (Author, 2020; Other, 2021) here' + result = xml_utils.get_text_from_node(xmltree) + self.assertEqual(expected, result) + + def test_get_text_from_node_skip_tags_drops_content_but_keeps_tail(self): + xmltree = etree.fromstring( + '

Before after

' + ) + result = xml_utils.get_text_from_node(xmltree, skip_tags={'fig'}) + self.assertEqual('Before after', result) + + def test_get_text_from_node_skip_tags_collapses_tail_whitespace(self): + xmltree = etree.fromstring( + '

Before\n\n after

' + ) + result = xml_utils.get_text_from_node(xmltree, skip_tags={'table-wrap'}) + self.assertEqual('Before after', result) + + def test_get_text_from_node_without_skip_tags_keeps_fig_content(self): + xmltree = etree.fromstring( + '

Before after

' + ) + result = xml_utils.get_text_from_node(xmltree) + self.assertEqual('Before Figure 1 after', result) + class TestGetTextFromMixedCitationNode(unittest.TestCase): From be8ec53281fae2454c2a7ef4247d9cd59144c6d4 Mon Sep 17 00:00:00 2001 From: Rossi-Luciano Date: Tue, 8 Sep 2026 09:33:41 -0300 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20preserva=20conte=C3=BAdo=20de=20/=20e=20tail=20aninhado=20em=20get=5Ftext=5Ffrom=5Fnode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Atende à revisão de @pitangainnovare no PR #1327: os branches especiais para e / em get_text_from_node só reconheciam subelementos italic/bold, descartando qualquer outra tag (ex.: , estilo Vancouver) e perdendo o tail de marcação aninhada (ex.: texto depois de um dentro de um ). Confirmado contra o corpus de 26 artigos: 659 ocorrências de dentro de em 17 artigos tinham a citação apagada (ex. a11.xml: "fonoterápico()." em vez de "fonoterápico(1)."). Substituídos os branches especiais por recursão genérica uniforme, que já trata texto+filhos+tail corretamente para qualquer tag. Adicionada _normalize_punctuation_spacing para remover espaços presos a parênteses/colchetes e antes de ";"/"," quando existem literalmente no XML fonte ao redor de um (segundo ponto da revisão). --- packtools/sps/formats/pdf/utils/xml_utils.py | 36 ++++++++++++------- tests/sps/formats/pdf/utils/test_xml_utils.py | 26 ++++++++++++++ 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/packtools/sps/formats/pdf/utils/xml_utils.py b/packtools/sps/formats/pdf/utils/xml_utils.py index 94438f970..3bf3b90a1 100644 --- a/packtools/sps/formats/pdf/utils/xml_utils.py +++ b/packtools/sps/formats/pdf/utils/xml_utils.py @@ -26,19 +26,6 @@ def get_text_from_node(node, skip_tags=None): for child in node: if child.tag in skip_tags: pass - elif child.tag == 'xref': - xref_text = child.text if child.text else '' - for subchild in child: - if subchild.tag in ('italic', 'bold'): - xref_text += (subchild.text if subchild.text else '') - if subchild.tail: - xref_text += (subchild.tail if subchild.tail else '') - texts_els.append(xref_text) - elif child.tag in ('italic', 'bold'): - if child.text: - texts_els.append(child.text) - for subchild in child: - texts_els.append(get_text_from_node(subchild, skip_tags=skip_tags)) else: texts_els.append(get_text_from_node(child, skip_tags=skip_tags)) @@ -47,6 +34,7 @@ def get_text_from_node(node, skip_tags=None): text = ''.join(texts_els) text = _remove_double_spaces(text) + text = _normalize_punctuation_spacing(text) return text def get_node_level(element, root): @@ -136,3 +124,25 @@ def _remove_double_spaces(text): str: The text with whitespace runs collapsed to single spaces. """ return re.sub(r'\s+', ' ', text) + +def _normalize_punctuation_spacing(text): + """ + Removes whitespace that ends up glued to the inside of parentheses and + brackets, or before a comma/semicolon, when the source XML has a space + directly before/after an inline element such as (e.g. "( + Fig. 1 )") — a common defect that survives adjacency-preserving + extraction because the space is literal text, not an artifact of it. + + Args: + text (str): The text to normalize. + + Returns: + str: The text with punctuation spacing normalized. + """ + text = re.sub(r'\(\s+', '(', text) + text = re.sub(r'\s+\)', ')', text) + text = re.sub(r'\[\s+', '[', text) + text = re.sub(r'\s+\]', ']', text) + text = re.sub(r'\s+;', ';', text) + text = re.sub(r'\s+,', ',', text) + return text diff --git a/tests/sps/formats/pdf/utils/test_xml_utils.py b/tests/sps/formats/pdf/utils/test_xml_utils.py index 3cf633c57..9e606f179 100644 --- a/tests/sps/formats/pdf/utils/test_xml_utils.py +++ b/tests/sps/formats/pdf/utils/test_xml_utils.py @@ -87,6 +87,32 @@ def test_get_text_from_node_without_skip_tags_keeps_fig_content(self): result = xml_utils.get_text_from_node(xmltree) self.assertEqual('Before Figure 1 after', result) + def test_get_text_from_node_with_sup_inside_xref(self): + xmltree = etree.fromstring( + '

Author 1,2 stated

' + ) + self.assertEqual('Author 1,2 stated', xml_utils.get_text_from_node(xmltree)) + + def test_get_text_from_node_nested_formatting_with_tail(self): + xmltree = etree.fromstring( + '

Start bold and italic still bold end

' + ) + self.assertEqual( + 'Start bold and italic still bold end', + xml_utils.get_text_from_node(xmltree), + ) + + def test_get_text_from_node_normalizes_spaces_around_parentheses_and_punctuation(self): + xmltree = etree.fromstring( + '

Studies ( Author, 2020 ; ' + 'Other, 2021 ) and [ 1 ] ' + 'with comma ( Foo, 2019 , more).

' + ) + self.assertEqual( + 'Studies (Author, 2020; Other, 2021) and [1] with comma (Foo, 2019, more).', + xml_utils.get_text_from_node(xmltree), + ) + class TestGetTextFromMixedCitationNode(unittest.TestCase):