From 0b4d3eaf9d5f3282f99b4eb32f6740660895b849 Mon Sep 17 00:00:00 2001 From: Rossi-Luciano Date: Fri, 4 Sep 2026 15:03:59 -0300 Subject: [PATCH 1/2] fix: extrai paragrafos de resumo estruturado (sec por subsecao) Corrige a issue #1332: extract_abstract_data e extract_trans_abstract_data usavam node.findall('p'), que so acha

filho DIRETO de / . Um resumo estruturado (subsecoes tipo Introduction/ Methods/Results, cada uma sua propria ...

...

) tem os

um nivel mais fundo, entao a extracao retornava content vazio - o PDF mostrava so o cabecalho "ABSTRACT"/"RESUMO" e ia direto pra "Keywords", sem nenhum texto de resumo. Adiciona _extract_abstract_paragraphs(node), helper recursivo que percorre

e em qualquer profundidade, incluindo o de cada <sec> como rotulo (ja vem com o ":" da propria fonte, ex.: "Methods:"), preservando a estrutura do resumo em vez de so concatenar tudo. Reaproveitado nas duas funcoes, que tinham o mesmo bug. Confirmado contra o corpus real de 26 artigos: 6 (23%) tinham resumo vazio por esse motivo (a10, a11, a14, a17, a20, a28) - a10 tinha o RESUMO em portugues vazio tambem. Todos os 6 corrigidos; os outros 20 (resumo simples, sem <sec>) continuam identicos. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01X8r2LRJ3PGTT9vLaPtb373 --- packtools/sps/formats/pdf/pipeline/xml.py | 65 +++++++++++++++++----- tests/sps/formats/pdf/pipeline/test_xml.py | 63 +++++++++++++++++++++ 2 files changed, 113 insertions(+), 15 deletions(-) diff --git a/packtools/sps/formats/pdf/pipeline/xml.py b/packtools/sps/formats/pdf/pipeline/xml.py index d0db5d2e7..0a870e132 100644 --- a/packtools/sps/formats/pdf/pipeline/xml.py +++ b/packtools/sps/formats/pdf/pipeline/xml.py @@ -164,14 +164,20 @@ def extract_contrib_data(xml_tree): def extract_abstract_data(xml_tree): """ Extracts the title and content of the abstract from the given XML tree. - + + Handles both a plain abstract (<p> direct children of <abstract>) and a + structured one (subsections wrapped in <sec>, e.g. Introduction/Methods/ + Results, each with its own <title> and <p>) - see _extract_abstract_paragraphs. + Args: xml_tree (ElementTree): The XML tree to extract the abstract from. - + Returns: dict: A dictionary containing the following keys: - 'title': The text content of the abstract title element, or an empty string if not found. - - 'content': The text content of the abstract paragraphs, concatenated into a single string. + - 'content': The text content of the abstract paragraphs (and, for a + structured abstract, each subsection's title), concatenated into a + single string. """ data = {'title': '', 'content': ''} @@ -182,22 +188,22 @@ def extract_abstract_data(xml_tree): if node_title is not None: data['title'] = ''.join(node_title.itertext()).strip() - abstract = [] - for p in node_abstract.findall('p'): - if p is not None: - abstract.append(''.join(p.itertext()).strip()) - data['content'] = ' '.join(abstract) + data['content'] = ' '.join(_extract_abstract_paragraphs(node_abstract)) return data def extract_trans_abstract_data(xml_tree, namespaces={'xml': 'http://www.w3.org/XML/1998/namespace'}): """ Extracts the title and content of translated abstracts from the given XML tree. - + + Handles both a plain and a structured trans-abstract (subsections wrapped + in <sec>) the same way extract_abstract_data does - see + _extract_abstract_paragraphs. + Args: xml_tree (ElementTree): The XML tree to extract the translated abstracts from. namespaces (dict, optional): A dictionary of XML namespaces to use in the XPath expressions. - + Returns: list: A list of dictionaries, where each dictionary contains the following keys: - 'lang': The language of the translated abstract. @@ -217,11 +223,7 @@ def extract_trans_abstract_data(xml_tree, namespaces={'xml': 'http://www.w3.org/ item['lang'] = node.attrib.get(lang_attrib_name) - abstract = [] - for p in node.findall('p'): - if p is not None: - abstract.append(''.join(p.itertext()).strip()) - item['content'] = ' '.join(abstract) + item['content'] = ' '.join(_extract_abstract_paragraphs(node)) data.append(item) @@ -797,6 +799,39 @@ def get_table_column_info(headers, rows): # Private helpers # ----------------- +def _extract_abstract_paragraphs(node): + """ + Collects an abstract's readable text as a list of strings, one per + <p> found at any depth. A structured abstract wraps each subsection + in its own <sec> (e.g. <sec><title>Methods:

...

), + so a plain `node.findall('p')` (direct children only) misses every + paragraph and returns an empty abstract. Recursing into finds + them, and including each 's own (already carries the + subsection label and its own trailing colon, e.g. "Methods:") + preserves the abstract's structure in the flattened output instead + of silently merging distinct subsections together. + + Args: + node (ElementTree): The <abstract> or <trans-abstract> element + (or a <sec> within one, for the recursive call). + + Returns: + list: Text fragments in document order - <sec> titles and <p> content. + """ + parts = [] + for child in node: + if child.tag == 'p': + parts.append(''.join(child.itertext()).strip()) + elif child.tag == 'sec': + sec_title = child.find('title') + if sec_title is not None: + title_text = ''.join(sec_title.itertext()).strip() + if title_text: + parts.append(title_text) + parts.extend(_extract_abstract_paragraphs(child)) + return parts + + def _extract_table_rows_with_merged_cells(table_section, cell_tag): """ Extracts table rows handling merged cells (colspan/rowspan). diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py index 774630aae..e25acd653 100644 --- a/tests/sps/formats/pdf/pipeline/test_xml.py +++ b/tests/sps/formats/pdf/pipeline/test_xml.py @@ -83,6 +83,45 @@ def test_extract_abstract_data_with_nested_elements(self): result = xml_pipe.extract_abstract_data(xml) self.assertEqual(result, expected) + def test_extract_abstract_data_structured_with_sections(self): + # Regression for issue #1332: a structured abstract wraps + # each subsection in its own <sec>, so a plain findall('p') (direct + # children only) found nothing and returned an empty content. + xml = etree.fromstring( + '<article><abstract>' + '<title>Abstract' + 'Introduction:

Some introduction text.

' + 'Methods:

Some methods text.

' + '' + ) + expected = { + 'title': 'Abstract', + 'content': 'Introduction: Some introduction text. Methods: Some methods text.', + } + result = xml_pipe.extract_abstract_data(xml) + self.assertEqual(result, expected) + + def test_extract_abstract_data_structured_section_without_title(self): + xml = etree.fromstring( + '
' + '

Untitled section text.

' + '
' + ) + expected = {'title': '', 'content': 'Untitled section text.'} + result = xml_pipe.extract_abstract_data(xml) + self.assertEqual(result, expected) + + def test_extract_abstract_data_mixed_direct_and_sectioned_paragraphs(self): + xml = etree.fromstring( + '
' + '

Lead paragraph.

' + 'Conclusion:

Final remarks.

' + '
' + ) + expected = {'title': '', 'content': 'Lead paragraph. Conclusion: Final remarks.'} + result = xml_pipe.extract_abstract_data(xml) + self.assertEqual(result, expected) + class TestExtractAcknowledgmentData(unittest.TestCase): @@ -1573,6 +1612,30 @@ def test_extract_trans_abstract_data_title_with_inline_markup(self): result = xml_pipe.extract_trans_abstract_data(xml) self.assertEqual(result[0]['title'], 'Resumo*') + def test_extract_trans_abstract_data_structured_with_sections(self): + # Regression for issue #1332: same bug as extract_abstract_data, + # a structured trans-abstract's

nested in was invisible + # to a plain findall('p'), so the translated abstract's content + # came out empty (e.g. a10.xml's RESUMO in the real test corpus). + xml = etree.fromstring( + '

' + '' + 'Resumo' + 'Contexto:

Texto de contexto.

' + 'Métodos:

Texto de métodos.

' + '' + '
' + ) + expected = [ + { + 'lang': 'pt', + 'title': 'Resumo', + 'content': 'Contexto: Texto de contexto. Métodos: Texto de métodos.', + } + ] + result = xml_pipe.extract_trans_abstract_data(xml) + self.assertEqual(result, expected) + class TestExtractFigureData(unittest.TestCase): """Tests for extract_figure_data's graphic href resolution, including From 9c6cb95672606966d2cb75e5dfbe02f4d39671f1 Mon Sep 17 00:00:00 2001 From: Rossi-Luciano Date: Wed, 9 Sep 2026 09:01:01 -0300 Subject: [PATCH 2/2] fix: exclui secoes do abstract/trans-abstract do corpo e normaliza titulo sem pontuacao extract_body_data() usava './/sec' sobre a arvore inteira, incluindo as internas do resumo estruturado (abstract/trans-abstract), o que duplicava o mesmo conteudo no resumo e no corpo (ex: a10.xml). _extract_abstract_paragraphs() assumia que o de cada subsecao ja trazia ':' (ex: "Methods:"), mas parte do corpus nao tem essa pontuacao (ex: a11/a17/a20.xml com "Objetivo"), gerando saida como "Objetivo descrever..." em vez de "Objetivo: descrever...". Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MwX5yYrVbsuZ9dB1VrMUCZ --- packtools/sps/formats/pdf/pipeline/xml.py | 22 ++++++++--- tests/sps/formats/pdf/pipeline/test_xml.py | 43 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packtools/sps/formats/pdf/pipeline/xml.py b/packtools/sps/formats/pdf/pipeline/xml.py index 0a870e132..a22404c7b 100644 --- a/packtools/sps/formats/pdf/pipeline/xml.py +++ b/packtools/sps/formats/pdf/pipeline/xml.py @@ -342,6 +342,11 @@ def extract_body_data(xml_tree, table_layout_overrides=None): """ Extracts the body data from an XML tree, including section titles, paragraphs, and tables. + Excludes any <sec> nested inside <abstract> or <trans-abstract> - those + are structured-abstract subsections handled by extract_abstract_data / + extract_trans_abstract_data, and would otherwise be picked up twice by + a plain './/sec' search. + Args: xml_tree (ElementTree): The XML tree to extract the body data from. table_layout_overrides (dict, optional): Maps a table-wrap @id to a forced @@ -359,7 +364,10 @@ def extract_body_data(xml_tree, table_layout_overrides=None): data = [] seen_fig_keys = set() - for document_section in xml_tree.findall('.//sec'): + body_sections = xml_tree.xpath( + './/sec[not(ancestor::abstract) and not(ancestor::trans-abstract)]' + ) + for document_section in body_sections: sec = {'paragraphs': [], 'tables': [], 'figures': []} sec['level'] = xml_utils.get_node_level(document_section, xml_tree) sec['title'] = document_section.find('title') @@ -806,10 +814,12 @@ def _extract_abstract_paragraphs(node): in its own <sec> (e.g. <sec><title>Methods:

...

), so a plain `node.findall('p')` (direct children only) misses every paragraph and returns an empty abstract. Recursing into finds - them, and including each 's own (already carries the - subsection label and its own trailing colon, e.g. "Methods:") - preserves the abstract's structure in the flattened output instead - of silently merging distinct subsections together. + them, and including each <sec>'s own <title> in the flattened output + preserves the abstract's structure instead of silently merging + distinct subsections together. Some XMLs already carry a trailing + colon in the title (e.g. "Methods:"), others don't (e.g. "Methods"); + a colon is appended only when the title lacks its own closing + punctuation, so it never gets duplicated. Args: node (ElementTree): The <abstract> or <trans-abstract> element @@ -827,6 +837,8 @@ def _extract_abstract_paragraphs(node): if sec_title is not None: title_text = ''.join(sec_title.itertext()).strip() if title_text: + if title_text[-1] not in ':.!?;': + title_text = f'{title_text}:' parts.append(title_text) parts.extend(_extract_abstract_paragraphs(child)) return parts diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py index e25acd653..ac6b686a8 100644 --- a/tests/sps/formats/pdf/pipeline/test_xml.py +++ b/tests/sps/formats/pdf/pipeline/test_xml.py @@ -101,6 +101,19 @@ def test_extract_abstract_data_structured_with_sections(self): result = xml_pipe.extract_abstract_data(xml) self.assertEqual(result, expected) + def test_extract_abstract_data_structured_section_title_without_punctuation(self): + # Some XMLs don't carry a trailing colon in the <sec><title>, unlike + # the "Methods:" style above - a colon must be added so the title + # doesn't run into the paragraph text (e.g. "Objetivodescrever..."). + xml = etree.fromstring( + '<article><abstract>' + '<sec><title>Objetivo

Descrever o metodo.

' + '' + ) + expected = {'title': '', 'content': 'Objetivo: Descrever o metodo.'} + result = xml_pipe.extract_abstract_data(xml) + self.assertEqual(result, expected) + def test_extract_abstract_data_structured_section_without_title(self): xml = etree.fromstring( '
' @@ -342,6 +355,36 @@ def test_extract_body_data_basic(self): result = xml_pipe.extract_body_data(xml) self.assertEqual(result, expected) + def test_extract_body_data_excludes_abstract_and_trans_abstract_sections(self): + # Regression: a structured abstract/trans-abstract wraps each + # subsection in its own (see extract_abstract_data), which a + # plain './/sec' search would also pick up as a body section, + # duplicating the same content in both the abstract and the body. + xml = etree.fromstring( + '
' + '' + 'Background:

Abstract text.

' + '' + '' + 'Contexto:

Texto do resumo.

' + '' + '' + 'Introduction

Body text.

' + '' + '
' + ) + expected = [ + { + 'level': 2, + 'title': 'Introduction', + 'paragraphs': ['Body text.'], + 'tables': [], + 'figures': [], + } + ] + result = xml_pipe.extract_body_data(xml) + self.assertEqual(result, expected) + def test_extract_body_data_with_tables(self): xml = etree.fromstring( '
'