diff --git a/packtools/sps/formats/pdf/pipeline/xml.py b/packtools/sps/formats/pdf/pipeline/xml.py index d0db5d2e7..a04738c7f 100644 --- a/packtools/sps/formats/pdf/pipeline/xml.py +++ b/packtools/sps/formats/pdf/pipeline/xml.py @@ -365,15 +365,14 @@ def extract_body_data(xml_tree, table_layout_overrides=None): if sec['title'] is not None: sec['title'] = ''.join(sec['title'].itertext()).strip() - # Collect textual paragraphs but exclude figure/table elements + # Collect textual paragraphs but exclude figure/table elements. Uses + # get_text_from_node (tail-preserving) rather than a bare + # `.xpath('.//text()...')` + `' '.join(...)`, which inserted an + # artificial space between every text-node fragment regardless of + # whether the source had one there (e.g. "(...)" came + # out as "( ... )", and "; " as "... ; ..."). for para in document_section.findall('p'): - try: - # Get text nodes that are not inside fig or table-wrap - texts = para.xpath('.//text()[not(ancestor::fig) and not(ancestor::table-wrap)]') - para_text = ' '.join(' '.join(texts).split()).strip() - except Exception: - # Fallback to generic text extraction - para_text = xml_utils.get_text_from_node(para) + para_text = xml_utils.get_text_from_node(para, skip_tags={'fig', 'table-wrap'}).strip() if para_text: sec['paragraphs'].append(para_text) diff --git a/packtools/sps/formats/pdf/utils/xml_utils.py b/packtools/sps/formats/pdf/utils/xml_utils.py index a250d945a..3bf3b90a1 100644 --- a/packtools/sps/formats/pdf/utils/xml_utils.py +++ b/packtools/sps/formats/pdf/utils/xml_utils.py @@ -1,40 +1,40 @@ -def get_text_from_node(node): +import re + + +def get_text_from_node(node, skip_tags=None): """ - Extracts text from an XML node, including its children. + Extracts text from an XML node, including its children, preserving the + adjacency of the source (no space is inserted between fragments unless + one was already there as literal text or a tail). Args: node (ElementTree): The XML node to extract text from. + skip_tags (set, optional): Child tag names to drop entirely from the + output; only their `.tail` (the text that follows them in the + source) is kept. Used to flatten a paragraph to readable text + while excluding embedded elements such as /. Returns: str: The text extracted from the given node. """ + skip_tags = skip_tags or set() texts_els = [] if node.text: texts_els.append(node.text) for child in node: - if child.tag == 'xref': - xref_text = child.text if child.text else '' - for subchild in child: - if subchild.tag in ('italic', 'bold'): - xref_text += (subchild.text if subchild.text else '') - if subchild.tail: - xref_text += (subchild.tail if subchild.tail else '') - texts_els.append(xref_text) - elif child.tag in ('italic', 'bold'): - if child.text: - texts_els.append(child.text) - for subchild in child: - texts_els.append(get_text_from_node(subchild)) + if child.tag in skip_tags: + pass else: - texts_els.append(get_text_from_node(child)) + texts_els.append(get_text_from_node(child, skip_tags=skip_tags)) if child.tail: texts_els.append(child.tail) text = ''.join(texts_els) text = _remove_double_spaces(text) + text = _normalize_punctuation_spacing(text) return text def get_node_level(element, root): @@ -113,14 +113,36 @@ def _add_period(text): def _remove_double_spaces(text): """ - Removes double spaces from the given text. + Collapses any run of whitespace (including tabs and newlines left over + from pretty-printed XML, e.g. the indentation tail of a skipped + /) into a single space. + + Args: + text (str): The text to normalize. + + Returns: + str: The text with whitespace runs collapsed to single spaces. + """ + return re.sub(r'\s+', ' ', text) + +def _normalize_punctuation_spacing(text): + """ + Removes whitespace that ends up glued to the inside of parentheses and + brackets, or before a comma/semicolon, when the source XML has a space + directly before/after an inline element such as (e.g. "( + Fig. 1 )") — a common defect that survives adjacency-preserving + extraction because the space is literal text, not an artifact of it. Args: - text (str): The text to remove double spaces from. + text (str): The text to normalize. Returns: - str: The text with double spaces removed. + str: The text with punctuation spacing normalized. """ - while ' ' in text: - text = text.replace(' ', ' ') + text = re.sub(r'\(\s+', '(', text) + text = re.sub(r'\s+\)', ')', text) + text = re.sub(r'\[\s+', '[', text) + text = re.sub(r'\s+\]', ']', text) + text = re.sub(r'\s+;', ';', text) + text = re.sub(r'\s+,', ',', text) return text diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py index 774630aae..583c41bc8 100644 --- a/tests/sps/formats/pdf/pipeline/test_xml.py +++ b/tests/sps/formats/pdf/pipeline/test_xml.py @@ -417,6 +417,42 @@ def test_extract_body_data_with_table_references(self): result = xml_pipe.extract_body_data(xml) self.assertEqual(result, expected) + def test_paragraph_citations_have_no_stray_space_around_parentheses(self): + # Regression: a naive `.xpath('.//text()...')` + `' '.join(...)` + # inserted a space between every text-node fragment regardless of + # adjacency in the source, turning "(...; ... + # )" into "( ... ; ... )". + xml = etree.fromstring( + '
Introduction' + '

Pressure is increasing ' + '(Lang and Barling, 2012' + '; Ripple et al., 2019) ' + 'worldwide.

' + '
' + ) + result = xml_pipe.extract_body_data(xml) + self.assertEqual( + result[0]['paragraphs'], + ['Pressure is increasing (Lang and Barling, 2012; Ripple et al., 2019) worldwide.'], + ) + + def test_embedded_fig_tail_whitespace_is_collapsed_not_left_raw(self): + # A skipped 's tail can carry the source's pretty-printing + # indentation (a newline + spaces); it must collapse to one space + # rather than leak into the rendered paragraph. + xml = etree.fromstring( + '
Results' + '

See the figure below\n' + '\n ' + 'for details.

' + '
' + ) + result = xml_pipe.extract_body_data(xml) + self.assertEqual( + result[0]['paragraphs'], + ['See the figure below for details.'], + ) + class TestExtractCategory(unittest.TestCase): diff --git a/tests/sps/formats/pdf/utils/test_xml_utils.py b/tests/sps/formats/pdf/utils/test_xml_utils.py index e36c6ab9f..9e606f179 100644 --- a/tests/sps/formats/pdf/utils/test_xml_utils.py +++ b/tests/sps/formats/pdf/utils/test_xml_utils.py @@ -55,6 +55,64 @@ def test_get_text_from_node_multiple_xref_italic(self): result = xml_utils.get_text_from_node(xmltree) self.assertEqual(expected, result) + def test_get_text_from_node_preserves_parenthesis_adjacency(self): + # No space should be inserted between "(" and the xref text, or + # between the xref text and ")", when none exists in the source. + xmltree = etree.fromstring( + '

seen (Author, 2020; ' + 'Other, 2021) here

' + ) + expected = 'seen (Author, 2020; Other, 2021) here' + result = xml_utils.get_text_from_node(xmltree) + self.assertEqual(expected, result) + + def test_get_text_from_node_skip_tags_drops_content_but_keeps_tail(self): + xmltree = etree.fromstring( + '

Before after

' + ) + result = xml_utils.get_text_from_node(xmltree, skip_tags={'fig'}) + self.assertEqual('Before after', result) + + def test_get_text_from_node_skip_tags_collapses_tail_whitespace(self): + xmltree = etree.fromstring( + '

Before\n\n after

' + ) + result = xml_utils.get_text_from_node(xmltree, skip_tags={'table-wrap'}) + self.assertEqual('Before after', result) + + def test_get_text_from_node_without_skip_tags_keeps_fig_content(self): + xmltree = etree.fromstring( + '

Before after

' + ) + result = xml_utils.get_text_from_node(xmltree) + self.assertEqual('Before Figure 1 after', result) + + def test_get_text_from_node_with_sup_inside_xref(self): + xmltree = etree.fromstring( + '

Author 1,2 stated

' + ) + self.assertEqual('Author 1,2 stated', xml_utils.get_text_from_node(xmltree)) + + def test_get_text_from_node_nested_formatting_with_tail(self): + xmltree = etree.fromstring( + '

Start bold and italic still bold end

' + ) + self.assertEqual( + 'Start bold and italic still bold end', + xml_utils.get_text_from_node(xmltree), + ) + + def test_get_text_from_node_normalizes_spaces_around_parentheses_and_punctuation(self): + xmltree = etree.fromstring( + '

Studies ( Author, 2020 ; ' + 'Other, 2021 ) and [ 1 ] ' + 'with comma ( Foo, 2019 , more).

' + ) + self.assertEqual( + 'Studies (Author, 2020; Other, 2021) and [1] with comma (Foo, 2019, more).', + xml_utils.get_text_from_node(xmltree), + ) + class TestGetTextFromMixedCitationNode(unittest.TestCase):