diff --git a/packtools/sps/formats/pdf/pipeline/xml.py b/packtools/sps/formats/pdf/pipeline/xml.py
index d0db5d2e7..a04738c7f 100644
--- a/packtools/sps/formats/pdf/pipeline/xml.py
+++ b/packtools/sps/formats/pdf/pipeline/xml.py
@@ -365,15 +365,14 @@ def extract_body_data(xml_tree, table_layout_overrides=None):
if sec['title'] is not None:
sec['title'] = ''.join(sec['title'].itertext()).strip()
- # Collect textual paragraphs but exclude figure/table elements
+ # Collect textual paragraphs but exclude figure/table elements. Uses
+ # get_text_from_node (tail-preserving) rather than a bare
+ # `.xpath('.//text()...')` + `' '.join(...)`, which inserted an
+ # artificial space between every text-node fragment regardless of
+ # whether the source had one there (e.g. "(...)" came
+ # out as "( ... )", and "; " as "... ; ...").
for para in document_section.findall('p'):
- try:
- # Get text nodes that are not inside fig or table-wrap
- texts = para.xpath('.//text()[not(ancestor::fig) and not(ancestor::table-wrap)]')
- para_text = ' '.join(' '.join(texts).split()).strip()
- except Exception:
- # Fallback to generic text extraction
- para_text = xml_utils.get_text_from_node(para)
+ para_text = xml_utils.get_text_from_node(para, skip_tags={'fig', 'table-wrap'}).strip()
if para_text:
sec['paragraphs'].append(para_text)
diff --git a/packtools/sps/formats/pdf/utils/xml_utils.py b/packtools/sps/formats/pdf/utils/xml_utils.py
index a250d945a..3bf3b90a1 100644
--- a/packtools/sps/formats/pdf/utils/xml_utils.py
+++ b/packtools/sps/formats/pdf/utils/xml_utils.py
@@ -1,40 +1,40 @@
-def get_text_from_node(node):
+import re
+
+
+def get_text_from_node(node, skip_tags=None):
"""
- Extracts text from an XML node, including its children.
+ Extracts text from an XML node, including its children, preserving the
+ adjacency of the source (no space is inserted between fragments unless
+ one was already there as literal text or a tail).
Args:
node (ElementTree): The XML node to extract text from.
+ skip_tags (set, optional): Child tag names to drop entirely from the
+ output; only their `.tail` (the text that follows them in the
+ source) is kept. Used to flatten a paragraph to readable text
+ while excluding embedded elements such as /.
Returns:
str: The text extracted from the given node.
"""
+ skip_tags = skip_tags or set()
texts_els = []
if node.text:
texts_els.append(node.text)
for child in node:
- if child.tag == 'xref':
- xref_text = child.text if child.text else ''
- for subchild in child:
- if subchild.tag in ('italic', 'bold'):
- xref_text += (subchild.text if subchild.text else '')
- if subchild.tail:
- xref_text += (subchild.tail if subchild.tail else '')
- texts_els.append(xref_text)
- elif child.tag in ('italic', 'bold'):
- if child.text:
- texts_els.append(child.text)
- for subchild in child:
- texts_els.append(get_text_from_node(subchild))
+ if child.tag in skip_tags:
+ pass
else:
- texts_els.append(get_text_from_node(child))
+ texts_els.append(get_text_from_node(child, skip_tags=skip_tags))
if child.tail:
texts_els.append(child.tail)
text = ''.join(texts_els)
text = _remove_double_spaces(text)
+ text = _normalize_punctuation_spacing(text)
return text
def get_node_level(element, root):
@@ -113,14 +113,36 @@ def _add_period(text):
def _remove_double_spaces(text):
"""
- Removes double spaces from the given text.
+ Collapses any run of whitespace (including tabs and newlines left over
+ from pretty-printed XML, e.g. the indentation tail of a skipped
+ /) into a single space.
+
+ Args:
+ text (str): The text to normalize.
+
+ Returns:
+ str: The text with whitespace runs collapsed to single spaces.
+ """
+ return re.sub(r'\s+', ' ', text)
+
+def _normalize_punctuation_spacing(text):
+ """
+ Removes whitespace that ends up glued to the inside of parentheses and
+ brackets, or before a comma/semicolon, when the source XML has a space
+ directly before/after an inline element such as (e.g. "(
+ Fig. 1 )") — a common defect that survives adjacency-preserving
+ extraction because the space is literal text, not an artifact of it.
Args:
- text (str): The text to remove double spaces from.
+ text (str): The text to normalize.
Returns:
- str: The text with double spaces removed.
+ str: The text with punctuation spacing normalized.
"""
- while ' ' in text:
- text = text.replace(' ', ' ')
+ text = re.sub(r'\(\s+', '(', text)
+ text = re.sub(r'\s+\)', ')', text)
+ text = re.sub(r'\[\s+', '[', text)
+ text = re.sub(r'\s+\]', ']', text)
+ text = re.sub(r'\s+;', ';', text)
+ text = re.sub(r'\s+,', ',', text)
return text
diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py
index 774630aae..583c41bc8 100644
--- a/tests/sps/formats/pdf/pipeline/test_xml.py
+++ b/tests/sps/formats/pdf/pipeline/test_xml.py
@@ -417,6 +417,42 @@ def test_extract_body_data_with_table_references(self):
result = xml_pipe.extract_body_data(xml)
self.assertEqual(result, expected)
+ def test_paragraph_citations_have_no_stray_space_around_parentheses(self):
+ # Regression: a naive `.xpath('.//text()...')` + `' '.join(...)`
+ # inserted a space between every text-node fragment regardless of
+ # adjacency in the source, turning "(...; ...
+ # )" into "( ... ; ... )".
+ xml = etree.fromstring(
+ 'Introduction'
+ '
Pressure is increasing '
+ '(Lang and Barling, 2012'
+ '; Ripple et al., 2019) '
+ 'worldwide.
'
+ ''
+ )
+ result = xml_pipe.extract_body_data(xml)
+ self.assertEqual(
+ result[0]['paragraphs'],
+ ['Pressure is increasing (Lang and Barling, 2012; Ripple et al., 2019) worldwide.'],
+ )
+
+ def test_embedded_fig_tail_whitespace_is_collapsed_not_left_raw(self):
+ # A skipped 's tail can carry the source's pretty-printing
+ # indentation (a newline + spaces); it must collapse to one space
+ # rather than leak into the rendered paragraph.
+ xml = etree.fromstring(
+ 'Results'
+ '
See the figure below\n'
+ '\n '
+ 'for details.
'
+ ''
+ )
+ result = xml_pipe.extract_body_data(xml)
+ self.assertEqual(
+ result[0]['paragraphs'],
+ ['See the figure below for details.'],
+ )
+
class TestExtractCategory(unittest.TestCase):
diff --git a/tests/sps/formats/pdf/utils/test_xml_utils.py b/tests/sps/formats/pdf/utils/test_xml_utils.py
index e36c6ab9f..9e606f179 100644
--- a/tests/sps/formats/pdf/utils/test_xml_utils.py
+++ b/tests/sps/formats/pdf/utils/test_xml_utils.py
@@ -55,6 +55,64 @@ def test_get_text_from_node_multiple_xref_italic(self):
result = xml_utils.get_text_from_node(xmltree)
self.assertEqual(expected, result)
+ def test_get_text_from_node_preserves_parenthesis_adjacency(self):
+ # No space should be inserted between "(" and the xref text, or
+ # between the xref text and ")", when none exists in the source.
+ xmltree = etree.fromstring(
+ '