From af8930b682048ddf69361120b40f83287924804f Mon Sep 17 00:00:00 2001 From: Rossi-Luciano Date: Fri, 4 Sep 2026 14:41:17 -0300 Subject: [PATCH 1/3] fix: nao imprime afiliacao duplicada nao referenciada por nenhum contrib extract_contrib_data pegava TODO do documento via findall('.//aff') pra montar a lista impressa no PDF. Alguns pacotes SciELO carregam uma copia duplicada/orfa das afiliacoes que nenhum xref de contribuidor referencia (confirmado em 7 dos 26 artigos do corpus real: 27%). O padrao de id da copia duplicada varia bastante entre pacotes - aff1e/aff2e (a9.xml), aff0100/aff0200/... (a11.xml, a17.xml, a20.xml), aff1s/aff2s (a28.xml), aff1001 (a23.xml), ou ate um segundo bloco inteiro renumerado aff13-aff24 duplicando aff1-aff12 (a14.xml, o pior caso: 24 afiliacoes impressas, 12 duplicadas, ocupando quase uma pagina inteira sozinha). Nao da pra filtrar por padrao de id (variam demais), mas em todos os casos a copia duplicada nunca e referenciada por nenhum xref de contribuidor - so a copia original e. Coleta os rid de TODOS os xref[ref-type=aff] de cada contrib (nao so o primeiro, ao contrario do que a resolucao de rotulo em authors_names ja fazia) e restringe a lista de afiliacoes impressas as que tem id referenciado. Mantem fallback pra imprimir tudo se nenhum aff for referenciado (nao visto no corpus real, so defensivo). aff_mapping (usado pra resolver o rotulo sobrescrito de cada autor) fica inalterado, continua olhando todo do documento - so a lista final impressa foi filtrada. Validado contra as 26 amostras do corpus: as 7 com o problema caem para a contagem correta (a9: 4->2, a11: 8->4, a14: 24->12, a17: 2->1, a20: 2->1, a23: 2->1, a28: 4->2); as outras 19 nao mudam. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01X8r2LRJ3PGTT9vLaPtb373 --- packtools/sps/formats/pdf/pipeline/xml.py | 25 +++++- tests/sps/formats/pdf/pipeline/test_xml.py | 93 ++++++++++++++++++++++ 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/packtools/sps/formats/pdf/pipeline/xml.py b/packtools/sps/formats/pdf/pipeline/xml.py index d0db5d2e7..c130b49d1 100644 --- a/packtools/sps/formats/pdf/pipeline/xml.py +++ b/packtools/sps/formats/pdf/pipeline/xml.py @@ -112,6 +112,7 @@ def extract_contrib_data(xml_tree): contrib_group = xml_tree.find('.//contrib-group') if contrib_group is not None: aff_mapping = {} + referenced_aff_ids = set() for aff in xml_tree.findall('.//aff'): aff_id = aff.get('id') @@ -140,12 +141,30 @@ def extract_contrib_data(xml_tree): full_name += label full_name += corresp_mark authors_names.append(full_name) - - for aff in xml_tree.findall('.//aff'): + + for xref in contrib.findall('.//xref[@ref-type="aff"]'): + rid = xref.get('rid') + if rid: + referenced_aff_ids.add(rid) + + # Some SciELO packages carry duplicate/orphaned elements (seen + # in 7/26 of the real test corpus) that aren't referenced by any + # contributor's xref - e.g. a second, unused copy of the same + # affiliations under ids like aff1e/aff0100/aff1s, or (in one case) + # a full second set aff13-aff24 duplicating aff1-aff12. The id + # suffix convention isn't consistent enough to filter on, but none + # of these duplicates are ever cited, so restrict the printed list + # to affiliations actually referenced. Falls back to every if + # none are referenced at all (defensive; not seen in practice). + affs_to_print = xml_tree.findall('.//aff') + if referenced_aff_ids: + affs_to_print = [aff for aff in affs_to_print if aff.get('id') in referenced_aff_ids] + + for aff in affs_to_print: label = aff.find('label').text if aff.find('label') is not None else '' institution = aff.find('institution[@content-type="original"]') institution_name = institution.text if institution is not None else '' - + if institution_name: aff_info = f"{label}[^] {institution_name}" affiliations.append(aff_info) diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py index 774630aae..cae5e919a 100644 --- a/tests/sps/formats/pdf/pipeline/test_xml.py +++ b/tests/sps/formats/pdf/pipeline/test_xml.py @@ -644,6 +644,99 @@ def test_extract_contrib_data_missing_label(self): self.assertEqual(result['authors_names'], ['John Smith[^]']) self.assertEqual(result['affiliations'], ['[^] University X']) + def test_unreferenced_duplicate_affiliation_is_not_printed(self): + # Regression: some SciELO packages carry a duplicate/orphaned + # that no contributor's xref points to (seen in a9.xml of the real + # test corpus: aff1e/aff2e duplicate aff1/aff2 under a different id + # suffix). findall('.//aff') picked up every in the document + # regardless of whether any contrib actually cited it, so the + # affiliation printed twice. + xml = etree.fromstring(""" +
+ + + + Smith + John + + + + + + + University A + + + + University A + +
+ """) + result = xml_pipe.extract_contrib_data(xml) + self.assertEqual(result['affiliations'], ['I[^] University A']) + + def test_all_referenced_affiliations_are_kept_regardless_of_id_pattern(self): + # The duplicate 's id doesn't follow one fixed naming + # convention across real packages (seen: aff1e, aff0100, aff1s, and + # a fully separate aff13-aff24 numbering) - the fix must not assume + # one, and must not drop a legitimately different affiliation that + # simply happens to use an unusual id. + xml = etree.fromstring(""" +
+ + + + Smith + John + + + + + + Doe + Jane + + + + + + + University A + + + + University B + +
+ """) + result = xml_pipe.extract_contrib_data(xml) + self.assertEqual( + result['affiliations'], + ['1[^] University A', '2[^] University B'], + ) + + def test_falls_back_to_every_affiliation_when_none_are_referenced(self): + # Defensive: an article with no aff xrefs at all (not seen in the + # real corpus) must not end up with an empty affiliation list. + xml = etree.fromstring(""" +
+ + + + Smith + John + + + + + + University A + +
+ """) + result = xml_pipe.extract_contrib_data(xml) + self.assertEqual(result['affiliations'], ['1[^] University A']) + class TestExtractDOI(unittest.TestCase): From c684053b3bc4096a58b5f307425f27f0abd4ddc6 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Tue, 8 Sep 2026 14:31:00 -0300 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20limita=20afilia=C3=A7=C3=B5es=20ao?= =?UTF-8?q?=20article-meta=20principal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packtools/sps/formats/pdf/pipeline/xml.py | 28 +++------- tests/sps/formats/pdf/pipeline/test_xml.py | 61 +++++++++++++--------- 2 files changed, 43 insertions(+), 46 deletions(-) diff --git a/packtools/sps/formats/pdf/pipeline/xml.py b/packtools/sps/formats/pdf/pipeline/xml.py index c130b49d1..9a6af1a48 100644 --- a/packtools/sps/formats/pdf/pipeline/xml.py +++ b/packtools/sps/formats/pdf/pipeline/xml.py @@ -109,12 +109,14 @@ def extract_contrib_data(xml_tree): affiliations = [] corresponding_author = '' - contrib_group = xml_tree.find('.//contrib-group') + article_meta = xml_tree.find('./front/article-meta') + metadata_scope = article_meta if article_meta is not None else xml_tree + contrib_group = metadata_scope.find('.//contrib-group') if contrib_group is not None: aff_mapping = {} - referenced_aff_ids = set() + affs = metadata_scope.findall('.//aff') - for aff in xml_tree.findall('.//aff'): + for aff in affs: aff_id = aff.get('id') label = aff.find('label').text if aff.find('label') is not None else '' institution = aff.find('institution[@content-type="original"]') @@ -142,25 +144,7 @@ def extract_contrib_data(xml_tree): full_name += corresp_mark authors_names.append(full_name) - for xref in contrib.findall('.//xref[@ref-type="aff"]'): - rid = xref.get('rid') - if rid: - referenced_aff_ids.add(rid) - - # Some SciELO packages carry duplicate/orphaned elements (seen - # in 7/26 of the real test corpus) that aren't referenced by any - # contributor's xref - e.g. a second, unused copy of the same - # affiliations under ids like aff1e/aff0100/aff1s, or (in one case) - # a full second set aff13-aff24 duplicating aff1-aff12. The id - # suffix convention isn't consistent enough to filter on, but none - # of these duplicates are ever cited, so restrict the printed list - # to affiliations actually referenced. Falls back to every if - # none are referenced at all (defensive; not seen in practice). - affs_to_print = xml_tree.findall('.//aff') - if referenced_aff_ids: - affs_to_print = [aff for aff in affs_to_print if aff.get('id') in referenced_aff_ids] - - for aff in affs_to_print: + for aff in affs: label = aff.find('label').text if aff.find('label') is not None else '' institution = aff.find('institution[@content-type="original"]') institution_name = institution.text if institution is not None else '' diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py index cae5e919a..41f588dae 100644 --- a/tests/sps/formats/pdf/pipeline/test_xml.py +++ b/tests/sps/formats/pdf/pipeline/test_xml.py @@ -644,32 +644,45 @@ def test_extract_contrib_data_missing_label(self): self.assertEqual(result['authors_names'], ['John Smith[^]']) self.assertEqual(result['affiliations'], ['[^] University X']) - def test_unreferenced_duplicate_affiliation_is_not_printed(self): - # Regression: some SciELO packages carry a duplicate/orphaned - # that no contributor's xref points to (seen in a9.xml of the real - # test corpus: aff1e/aff2e duplicate aff1/aff2 under a different id - # suffix). findall('.//aff') picked up every in the document - # regardless of whether any contrib actually cited it, so the - # affiliation printed twice. + def test_subarticle_affiliation_is_not_printed(self): + # Regression: translated affiliations in a sub-article must not be + # included in the affiliation list of the main article. xml = etree.fromstring("""
- - - - Smith - John - - - - - - - University A - - - - University A - + + + + + + Smith + John + + + + + + + University A + + + + + + + + + Smith + John + + + + + + + Universidade A + + +
""") result = xml_pipe.extract_contrib_data(xml) From c3e997447af3d08c05a795a5ed6640d01303171a Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Tue, 8 Sep 2026 14:31:19 -0300 Subject: [PATCH 3/3] =?UTF-8?q?test:=20cobre=20escopo=20de=20afilia=C3=A7?= =?UTF-8?q?=C3=B5es=20do=20artigo=20principal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/sps/formats/pdf/pipeline/test_xml.py | 99 ++++++++++++---------- 1 file changed, 54 insertions(+), 45 deletions(-) diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py index 41f588dae..6939b05f2 100644 --- a/tests/sps/formats/pdf/pipeline/test_xml.py +++ b/tests/sps/formats/pdf/pipeline/test_xml.py @@ -688,38 +688,37 @@ def test_subarticle_affiliation_is_not_printed(self): result = xml_pipe.extract_contrib_data(xml) self.assertEqual(result['affiliations'], ['I[^] University A']) - def test_all_referenced_affiliations_are_kept_regardless_of_id_pattern(self): - # The duplicate 's id doesn't follow one fixed naming - # convention across real packages (seen: aff1e, aff0100, aff1s, and - # a fully separate aff13-aff24 numbering) - the fix must not assume - # one, and must not drop a legitimately different affiliation that - # simply happens to use an unusual id. + def test_main_article_affiliations_are_kept_regardless_of_id_pattern(self): xml = etree.fromstring("""
- - - - Smith - John - - - - - - Doe - Jane - - - - - - - University A - - - - University B - + + + + + + Smith + John + + + + + + Doe + Jane + + + + + + + University A + + + + University B + + +
""") result = xml_pipe.extract_contrib_data(xml) @@ -728,23 +727,33 @@ def test_all_referenced_affiliations_are_kept_regardless_of_id_pattern(self): ['1[^] University A', '2[^] University B'], ) - def test_falls_back_to_every_affiliation_when_none_are_referenced(self): - # Defensive: an article with no aff xrefs at all (not seen in the - # real corpus) must not end up with an empty affiliation list. + def test_main_article_affiliation_without_xref_is_printed(self): xml = etree.fromstring("""
- - - - Smith - John - - - - - - University A - + + + + + + Smith + John + + + + + + University A + + + + + + + + Universidade A + + +
""") result = xml_pipe.extract_contrib_data(xml)