diff --git a/VERSION b/VERSION index 3f684d2..cc6c9a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.4 +2.3.5 diff --git a/core/collectors/dataverse.py b/core/collectors/dataverse.py index ca51fd7..d154c11 100644 --- a/core/collectors/dataverse.py +++ b/core/collectors/dataverse.py @@ -16,13 +16,8 @@ def _request_json(url): return {} -def _get_subdataverses(): - url = f"{settings.DATAVERSE_ENDPOINT}/dataverses/{settings.DATAVERSE_ROOT_COLLECTION}/contents" - return _request_json(url).get("data", []) - - -def _get_datasets(subdataverse_id): - url = f"{settings.DATAVERSE_ENDPOINT}/dataverses/{subdataverse_id}/contents" +def _get_dataverse_contents(dataverse_id): + url = f"{settings.DATAVERSE_ENDPOINT}/dataverses/{dataverse_id}/contents" return _request_json(url).get("data", []) @@ -32,44 +27,71 @@ def _get_files(dataset_id): def iter_dataset_metadata(from_date=None, until_date=None): - for subdataverse in _get_subdataverses(): - if subdataverse.get("type") != "dataverse": + yield from _iter_dataverse_contents( + settings.DATAVERSE_ROOT_COLLECTION, + settings.DATAVERSE_ROOT_COLLECTION, + from_date, + until_date, + ) + + +def _iter_dataverse_contents( + dataverse_id, + dataverse_title, + from_date=None, + until_date=None, +): + for item in _get_dataverse_contents(dataverse_id): + item_type = item.get("type") + + if item_type == "dataverse": + yield from _iter_dataverse_contents( + item["id"], + item["title"], + from_date, + until_date, + ) + continue + + if item_type != "dataset": continue - subdataverse_id = subdataverse["id"] - subdataverse_title = subdataverse["title"] - - for dataset in _get_datasets(subdataverse_id): - if dataset.get("type") != "dataset": - continue - - dataset_id = dataset["id"] - doi = standardizer.standardize_doi(dataset.get("persistentUrl")) - if not doi: - logging.warning("Dataset %s does not have a DOI.", dataset_id) - continue - - publication_date = dataset.get("publicationDate") - if publication_date: - if (from_date and publication_date < from_date) or ( - until_date and publication_date > until_date - ): - continue - - for file_data in _get_files(dataset_id): - file_persistent_id = file_data["dataFile"].get("persistentId") - standardized_persistent_id = ( - standardizer.standardize_pid_generic(file_persistent_id) - if file_persistent_id - else None - ) - - yield { - "title": subdataverse_title, - "dataset_doi": doi, - "dataset_published": publication_date, - "file_id": file_data["dataFile"]["id"], - "file_name": file_data["label"], - "file_url": f"{settings.DATAVERSE_ENDPOINT}/access/datafile/{file_data['dataFile']['id']}", - "file_persistent_id": standardized_persistent_id, - } + yield from _iter_dataset_files( + item, + dataverse_title, + from_date, + until_date, + ) + + +def _iter_dataset_files(dataset, dataverse_title, from_date=None, until_date=None): + dataset_id = dataset["id"] + doi = standardizer.standardize_doi(dataset.get("persistentUrl")) + if not doi: + logging.warning("Dataset %s does not have a DOI.", dataset_id) + return + + publication_date = dataset.get("publicationDate") + if publication_date: + if (from_date and publication_date < from_date) or ( + until_date and publication_date > until_date + ): + return + + for file_data in _get_files(dataset_id): + file_persistent_id = file_data["dataFile"].get("persistentId") + standardized_persistent_id = ( + standardizer.standardize_pid_generic(file_persistent_id) + if file_persistent_id + else None + ) + + yield { + "title": dataverse_title, + "dataset_doi": doi, + "dataset_published": publication_date, + "file_id": file_data["dataFile"]["id"], + "file_name": file_data["label"], + "file_url": f"{settings.DATAVERSE_ENDPOINT}/access/datafile/{file_data['dataFile']['id']}", + "file_persistent_id": standardized_persistent_id, + } diff --git a/core/collectors/preprints.py b/core/collectors/preprints.py index bead72c..134b5ac 100644 --- a/core/collectors/preprints.py +++ b/core/collectors/preprints.py @@ -1,8 +1,13 @@ +import logging + from django.conf import settings +from requests.exceptions import HTTPError from sickle import Sickle from core.utils import standardizer +FILTER_FALLBACK_STATUS_CODE = 500 + def iter_records(from_date, until_date): oai_client = Sickle( @@ -10,16 +15,63 @@ def iter_records(from_date, until_date): max_retries=settings.OAI_PMH_MAX_RETRIES, verify=False, ) - records = oai_client.ListRecords( - **{ - "metadataPrefix": settings.OAI_METADATA_PREFIX, - "from": from_date, - "until": until_date, - } + yielded_identifiers = set() + + try: + for record in _list_records(oai_client, from_date, until_date): + yielded_identifiers.add(record.header.identifier) + yield record + return + except HTTPError as exc: + response = exc.response + has_date_filter = from_date or until_date + if ( + response is None + or response.status_code != FILTER_FALLBACK_STATUS_CODE + or not has_date_filter + ): + raise + + logging.warning( + "Preprints OAI rejected date filters with HTTP 500. " + "Falling back to the full feed and filtering locally. " + "From: %s, Until: %s", + from_date, + until_date, ) - for record in records: - yield record + for record in _list_records(oai_client): + if record.header.identifier in yielded_identifiers: + continue + + if _is_record_in_date_range(record, from_date, until_date): + yield record + + +def _list_records(oai_client, from_date=None, until_date=None): + params = {"metadataPrefix": settings.OAI_METADATA_PREFIX} + + if from_date: + params["from"] = from_date + + if until_date: + params["until"] = until_date + + return oai_client.ListRecords(ignore_deleted=True, **params) + + +def _is_record_in_date_range(record, from_date=None, until_date=None): + datestamp = str(getattr(record.header, "datestamp", ""))[:10] + if not datestamp: + return False + + if from_date and datestamp < from_date: + return False + + if until_date and datestamp > until_date: + return False + + return True def extract_record_data(record): diff --git a/core/tests/tests_dataverse_collector.py b/core/tests/tests_dataverse_collector.py new file mode 100644 index 0000000..1d9602d --- /dev/null +++ b/core/tests/tests_dataverse_collector.py @@ -0,0 +1,90 @@ +from unittest.mock import patch + +from django.test import SimpleTestCase, override_settings + +from core.collectors import dataverse + + +@override_settings( + DATAVERSE_ENDPOINT="https://data.example/api", + DATAVERSE_ROOT_COLLECTION="root", +) +class DataverseCollectorTests(SimpleTestCase): + @patch("core.collectors.dataverse._get_files") + @patch("core.collectors.dataverse._get_dataverse_contents") + def test_iter_dataset_metadata_includes_root_and_nested_datasets( + self, + mock_get_contents, + mock_get_files, + ): + root_dataset = { + "id": 1, + "type": "dataset", + "persistentUrl": "https://doi.org/10.1234/root", + "publicationDate": "2026-09-01", + } + child_dataset = { + "id": 2, + "type": "dataset", + "persistentUrl": "https://doi.org/10.1234/child", + "publicationDate": "2026-09-02", + } + nested_dataset = { + "id": 3, + "type": "dataset", + "persistentUrl": "https://doi.org/10.1234/nested", + "publicationDate": "2026-09-03", + } + mock_get_contents.side_effect = { + "root": [ + root_dataset, + {"id": 10, "type": "dataverse", "title": "Child"}, + ], + 10: [ + child_dataset, + {"id": 20, "type": "dataverse", "title": "Nested"}, + ], + 20: [nested_dataset], + }.get + mock_get_files.side_effect = lambda dataset_id: [ + { + "label": f"file-{dataset_id}.csv", + "dataFile": { + "id": dataset_id * 100, + "persistentId": f"doi:10.1234/file-{dataset_id}", + }, + } + ] + + payloads = list(dataverse.iter_dataset_metadata("2026-08-29", "2026-09-05")) + + self.assertEqual( + [(payload["dataset_doi"], payload["title"]) for payload in payloads], + [ + ("10.1234/root", "root"), + ("10.1234/child", "Child"), + ("10.1234/nested", "Nested"), + ], + ) + self.assertEqual(mock_get_files.call_count, 3) + + @patch("core.collectors.dataverse._get_files") + @patch("core.collectors.dataverse._get_dataverse_contents") + def test_iter_dataset_metadata_filters_root_dataset_by_date( + self, + mock_get_contents, + mock_get_files, + ): + mock_get_contents.return_value = [ + { + "id": 1, + "type": "dataset", + "persistentUrl": "https://doi.org/10.1234/old", + "publicationDate": "2026-08-28", + } + ] + + payloads = list(dataverse.iter_dataset_metadata("2026-08-29", "2026-09-05")) + + self.assertEqual(payloads, []) + mock_get_files.assert_not_called() diff --git a/core/tests/tests_preprints_collector.py b/core/tests/tests_preprints_collector.py new file mode 100644 index 0000000..376644a --- /dev/null +++ b/core/tests/tests_preprints_collector.py @@ -0,0 +1,83 @@ +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from django.test import SimpleTestCase, override_settings +from requests.exceptions import HTTPError + +from core.collectors import preprints + + +@override_settings( + OAI_PMH_PREPRINT_ENDPOINT="https://preprints.example/oai", + OAI_PMH_MAX_RETRIES=2, + OAI_METADATA_PREFIX="oai_dc", +) +class PreprintsCollectorTests(SimpleTestCase): + @patch("core.collectors.preprints.Sickle") + def test_iter_records_uses_remote_date_filter_and_ignores_deleted( + self, mock_sickle + ): + record = SimpleNamespace( + header=SimpleNamespace( + identifier="oai:preprints:1", + datestamp="2026-09-01T12:00:00Z", + ) + ) + mock_sickle.return_value.ListRecords.return_value = [record] + + records = list(preprints.iter_records("2026-08-29", "2026-09-05")) + + self.assertEqual(records, [record]) + mock_sickle.return_value.ListRecords.assert_called_once_with( + ignore_deleted=True, + metadataPrefix="oai_dc", + **{"from": "2026-08-29", "until": "2026-09-05"}, + ) + + @patch("core.collectors.preprints.Sickle") + def test_iter_records_filters_full_feed_after_date_filter_http_500( + self, mock_sickle + ): + response = Mock(status_code=500) + error = HTTPError(response=response) + old_record = SimpleNamespace( + header=SimpleNamespace( + identifier="oai:preprints:old", + datestamp="2026-08-28T23:59:59Z", + ) + ) + recent_record = SimpleNamespace( + header=SimpleNamespace( + identifier="oai:preprints:recent", + datestamp="2026-09-01T12:00:00Z", + ) + ) + future_record = SimpleNamespace( + header=SimpleNamespace( + identifier="oai:preprints:future", + datestamp="2026-09-06T00:00:00Z", + ) + ) + mock_sickle.return_value.ListRecords.side_effect = [ + error, + [old_record, recent_record, future_record], + ] + + records = list(preprints.iter_records("2026-08-29", "2026-09-05")) + + self.assertEqual(records, [recent_record]) + self.assertEqual(mock_sickle.return_value.ListRecords.call_count, 2) + mock_sickle.return_value.ListRecords.assert_called_with( + ignore_deleted=True, + metadataPrefix="oai_dc", + ) + + @patch("core.collectors.preprints.Sickle") + def test_iter_records_does_not_fallback_for_other_http_errors(self, mock_sickle): + response = Mock(status_code=503) + mock_sickle.return_value.ListRecords.side_effect = HTTPError(response=response) + + with self.assertRaises(HTTPError): + list(preprints.iter_records("2026-08-29", "2026-09-05")) + + self.assertEqual(mock_sickle.return_value.ListRecords.call_count, 1) diff --git a/document/tasks/preprints.py b/document/tasks/preprints.py index 1f2d2e2..1ed2039 100644 --- a/document/tasks/preprints.py +++ b/document/tasks/preprints.py @@ -8,7 +8,6 @@ from core.utils import date_utils from core.utils.request_utils import _get_user from document.services import preprint as preprint_service - from document.tasks.common import _get_collection @@ -36,6 +35,10 @@ def load_preprints_from_preprints_api( return False for record in preprints_collector.iter_records(from_date, until_date): + if getattr(record, "deleted", False) or not getattr(record, "metadata", None): + logging.info("Skipping deleted or metadata-less preprint: %s", record) + continue + payload = preprints_collector.extract_record_data(record) if not payload.get("pid_generic"): diff --git a/document/tests/test_tasks.py b/document/tests/test_tasks.py index 2291846..d64e9ec 100644 --- a/document/tests/test_tasks.py +++ b/document/tests/test_tasks.py @@ -1,3 +1,4 @@ +from types import SimpleNamespace from unittest.mock import patch import pytest @@ -7,6 +8,7 @@ from document.models import Document from document.tasks import common as document_tasks_common from document.tasks import opac as document_tasks_opac +from document.tasks import preprints as document_tasks_preprints from document.tasks import scielo_books as document_tasks_scielo_books from source.models import Source @@ -140,3 +142,42 @@ def test_load_documents_refuses_collection_without_endpoint(self): assert result is False mock_fetch_counter_dict.assert_not_called() + + +class DocumentPreprintsSyncTests(TestCase): + @patch("document.tasks.preprints.preprint_service.upsert_preprint_document") + @patch("document.tasks.preprints.preprints_collector.extract_record_data") + @patch("document.tasks.preprints.preprints_collector.iter_records") + @patch("document.tasks.preprints._get_collection") + def test_load_skips_deleted_and_metadata_less_records( + self, + mock_get_collection, + mock_iter_records, + mock_extract_record_data, + mock_upsert, + ): + collection = SimpleNamespace(acron3="preprints") + deleted_record = SimpleNamespace(deleted=True) + metadata_less_record = SimpleNamespace(deleted=False) + valid_record = SimpleNamespace(deleted=False, metadata={"title": ["Title"]}) + mock_get_collection.return_value = collection + mock_iter_records.return_value = [ + deleted_record, + metadata_less_record, + valid_record, + ] + mock_extract_record_data.return_value = {"pid_generic": "123"} + + result = document_tasks_preprints.load_preprints_from_preprints_api( + from_date="2026-08-29", + until_date="2026-09-05", + ) + + self.assertTrue(result) + mock_extract_record_data.assert_called_once_with(valid_record) + mock_upsert.assert_called_once_with( + {"pid_generic": "123"}, + collection=collection, + user=None, + force_update=True, + )