Corrige coletores de Preprints e Dataverse

VERSION

@@ -1 +1 @@
-2.3.4
+2.3.5

core/collectors/dataverse.py

@@ -16,13 +16,8 @@ def _request_json(url):
         return {}
 
 
-def _get_subdataverses():
-    url = f"{settings.DATAVERSE_ENDPOINT}/dataverses/{settings.DATAVERSE_ROOT_COLLECTION}/contents"
-    return _request_json(url).get("data", [])
-
-
-def _get_datasets(subdataverse_id):
-    url = f"{settings.DATAVERSE_ENDPOINT}/dataverses/{subdataverse_id}/contents"
+def _get_dataverse_contents(dataverse_id):
+    url = f"{settings.DATAVERSE_ENDPOINT}/dataverses/{dataverse_id}/contents"
     return _request_json(url).get("data", [])
 
 
@@ -32,44 +27,71 @@ def _get_files(dataset_id):
 
 
 def iter_dataset_metadata(from_date=None, until_date=None):
-    for subdataverse in _get_subdataverses():
-        if subdataverse.get("type") != "dataverse":
+    yield from _iter_dataverse_contents(
+        settings.DATAVERSE_ROOT_COLLECTION,
+        settings.DATAVERSE_ROOT_COLLECTION,
+        from_date,
+        until_date,
+    )
+
+
+def _iter_dataverse_contents(
+    dataverse_id,
+    dataverse_title,
+    from_date=None,
+    until_date=None,
+):
+    for item in _get_dataverse_contents(dataverse_id):
+        item_type = item.get("type")
+
+        if item_type == "dataverse":
+            yield from _iter_dataverse_contents(
+                item["id"],
+                item["title"],
+                from_date,
+                until_date,
+            )
+            continue
+
+        if item_type != "dataset":
             continue
 
-        subdataverse_id = subdataverse["id"]
-        subdataverse_title = subdataverse["title"]
-
-        for dataset in _get_datasets(subdataverse_id):
-            if dataset.get("type") != "dataset":
-                continue
-
-            dataset_id = dataset["id"]
-            doi = standardizer.standardize_doi(dataset.get("persistentUrl"))
-            if not doi:
-                logging.warning("Dataset %s does not have a DOI.", dataset_id)
-                continue
-
-            publication_date = dataset.get("publicationDate")
-            if publication_date:
-                if (from_date and publication_date < from_date) or (
-                    until_date and publication_date > until_date
-                ):
-                    continue
-
-            for file_data in _get_files(dataset_id):
-                file_persistent_id = file_data["dataFile"].get("persistentId")
-                standardized_persistent_id = (
-                    standardizer.standardize_pid_generic(file_persistent_id)
-                    if file_persistent_id
-                    else None
-                )
-
-                yield {
-                    "title": subdataverse_title,
-                    "dataset_doi": doi,
-                    "dataset_published": publication_date,
-                    "file_id": file_data["dataFile"]["id"],
-                    "file_name": file_data["label"],
-                    "file_url": f"{settings.DATAVERSE_ENDPOINT}/access/datafile/{file_data['dataFile']['id']}",
-                    "file_persistent_id": standardized_persistent_id,
-                }
+        yield from _iter_dataset_files(
+            item,
+            dataverse_title,
+            from_date,
+            until_date,
+        )
+
+
+def _iter_dataset_files(dataset, dataverse_title, from_date=None, until_date=None):
+    dataset_id = dataset["id"]
+    doi = standardizer.standardize_doi(dataset.get("persistentUrl"))
+    if not doi:
+        logging.warning("Dataset %s does not have a DOI.", dataset_id)
+        return
+
+    publication_date = dataset.get("publicationDate")
+    if publication_date:
+        if (from_date and publication_date < from_date) or (
+            until_date and publication_date > until_date
+        ):
+            return
+
+    for file_data in _get_files(dataset_id):
+        file_persistent_id = file_data["dataFile"].get("persistentId")
+        standardized_persistent_id = (
+            standardizer.standardize_pid_generic(file_persistent_id)
+            if file_persistent_id
+            else None
+        )
+
+        yield {
+            "title": dataverse_title,
+            "dataset_doi": doi,
+            "dataset_published": publication_date,
+            "file_id": file_data["dataFile"]["id"],
+            "file_name": file_data["label"],
+            "file_url": f"{settings.DATAVERSE_ENDPOINT}/access/datafile/{file_data['dataFile']['id']}",
+            "file_persistent_id": standardized_persistent_id,
+        }

core/collectors/preprints.py

@@ -1,25 +1,77 @@
+import logging
+
 from django.conf import settings
+from requests.exceptions import HTTPError
 from sickle import Sickle
 
 from core.utils import standardizer
 
+FILTER_FALLBACK_STATUS_CODE = 500
+
 
 def iter_records(from_date, until_date):
     oai_client = Sickle(
         endpoint=settings.OAI_PMH_PREPRINT_ENDPOINT,
         max_retries=settings.OAI_PMH_MAX_RETRIES,
         verify=False,
     )
-    records = oai_client.ListRecords(
-        **{
-            "metadataPrefix": settings.OAI_METADATA_PREFIX,
-            "from": from_date,
-            "until": until_date,
-        }
+    yielded_identifiers = set()
+
+    try:
+        for record in _list_records(oai_client, from_date, until_date):
+            yielded_identifiers.add(record.header.identifier)
+            yield record
+        return
+    except HTTPError as exc:
+        response = exc.response
+        has_date_filter = from_date or until_date
+        if (
+            response is None
+            or response.status_code != FILTER_FALLBACK_STATUS_CODE
+            or not has_date_filter
+        ):
+            raise
+
+    logging.warning(
+        "Preprints OAI rejected date filters with HTTP 500. "
+        "Falling back to the full feed and filtering locally. "
+        "From: %s, Until: %s",
+        from_date,
+        until_date,
     )
 
-    for record in records:
-        yield record
+    for record in _list_records(oai_client):
+        if record.header.identifier in yielded_identifiers:
+            continue
+
+        if _is_record_in_date_range(record, from_date, until_date):
+            yield record
+
+
+def _list_records(oai_client, from_date=None, until_date=None):
+    params = {"metadataPrefix": settings.OAI_METADATA_PREFIX}
+
+    if from_date:
+        params["from"] = from_date
+
+    if until_date:
+        params["until"] = until_date
+
+    return oai_client.ListRecords(ignore_deleted=True, **params)
+
+
+def _is_record_in_date_range(record, from_date=None, until_date=None):
+    datestamp = str(getattr(record.header, "datestamp", ""))[:10]
+    if not datestamp:
+        return False
+
+    if from_date and datestamp < from_date:
+        return False
+
+    if until_date and datestamp > until_date:
+        return False
+
+    return True
 
 
 def extract_record_data(record):

core/tests/tests_dataverse_collector.py

@@ -0,0 +1,90 @@
+from unittest.mock import patch
+
+from django.test import SimpleTestCase, override_settings
+
+from core.collectors import dataverse
+
+
+@override_settings(
+    DATAVERSE_ENDPOINT="https://data.example/api",
+    DATAVERSE_ROOT_COLLECTION="root",
+)
+class DataverseCollectorTests(SimpleTestCase):
+    @patch("core.collectors.dataverse._get_files")
+    @patch("core.collectors.dataverse._get_dataverse_contents")
+    def test_iter_dataset_metadata_includes_root_and_nested_datasets(
+        self,
+        mock_get_contents,
+        mock_get_files,
+    ):
+        root_dataset = {
+            "id": 1,
+            "type": "dataset",
+            "persistentUrl": "https://doi.org/10.1234/root",
+            "publicationDate": "2026-09-01",
+        }
+        child_dataset = {
+            "id": 2,
+            "type": "dataset",
+            "persistentUrl": "https://doi.org/10.1234/child",
+            "publicationDate": "2026-09-02",
+        }
+        nested_dataset = {
+            "id": 3,
+            "type": "dataset",
+            "persistentUrl": "https://doi.org/10.1234/nested",
+            "publicationDate": "2026-09-03",
+        }
+        mock_get_contents.side_effect = {
+            "root": [
+                root_dataset,
+                {"id": 10, "type": "dataverse", "title": "Child"},
+            ],
+            10: [
+                child_dataset,
+                {"id": 20, "type": "dataverse", "title": "Nested"},
+            ],
+            20: [nested_dataset],
+        }.get
+        mock_get_files.side_effect = lambda dataset_id: [
+            {
+                "label": f"file-{dataset_id}.csv",
+                "dataFile": {
+                    "id": dataset_id * 100,
+                    "persistentId": f"doi:10.1234/file-{dataset_id}",
+                },
+            }
+        ]
+
+        payloads = list(dataverse.iter_dataset_metadata("2026-08-29", "2026-09-05"))
+
+        self.assertEqual(
+            [(payload["dataset_doi"], payload["title"]) for payload in payloads],
+            [
+                ("10.1234/root", "root"),
+                ("10.1234/child", "Child"),
+                ("10.1234/nested", "Nested"),
+            ],
+        )
+        self.assertEqual(mock_get_files.call_count, 3)
+
+    @patch("core.collectors.dataverse._get_files")
+    @patch("core.collectors.dataverse._get_dataverse_contents")
+    def test_iter_dataset_metadata_filters_root_dataset_by_date(
+        self,
+        mock_get_contents,
+        mock_get_files,
+    ):
+        mock_get_contents.return_value = [
+            {
+                "id": 1,
+                "type": "dataset",
+                "persistentUrl": "https://doi.org/10.1234/old",
+                "publicationDate": "2026-08-28",
+            }
+        ]
+
+        payloads = list(dataverse.iter_dataset_metadata("2026-08-29", "2026-09-05"))
+
+        self.assertEqual(payloads, [])
+        mock_get_files.assert_not_called()

core/tests/tests_preprints_collector.py

@@ -0,0 +1,83 @@
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+from django.test import SimpleTestCase, override_settings
+from requests.exceptions import HTTPError
+
+from core.collectors import preprints
+
+
+@override_settings(
+    OAI_PMH_PREPRINT_ENDPOINT="https://preprints.example/oai",
+    OAI_PMH_MAX_RETRIES=2,
+    OAI_METADATA_PREFIX="oai_dc",
+)
+class PreprintsCollectorTests(SimpleTestCase):
+    @patch("core.collectors.preprints.Sickle")
+    def test_iter_records_uses_remote_date_filter_and_ignores_deleted(
+        self, mock_sickle
+    ):
+        record = SimpleNamespace(
+            header=SimpleNamespace(
+                identifier="oai:preprints:1",
+                datestamp="2026-09-01T12:00:00Z",
+            )
+        )
+        mock_sickle.return_value.ListRecords.return_value = [record]
+
+        records = list(preprints.iter_records("2026-08-29", "2026-09-05"))
+
+        self.assertEqual(records, [record])
+        mock_sickle.return_value.ListRecords.assert_called_once_with(
+            ignore_deleted=True,
+            metadataPrefix="oai_dc",
+            **{"from": "2026-08-29", "until": "2026-09-05"},
+        )
+
+    @patch("core.collectors.preprints.Sickle")
+    def test_iter_records_filters_full_feed_after_date_filter_http_500(
+        self, mock_sickle
+    ):
+        response = Mock(status_code=500)
+        error = HTTPError(response=response)
+        old_record = SimpleNamespace(
+            header=SimpleNamespace(
+                identifier="oai:preprints:old",
+                datestamp="2026-08-28T23:59:59Z",
+            )
+        )
+        recent_record = SimpleNamespace(
+            header=SimpleNamespace(
+                identifier="oai:preprints:recent",
+                datestamp="2026-09-01T12:00:00Z",
+            )
+        )
+        future_record = SimpleNamespace(
+            header=SimpleNamespace(
+                identifier="oai:preprints:future",
+                datestamp="2026-09-06T00:00:00Z",
+            )
+        )
+        mock_sickle.return_value.ListRecords.side_effect = [
+            error,
+            [old_record, recent_record, future_record],
+        ]
+
+        records = list(preprints.iter_records("2026-08-29", "2026-09-05"))
+
+        self.assertEqual(records, [recent_record])
+        self.assertEqual(mock_sickle.return_value.ListRecords.call_count, 2)
+        mock_sickle.return_value.ListRecords.assert_called_with(
+            ignore_deleted=True,
+            metadataPrefix="oai_dc",
+        )
+
+    @patch("core.collectors.preprints.Sickle")
+    def test_iter_records_does_not_fallback_for_other_http_errors(self, mock_sickle):
+        response = Mock(status_code=503)
+        mock_sickle.return_value.ListRecords.side_effect = HTTPError(response=response)
+
+        with self.assertRaises(HTTPError):
+            list(preprints.iter_records("2026-08-29", "2026-09-05"))
+
+        self.assertEqual(mock_sickle.return_value.ListRecords.call_count, 1)