diff --git a/VERSION b/VERSION index 197c4d5..005119b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0 +2.4.1 diff --git a/log_manager/file_errors.py b/log_manager/file_errors.py index 9987819..50217f4 100644 --- a/log_manager/file_errors.py +++ b/log_manager/file_errors.py @@ -2,9 +2,10 @@ import hashlib import zlib - FILE_READ_ERROR_CODE = "file_read_error" FILE_READ_EXCEPTIONS = (EOFError, OSError, zlib.error) +CATALOG_EMPTY_PREFIX = "catalog-empty" +CATALOG_ERROR_PREFIX = "catalog-error" def build_file_read_error(exc, stage): @@ -18,15 +19,16 @@ def build_file_read_error(exc, stage): def build_catalog_error_hash(collection_code, path): - identity = f"catalog-error\0{collection_code}\0{path}".encode("utf-8") - return hashlib.md5(identity).hexdigest() + return _build_catalog_path_hash(CATALOG_ERROR_PREFIX, collection_code, path) + +def build_catalog_empty_hash(collection_code, path): + return _build_catalog_path_hash(CATALOG_EMPTY_PREFIX, collection_code, path) -def get_file_read_error(validation): - file_error = (validation or {}).get("file_error") or {} - if file_error.get("code") == FILE_READ_ERROR_CODE: - return file_error - return None + +def _build_catalog_path_hash(prefix, collection_code, path): + identity = f"{prefix}\0{collection_code}\0{path}".encode("utf-8") + return hashlib.md5(identity).hexdigest() def _get_error_kind(exc): diff --git a/log_manager/services/catalog.py b/log_manager/services/catalog.py index 100c500..aa5039f 100644 --- a/log_manager/services/catalog.py +++ b/log_manager/services/catalog.py @@ -100,6 +100,13 @@ def _catalog_log_files_in_directory( ) continue + # Path identity lets validation track empty files individually. + if file_hash == utils.EMPTY_CONTENT_HASH: + file_hash = file_errors.build_catalog_empty_hash( + collection.acron3, + file_path, + ) + _catalog_readable_file( collection=collection, path=file_path, @@ -122,16 +129,7 @@ def _get_file_read_error_paths(collection, directory_path): def _record_file_read_error(collection, path, stat_result, exc): error_hash = file_errors.build_catalog_error_hash(collection.acron3, path) with transaction.atomic(): - log_file = ( - models.LogFile.objects.select_for_update() - .filter( - collection=collection, - path=path, - status=choices.LOG_FILE_STATUS_ERROR, - validation__file_error__code=file_errors.FILE_READ_ERROR_CODE, - ) - .first() - ) + log_file = _get_path_placeholder(collection, path) if log_file is None: log_file = models.LogFile.create_or_update( collection=collection, @@ -141,6 +139,7 @@ def _record_file_read_error(collection, path, stat_result, exc): status=choices.LOG_FILE_STATUS_ERROR, ) + log_file.hash = error_hash log_file.path = path log_file.stat_result = stat_result log_file.status = choices.LOG_FILE_STATUS_ERROR @@ -156,29 +155,20 @@ def _record_file_read_error(collection, path, stat_result, exc): def _catalog_readable_file(collection, path, stat_result, file_hash): with transaction.atomic(): - path_error = ( - models.LogFile.objects.select_for_update() - .filter( - collection=collection, - path=path, - status=choices.LOG_FILE_STATUS_ERROR, - validation__file_error__code=file_errors.FILE_READ_ERROR_CODE, - ) - .first() - ) + path_placeholder = _get_path_placeholder(collection, path) canonical = ( models.LogFile.objects.select_for_update().filter(hash=file_hash).first() ) - if canonical and path_error and canonical.pk != path_error.pk: - path_error.delete() + if canonical and path_placeholder and canonical.pk != path_placeholder.pk: + path_placeholder.delete() canonical.updated = timezone.now() canonical.save(update_fields=["updated"]) return canonical - log_file = canonical or path_error + log_file = canonical or path_placeholder if log_file: - if file_errors.get_file_read_error(log_file.validation): + if path_placeholder and path_placeholder.hash != file_hash: _recover_readable_log_file(log_file, file_hash, path, stat_result) else: log_file.updated = timezone.now() @@ -193,6 +183,22 @@ def _catalog_readable_file(collection, path, stat_result, file_hash): ) +def _get_path_placeholder(collection, path): + placeholder_hashes = ( + file_errors.build_catalog_error_hash(collection.acron3, path), + file_errors.build_catalog_empty_hash(collection.acron3, path), + ) + return ( + models.LogFile.objects.select_for_update() + .filter( + collection=collection, + path=path, + hash__in=placeholder_hashes, + ) + .first() + ) + + def _recover_readable_log_file(log_file, file_hash, path, stat_result): log_file.hash = file_hash log_file.path = path diff --git a/log_manager/tests/test_catalog.py b/log_manager/tests/test_catalog.py index 3219969..079865d 100644 --- a/log_manager/tests/test_catalog.py +++ b/log_manager/tests/test_catalog.py @@ -9,7 +9,7 @@ from collection.models import Collection from log_manager import choices, file_errors, utils from log_manager.models import LogFile -from log_manager.services import catalog +from log_manager.services import catalog, validation class CatalogLogFilesTests(TestCase): @@ -62,6 +62,64 @@ def test_file_metadata_io_error_is_recorded(self): self.assertEqual(log_file.stat_result, {}) self.assertEqual(log_file.validation["file_error"]["kind"], "io") + def test_empty_files_are_cataloged_separately(self): + with tempfile.TemporaryDirectory() as directory: + first_path = Path(directory) / "2026-09-03_scielo.pe.log.gz" + second_path = Path(directory) / "2026-09-04_scielo.pe.log.gz" + for path in (first_path, second_path): + with gzip.open(path, "wb"): + pass + + self._catalog_directory(directory) + self._catalog_directory(directory) + + log_files = list(LogFile.objects.order_by("path")) + for log_file in log_files: + validation.validate_log_file_and_update_status(log_file.hash) + log_files = list(LogFile.objects.order_by("path")) + + self.assertEqual(len(log_files), 2) + self.assertEqual( + [log_file.hash for log_file in log_files], + [ + file_errors.build_catalog_empty_hash("per", str(first_path)), + file_errors.build_catalog_empty_hash("per", str(second_path)), + ], + ) + self.assertEqual( + {log_file.status for log_file in log_files}, + {choices.LOG_FILE_STATUS_INVALIDATED}, + ) + self.assertEqual( + {log_file.validation["probably_date"] for log_file in log_files}, + {None}, + ) + + def test_valid_replacement_recovers_empty_file_record(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "2026-09-04_scielo.pe.log.gz" + with gzip.open(path, "wb"): + pass + self._catalog_directory(directory) + + log_file = LogFile.objects.get() + log_file_id = log_file.pk + log_file.status = choices.LOG_FILE_STATUS_INVALIDATED + log_file.validation = {"probably_date": None} + log_file.save(update_fields=["status", "validation"]) + + with gzip.open(path, "wb") as output: + output.write(b"new content\n") + expected_hash = utils.hash_file(path) + self._catalog_directory(directory) + + log_file = LogFile.objects.get() + + self.assertEqual(log_file.pk, log_file_id) + self.assertEqual(log_file.hash, expected_hash) + self.assertEqual(log_file.status, choices.LOG_FILE_STATUS_CREATED) + self.assertEqual(log_file.validation, {}) + def test_valid_replacement_recovers_the_error_record(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "2026-09-04_scielo.pe.log.gz" diff --git a/log_manager/utils.py b/log_manager/utils.py index 7ddf6f1..7561b33 100644 --- a/log_manager/utils.py +++ b/log_manager/utils.py @@ -4,6 +4,8 @@ from scielo_log_validator import exceptions, validator +EMPTY_CONTENT_HASH = hashlib.md5(b"").hexdigest() + def hash_file(path, num_lines=500): """