diff --git a/.secrets.baseline b/.secrets.baseline index 5887210..1ce7e54 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -459,15 +459,6 @@ "line_number": 200 } ], - "tests/unit/jobs/graph/test_neo4j_sync.py": [ - { - "type": "Secret Keyword", - "filename": "tests/unit/jobs/graph/test_neo4j_sync.py", - "hashed_secret": "25ab86bed149ca6ca9c1c0d5db7c9a91388ddeab", - "is_verified": false, - "line_number": 96 - } - ], "tests/unit/managers/test_postgres_manager.py": [ { "type": "Basic Auth Credentials", diff --git a/src/data_platform/dags/canonicalize_backfill.py b/src/data_platform/dags/canonicalize_backfill.py index aabda44..5397fd4 100644 --- a/src/data_platform/dags/canonicalize_backfill.py +++ b/src/data_platform/dags/canonicalize_backfill.py @@ -9,8 +9,8 @@ Schedule diario as 02:00 (defasado do ner_backfill, que roda as 04:00). O governador e o teto PRIMARIO; os args `--since`/`--limit` sao apenas guard secundario por execucao. -Espelha o padrao de generate_video_thumbnails (dispara Cloud Run) e sync_graph_to_neo4j -(import de airflow protegido + config via Variable.get com fallback). +Espelha o padrao de generate_video_thumbnails (dispara Cloud Run), com import de airflow +protegido e config via Variable.get com fallback. """ import logging diff --git a/src/data_platform/dags/sync_graph_to_neo4j.py b/src/data_platform/dags/sync_graph_to_neo4j.py deleted file mode 100644 index 5ec6b83..0000000 --- a/src/data_platform/dags/sync_graph_to_neo4j.py +++ /dev/null @@ -1,99 +0,0 @@ -"""DAG: Sincroniza a projecao de grafo de entidades para o Neo4j (Fase 6b). - -Le entity_registry (nos) + entity_edges (arestas agregadas) do Postgres e faz MERGE -idempotente no Neo4j via driver Bolt. Roda APOS project_entity_graph (que reconstroi -as tabelas), entao o grafo Neo4j espelha o estado mais recente da projecao Postgres. - -1o corte: so nos :Entity + arestas agregadas (sem nos :Article). Grafo compacto -(milhares de nos) para exploracao via Browser/Cypher. - -Espelha project_entity_graph.py / sync_pg_to_bigquery.py: -- Postgres via BaseHook.get_connection("postgres_default"). -- Config Neo4j via Airflow Variable "neo4j_bolt_url" (Secret Manager backend), - com fallback para env vars (NEO4J_BOLT_URL / NEO4J_BOLT_USER / NEO4J_BOLT_PASSWORD). -""" - -import json -import logging -import os -from datetime import datetime, timedelta - -try: - from airflow.decorators import dag, task - from airflow.hooks.base import BaseHook - from airflow.models import Variable -except ImportError: - pass - -logger = logging.getLogger(__name__) - - -def _resolve_bolt_config() -> dict: - """Resolve a config Bolt do Neo4j (Airflow Variable -> fallback env vars). - - A Variable "neo4j_bolt_url" guarda um JSON {"url","user","password"} (vem do secret - airflow-variables-neo4j_bolt_url). Se ausente, monta a partir de env vars. - """ - raw = Variable.get("neo4j_bolt_url", default_var=None) - if raw: - return raw if isinstance(raw, dict) else json.loads(raw) - - url = os.environ.get("NEO4J_BOLT_URL") - if not url: - raise ValueError( - "Config Neo4j ausente: defina a Airflow Variable 'neo4j_bolt_url' " - "(JSON com url/user/password) ou as env vars NEO4J_BOLT_URL/_USER/_PASSWORD." - ) - return { - "url": url, - "user": os.environ.get("NEO4J_BOLT_USER", "neo4j"), - "password": os.environ.get("NEO4J_BOLT_PASSWORD", ""), - } - - -@dag( - dag_id="sync_graph_to_neo4j", - description="Sincroniza entity_registry + entity_edges (Postgres) para o Neo4j via MERGE idempotente", - schedule="30 */6 * * *", # 30min apos project_entity_graph (que roda 0 */6) - start_date=datetime(2025, 1, 1), - catchup=False, - max_active_runs=1, - tags=["silver", "entities", "graph", "neo4j"], - default_args={ - "owner": "data-platform", - "retries": 1, - "retry_delay": timedelta(minutes=5), - "execution_timeout": timedelta(minutes=30), - }, - doc_md=""" - ### Sync da projecao de grafo para o Neo4j (Fase 6b) - - Le `entity_registry` (nos) + `entity_edges` (arestas agregadas) do Postgres e faz - `MERGE` idempotente no Neo4j (driver Bolt): - - - Nos: `(:Entity {entity_id, name, type, wikidata_id, agency_key})`. - - Arestas: `[:CO_MENTIONED_WITH {weight, article_count, first_seen, last_seen}]`, - `[:SUBORDINATE_TO]`, `[:IS_AGENCY]`. - - Roda apos `project_entity_graph`. Idempotente: re-rodar nao duplica nos/arestas. - """, -) -def sync_graph_to_neo4j(): - @task - def sync_to_neo4j(**context): - """Le do Postgres e faz MERGE idempotente no Neo4j.""" - from data_platform.jobs.graph.neo4j_sync import sync_graph_to_neo4j as run_sync - - conn = BaseHook.get_connection("postgres_default") - db_url = conn.get_uri().replace("postgres://", "postgresql://", 1) - - bolt_config = _resolve_bolt_config() - - result = run_sync(db_url, bolt_config) - logger.info("Sync do grafo para Neo4j concluido: %s", result) - return result - - sync_to_neo4j() - - -dag_instance = sync_graph_to_neo4j() diff --git a/src/data_platform/jobs/graph/neo4j_sync.py b/src/data_platform/jobs/graph/neo4j_sync.py deleted file mode 100644 index ed71b2f..0000000 --- a/src/data_platform/jobs/graph/neo4j_sync.py +++ /dev/null @@ -1,223 +0,0 @@ -"""Sync da projecao de grafo de entidades para o Neo4j (Fase 6b). - -Le os nos (entity_registry) e as arestas agregadas (entity_edges) do Postgres e faz -MERGE idempotente no Neo4j via driver Bolt. 1o corte: so nos :Entity + arestas -agregadas (sem nos :Article — o grafo fica compacto, milhares de nos). - -Idempotente: usa MERGE por chave estavel (entity_id nos nos; par + tipo nas arestas), -entao re-rodar nao duplica. As arestas estruturais sao direcionadas (subordinate_to / -is_agency); co_mention e nao-direcionada mas armazenada com ordem canonica src < dst -no Postgres, refletida aqui como uma unica aresta CO_MENTIONED_WITH por par. - -Conexao Neo4j: dict {"url","user","password"} vindo da Airflow Variable "neo4j_bolt_url" -(Secret Manager backend) com fallback para env vars (NEO4J_BOLT_URL/_USER/_PASSWORD). -""" - -import json -import logging - -from sqlalchemy import create_engine, text -from sqlalchemy.pool import NullPool - -logger = logging.getLogger(__name__) - -# Mapeia o `kind` em entity_edges para o tipo de relacionamento no Neo4j. -EDGE_KIND_TO_REL = { - "co_mention": "CO_MENTIONED_WITH", - "subordinate_to": "SUBORDINATE_TO", - "is_agency": "IS_AGENCY", -} - -# Tamanho de lote para o UNWIND (evita transacoes gigantes / estouro de memoria). -BATCH_SIZE = 1000 - -# --- Leitura do Postgres ----------------------------------------------------- - -SELECT_NODES_SQL = """ - SELECT - entity_id, - canonical_name AS name, - type, - wikidata_id, - agency_key - FROM entity_registry -""" - -SELECT_EDGES_SQL = """ - SELECT - src_id, - dst_id, - kind, - weight, - article_count, - first_seen, - last_seen - FROM entity_edges -""" - - -def fetch_nodes(db_url: str) -> list[dict]: - """Le todos os nos (entity_registry) do Postgres.""" - engine = create_engine(db_url, poolclass=NullPool) - try: - with engine.connect() as conn: - rows = conn.execute(text(SELECT_NODES_SQL)).mappings().all() - return [dict(r) for r in rows] - finally: - engine.dispose() - - -def fetch_edges(db_url: str) -> list[dict]: - """Le todas as arestas agregadas (entity_edges) do Postgres.""" - engine = create_engine(db_url, poolclass=NullPool) - try: - with engine.connect() as conn: - rows = conn.execute(text(SELECT_EDGES_SQL)).mappings().all() - edges: list[dict] = [] - for r in rows: - edge = dict(r) - # Datas como ISO string (o driver Neo4j aceita string; mantemos simples - # e deterministico no MERGE de propriedades). - for k in ("first_seen", "last_seen"): - if edge.get(k) is not None: - edge[k] = edge[k].isoformat() - edges.append(edge) - return edges - finally: - engine.dispose() - - -# --- Cypher (MERGE idempotente) --------------------------------------------- - -# Nos: MERGE por entity_id (chave estavel) + SET das propriedades. -MERGE_NODES_CYPHER = """ - UNWIND $rows AS row - MERGE (e:Entity {entity_id: row.entity_id}) - SET e.name = row.name, - e.type = row.type, - e.wikidata_id = row.wikidata_id, - e.agency_key = row.agency_key -""" - -# Arestas: o tipo de relacionamento e fixo por chamada (nao da pra parametrizar o label -# do relacionamento no Cypher), entao agrupamos por rel_type e usamos um template. -# MERGE pelo par (src, dst) garante idempotencia; SET atualiza os pesos a cada sync. -_MERGE_EDGES_CYPHER_TMPL = """ - UNWIND $rows AS row - MATCH (src:Entity {{entity_id: row.src_id}}) - MATCH (dst:Entity {{entity_id: row.dst_id}}) - MERGE (src)-[r:{rel_type}]->(dst) - SET r.weight = row.weight, - r.article_count = row.article_count, - r.first_seen = row.first_seen, - r.last_seen = row.last_seen -""" - -# Cleanup de nos obsoletos: o MERGE acima nunca remove nos que sairam do Postgres -# (ex.: o source de um merge/dedup, ou a promocao Wikidata-wins do add_alias que -# funde dgb_->QID continuamente no canon job). Sem este passo, o Neo4j acumula -# nos stale com arestas mortas. Remove todo :Entity cujo entity_id nao esta no -# conjunto corrente do Postgres. `count(*)` apos DETACH DELETE conta as linhas -# processadas (idioma valido no Cypher). -DELETE_STALE_NODES_CYPHER = """ - MATCH (e:Entity) - WHERE NOT e.entity_id IN $valid_ids - DETACH DELETE e - RETURN count(*) AS deleted -""" - - -def _chunked(items: list, size: int): - """Divide uma lista em lotes de tamanho `size`.""" - for i in range(0, len(items), size): - yield items[i : i + size] - - -def _resolve_neo4j_config(bolt_config: dict) -> tuple[str, str, str]: - """Extrai (url, user, password) do dict de configuracao, com defaults seguros.""" - url = bolt_config.get("url") or bolt_config.get("uri") - user = bolt_config.get("user", "neo4j") - password = bolt_config.get("password") - if not url or not password: - raise ValueError( - "Configuracao Neo4j incompleta: 'url' e 'password' sao obrigatorios " - "(recebido: chaves=%s)" % sorted(bolt_config.keys()) - ) - return url, user, password - - -def parse_bolt_config(raw: str | dict) -> dict: - """Normaliza a config Bolt (aceita JSON string ou dict).""" - if isinstance(raw, dict): - return raw - return json.loads(raw) - - -def sync_graph_to_neo4j(db_url: str, bolt_config: dict) -> dict: - """Sincroniza nos + arestas do Postgres para o Neo4j (MERGE idempotente). - - Args: - db_url: connection string do Postgres (postgresql://...). - bolt_config: dict {"url","user","password"} de conexao Bolt. - - Returns: - dict com contagem de nos/arestas sincronizados por tipo. - """ - # Import local: o driver so existe no ambiente do Composer (pypi_packages), - # nao no import-time dos testes / parsing de DAG. - from neo4j import GraphDatabase - - url, user, password = _resolve_neo4j_config(bolt_config) - - nodes = fetch_nodes(db_url) - edges = fetch_edges(db_url) - logger.info("Lidos do Postgres: %d nos, %d arestas", len(nodes), len(edges)) - - # Agrupa arestas por tipo de relacionamento (label nao e parametrizavel no Cypher). - edges_by_rel: dict[str, list[dict]] = {} - for edge in edges: - rel_type = EDGE_KIND_TO_REL.get(edge["kind"]) - if rel_type is None: - logger.warning("Ignorando aresta de kind desconhecido: %s", edge["kind"]) - continue - edges_by_rel.setdefault(rel_type, []).append(edge) - - result = {"nodes": 0, "edges": {}} - driver = GraphDatabase.driver(url, auth=(user, password)) - try: - with driver.session() as session: - # Constraint de unicidade (idempotente) garante MERGE rapido por entity_id. - session.run( - "CREATE CONSTRAINT entity_id_unique IF NOT EXISTS " - "FOR (e:Entity) REQUIRE e.entity_id IS UNIQUE" - ) - - # Nos primeiro (as arestas dependem deles via MATCH). - for batch in _chunked(nodes, BATCH_SIZE): - session.run(MERGE_NODES_CYPHER, rows=batch) - result["nodes"] += len(batch) - - # Arestas, por tipo de relacionamento. - for rel_type, rel_edges in edges_by_rel.items(): - cypher = _MERGE_EDGES_CYPHER_TMPL.format(rel_type=rel_type) - count = 0 - for batch in _chunked(rel_edges, BATCH_SIZE): - session.run(cypher, rows=batch) - count += len(batch) - result["edges"][rel_type] = count - - # Cleanup de nos stale (nao existem mais no Postgres). GUARD: so limpa - # se houve nos sincronizados — um fetch_nodes vazio (erro/transitorio) - # NUNCA deve zerar o grafo. - if nodes: - valid_ids = [n["entity_id"] for n in nodes] - rec = session.run(DELETE_STALE_NODES_CYPHER, valid_ids=valid_ids).single() - result["deleted_stale"] = rec["deleted"] if rec else 0 - else: - logger.warning("fetch_nodes vazio — pulando cleanup de nos stale (protecao)") - result["deleted_stale"] = 0 - finally: - driver.close() - - logger.info("Sync Neo4j concluido: %s", result) - return result diff --git a/tests/unit/dags/test_canonicalize_backfill.py b/tests/unit/dags/test_canonicalize_backfill.py index 4d58b31..6eea42d 100644 --- a/tests/unit/dags/test_canonicalize_backfill.py +++ b/tests/unit/dags/test_canonicalize_backfill.py @@ -2,8 +2,7 @@ Airflow + provider google nao estao instalados localmente e o modulo executa dag_instance = canonicalize_backfill() no import (apos try/except ImportError), -entao usamos testes estruturais via AST (mesmo padrao de test_sync_graph_to_neo4j -e test_generate_video_thumbnails). +entao usamos testes estruturais via AST (mesmo padrao de test_generate_video_thumbnails). """ import ast diff --git a/tests/unit/dags/test_ner_backfill.py b/tests/unit/dags/test_ner_backfill.py index fdae94f..af3b0d2 100644 --- a/tests/unit/dags/test_ner_backfill.py +++ b/tests/unit/dags/test_ner_backfill.py @@ -2,8 +2,7 @@ Airflow + provider google nao estao instalados localmente e o modulo executa dag_instance = ner_backfill() no import (apos try/except ImportError), entao -usamos testes estruturais via AST (mesmo padrao de test_sync_graph_to_neo4j -e test_generate_video_thumbnails). +usamos testes estruturais via AST (mesmo padrao de test_generate_video_thumbnails). """ import ast diff --git a/tests/unit/dags/test_sync_graph_to_neo4j.py b/tests/unit/dags/test_sync_graph_to_neo4j.py deleted file mode 100644 index 78f3c45..0000000 --- a/tests/unit/dags/test_sync_graph_to_neo4j.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Testes estruturais (AST) da DAG sync_graph_to_neo4j (Fase 6b). - -Airflow nao esta instalado localmente e o modulo executa dag_instance = sync_graph_to_neo4j() -no import, entao usamos testes estruturais via AST (mesmo padrao de test_generate_video_thumbnails). -""" - -import ast -from pathlib import Path - -DAG_FILE = Path("src/data_platform/dags/sync_graph_to_neo4j.py") - - -def _parse() -> ast.Module: - return ast.parse(DAG_FILE.read_text()) - - -def _source() -> str: - return DAG_FILE.read_text() - - -def test_neo4j_nao_importado_no_topo(): - """O driver neo4j so existe no Composer: nao pode ser import de topo (quebraria o parse).""" - tree = _parse() - top_level_imports = [] - for node in tree.body: - if isinstance(node, ast.Import): - top_level_imports += [a.name for a in node.names] - elif isinstance(node, ast.ImportFrom): - top_level_imports.append(node.module or "") - assert not any("neo4j" in (m or "") for m in top_level_imports) - - -def test_usa_postgres_default(): - assert 'get_connection("postgres_default")' in _source() - - -def test_dag_id_correto(): - assert 'dag_id="sync_graph_to_neo4j"' in _source() - - -def test_schedule_apos_project_entity_graph(): - """Roda 30min apos project_entity_graph (0 */6) -> 30 */6.""" - assert 'schedule="30 */6 * * *"' in _source() - - -def test_resolve_config_via_variable_e_fallback_env(): - src = _source() - assert 'Variable.get("neo4j_bolt_url"' in src - assert "NEO4J_BOLT_URL" in src - - -def test_delega_ao_job_neo4j_sync(): - assert "from data_platform.jobs.graph.neo4j_sync import" in _source() - - -def test_converte_postgres_para_postgresql(): - """Mesma normalizacao das demais DAGs (postgres:// -> postgresql://).""" - assert 'replace("postgres://", "postgresql://", 1)' in _source() diff --git a/tests/unit/jobs/graph/test_neo4j_sync.py b/tests/unit/jobs/graph/test_neo4j_sync.py deleted file mode 100644 index 099fa35..0000000 --- a/tests/unit/jobs/graph/test_neo4j_sync.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Testes do sync da projecao de grafo para o Neo4j (Fase 6b) — jobs/graph/neo4j_sync.py. - -Sem banco/Neo4j real: validamos os invariantes do Cypher (MERGE idempotente por chave -estavel), o mapeamento kind->relacionamento, o batching e a resolucao de config. A -orquestracao e testada com driver/session/engine mockados — o driver `neo4j` so existe -no ambiente do Composer e e importado lazily dentro da funcao. -""" - -import re -import sys -from unittest.mock import MagicMock, patch - -import pytest - -from data_platform.jobs.graph.neo4j_sync import ( - _MERGE_EDGES_CYPHER_TMPL, - BATCH_SIZE, - DELETE_STALE_NODES_CYPHER, - EDGE_KIND_TO_REL, - MERGE_NODES_CYPHER, - _chunked, - _resolve_neo4j_config, - parse_bolt_config, -) - - -def _normalize(s: str) -> str: - return re.sub(r"\s+", " ", s).strip().lower() - - -# --------------------------------------------------------------------------- -# Mapeamento kind -> relacionamento -# --------------------------------------------------------------------------- -class TestKindToRel: - def test_cobre_os_tres_tipos_de_aresta(self): - assert EDGE_KIND_TO_REL["co_mention"] == "CO_MENTIONED_WITH" - assert EDGE_KIND_TO_REL["subordinate_to"] == "SUBORDINATE_TO" - assert EDGE_KIND_TO_REL["is_agency"] == "IS_AGENCY" - - -# --------------------------------------------------------------------------- -# Invariantes do Cypher (idempotencia) -# --------------------------------------------------------------------------- -class TestCypherInvariants: - def test_nos_usam_merge_por_entity_id(self): - """MERGE por entity_id garante idempotencia (nao duplica nos).""" - norm = _normalize(MERGE_NODES_CYPHER) - assert "merge (e:entity {entity_id: row.entity_id})" in norm - # nao deve usar CREATE (que duplicaria a cada run) - assert "create (e:entity" not in norm - - def test_nos_setam_as_propriedades_esperadas(self): - norm = _normalize(MERGE_NODES_CYPHER) - for prop in ("e.name", "e.type", "e.wikidata_id", "e.agency_key"): - assert prop in norm - - def test_arestas_usam_match_dos_nos_e_merge_do_rel(self): - """Arestas: MATCH src/dst (nao MERGE de no) + MERGE do relacionamento.""" - cypher = _MERGE_EDGES_CYPHER_TMPL.format(rel_type="CO_MENTIONED_WITH") - norm = _normalize(cypher) - assert "match (src:entity {entity_id: row.src_id})" in norm - assert "match (dst:entity {entity_id: row.dst_id})" in norm - assert "merge (src)-[r:co_mentioned_with]->(dst)" in norm - - def test_arestas_setam_pesos(self): - cypher = _MERGE_EDGES_CYPHER_TMPL.format(rel_type="CO_MENTIONED_WITH") - norm = _normalize(cypher) - for prop in ("r.weight", "r.article_count", "r.first_seen", "r.last_seen"): - assert prop in norm - - def test_template_de_aresta_injeta_o_rel_type(self): - cypher = _MERGE_EDGES_CYPHER_TMPL.format(rel_type="SUBORDINATE_TO") - assert "[r:SUBORDINATE_TO]" in cypher - - def test_cleanup_remove_nos_fora_do_conjunto_valido(self): - """O cleanup deleta :Entity cujo entity_id NAO esta no conjunto corrente.""" - norm = _normalize(DELETE_STALE_NODES_CYPHER) - assert "match (e:entity)" in norm - assert "where not e.entity_id in $valid_ids" in norm - assert "detach delete e" in norm - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- -class TestChunked: - def test_divide_em_lotes(self): - assert list(_chunked([1, 2, 3, 4, 5], 2)) == [[1, 2], [3, 4], [5]] - - def test_lista_vazia(self): - assert list(_chunked([], 10)) == [] - - -class TestResolveConfig: - def test_extrai_url_user_password(self): - cfg = {"url": "bolt://10.0.0.5:7687", "user": "neo4j", "password": "s3cr3t"} - assert _resolve_neo4j_config(cfg) == ("bolt://10.0.0.5:7687", "neo4j", "s3cr3t") - - def test_aceita_uri_como_alias_de_url(self): - cfg = {"uri": "bolt://x:7687", "password": "p"} - url, user, _ = _resolve_neo4j_config(cfg) - assert url == "bolt://x:7687" - assert user == "neo4j" # default - - def test_falha_sem_password(self): - with pytest.raises(ValueError): - _resolve_neo4j_config({"url": "bolt://x:7687"}) - - def test_falha_sem_url(self): - with pytest.raises(ValueError): - _resolve_neo4j_config({"password": "p"}) - - -class TestParseBoltConfig: - def test_dict_passthrough(self): - d = {"url": "bolt://x", "password": "p"} - assert parse_bolt_config(d) is d - - def test_json_string(self): - assert parse_bolt_config('{"url": "bolt://x", "password": "p"}') == { - "url": "bolt://x", - "password": "p", - } - - -# --------------------------------------------------------------------------- -# Orquestracao (driver/session/engine mockados) -# --------------------------------------------------------------------------- -class TestSyncOrchestration: - def _install_fake_neo4j(self, driver): - """Injeta um modulo `neo4j` falso (nao instalado localmente).""" - fake = MagicMock() - fake.GraphDatabase.driver.return_value = driver - return patch.dict(sys.modules, {"neo4j": fake}), fake - - def test_merge_nos_antes_de_arestas_e_agrupa_por_rel(self): - from data_platform.jobs.graph import neo4j_sync - - session = MagicMock() - # session usado como context manager - driver = MagicMock() - driver.session.return_value.__enter__.return_value = session - - nodes = [ - {"entity_id": "Q1", "name": "A", "type": "ORG", "wikidata_id": "Q1", "agency_key": "a"}, - { - "entity_id": "Q2", - "name": "B", - "type": "PER", - "wikidata_id": None, - "agency_key": None, - }, - ] - edges = [ - { - "src_id": "Q1", - "dst_id": "Q2", - "kind": "co_mention", - "weight": 3, - "article_count": 3, - "first_seen": "2025-01-01", - "last_seen": "2025-02-01", - }, - { - "src_id": "Q1", - "dst_id": "Q2", - "kind": "subordinate_to", - "weight": 0, - "article_count": 0, - "first_seen": None, - "last_seen": None, - }, - { - "src_id": "Q9", - "dst_id": "Q9", - "kind": "lixo_desconhecido", - "weight": 1, - "article_count": 1, - "first_seen": None, - "last_seen": None, - }, - ] - - patcher, _ = self._install_fake_neo4j(driver) - with ( - patcher, - patch.object(neo4j_sync, "fetch_nodes", return_value=nodes), - patch.object(neo4j_sync, "fetch_edges", return_value=edges), - ): - result = neo4j_sync.sync_graph_to_neo4j( - "postgresql://x", {"url": "bolt://x:7687", "user": "neo4j", "password": "p"} - ) - - assert result["nodes"] == 2 - assert result["edges"] == {"CO_MENTIONED_WITH": 1, "SUBORDINATE_TO": 1} - # aresta de kind desconhecido foi ignorada (nao virou rel) - assert "lixo_desconhecido" not in result["edges"] - - # Verifica que houve a criacao da constraint de unicidade (idempotencia rapida). - constraint_calls = [ - c - for c in session.run.call_args_list - if "CONSTRAINT" in str(c.args[0]) and "IS UNIQUE" in str(c.args[0]) - ] - assert constraint_calls, "deve criar CONSTRAINT entity_id IS UNIQUE" - - driver.close.assert_called_once() - - def test_cleanup_deleta_nos_stale_com_valid_ids(self): - """Após o MERGE, roda o DELETE de nós stale com os entity_ids correntes.""" - from data_platform.jobs.graph import neo4j_sync - - session = MagicMock() - session.run.return_value.single.return_value = {"deleted": 4} - driver = MagicMock() - driver.session.return_value.__enter__.return_value = session - - nodes = [ - { - "entity_id": "Q1", - "name": "A", - "type": "ORG", - "wikidata_id": "Q1", - "agency_key": None, - }, - { - "entity_id": "Q2", - "name": "B", - "type": "PER", - "wikidata_id": None, - "agency_key": None, - }, - ] - - patcher, _ = self._install_fake_neo4j(driver) - with ( - patcher, - patch.object(neo4j_sync, "fetch_nodes", return_value=nodes), - patch.object(neo4j_sync, "fetch_edges", return_value=[]), - ): - result = neo4j_sync.sync_graph_to_neo4j( - "postgresql://x", {"url": "bolt://x:7687", "password": "p"} - ) - - # achou a chamada de DELETE com valid_ids = ids dos nós sincronizados - delete_calls = [c for c in session.run.call_args_list if "DETACH DELETE" in str(c.args[0])] - assert len(delete_calls) == 1 - assert delete_calls[0].kwargs["valid_ids"] == ["Q1", "Q2"] - assert result["deleted_stale"] == 4 - - def test_cleanup_pulado_quando_fetch_nodes_vazio(self): - """GUARD: fetch_nodes vazio NUNCA dispara o DELETE (não zera o grafo).""" - from data_platform.jobs.graph import neo4j_sync - - session = MagicMock() - driver = MagicMock() - driver.session.return_value.__enter__.return_value = session - - patcher, _ = self._install_fake_neo4j(driver) - with ( - patcher, - patch.object(neo4j_sync, "fetch_nodes", return_value=[]), - patch.object(neo4j_sync, "fetch_edges", return_value=[]), - ): - result = neo4j_sync.sync_graph_to_neo4j( - "postgresql://x", {"url": "bolt://x", "password": "p"} - ) - - delete_calls = [c for c in session.run.call_args_list if "DETACH DELETE" in str(c.args[0])] - assert delete_calls == [] - assert result["deleted_stale"] == 0 - - def test_driver_fechado_mesmo_com_erro(self): - from data_platform.jobs.graph import neo4j_sync - - driver = MagicMock() - driver.session.side_effect = RuntimeError("boom") - - patcher, _ = self._install_fake_neo4j(driver) - with ( - patcher, - patch.object(neo4j_sync, "fetch_nodes", return_value=[]), - patch.object(neo4j_sync, "fetch_edges", return_value=[]), - ): - with pytest.raises(RuntimeError): - neo4j_sync.sync_graph_to_neo4j( - "postgresql://x", {"url": "bolt://x", "password": "p"} - ) - - driver.close.assert_called_once() - - -def test_batch_size_positivo(): - assert BATCH_SIZE > 0