2727 4. upsert edges owned by changed modules (+ the shared edges).
2828 5. on a FULL run only, prune modules whose source file vanished.
2929
30+ An eager full run also reconciles the app's artifacts first (``_reconcile_artifacts``).
31+
3032**A push never deletes by default** (#171). Steps 3 and 5 are the only destructive
3133ones and both run on ``eager`` (``--eager``) only; a default ``--lazy`` push is purely
3234additive — MERGE-upsert of nodes and edges, nothing removed. The cost of the default is
8587 "RETURN count(*) AS pruned"
8688)
8789
90+ # Artifact reconciliation (--eager full runs). The artifact vocabulary is shared with the
91+ # sibling analyzers, so only python-owned edges are dropped and rebuilt: HAS_ARTIFACT from
92+ # a :PyApplication, DECLARES_DEPENDENCY/LOCKS to pkg:pypi/, incoming PY_ edges, and the
93+ # app's PY_READS_CONFIG_UNRESOLVED. Not sha-diffed: siblings overwrite the shared sha256.
94+ PURGE_ARTIFACT_EDGES = (
95+ "UNWIND $pres AS pre "
96+ "MATCH (t) WHERE (t:Artifact OR t:ConfigKey) AND t.id STARTS WITH pre "
97+ "CALL { WITH t "
98+ "WITH [(t)<-[r]-(s) WHERE (type(r) = 'HAS_ARTIFACT' AND s:PyApplication) "
99+ "OR type(r) STARTS WITH 'PY_' | r] "
100+ "+ [(t)-[d:DECLARES_DEPENDENCY|LOCKS]->(p:Package) WHERE p.id STARTS WITH 'pkg:pypi/' | d] AS rs "
101+ "FOREACH (r IN rs | DELETE r) } IN TRANSACTIONS OF 1000 ROWS"
102+ )
103+ PURGE_UNRESOLVED_CONFIG_READS = (
104+ "MATCH (:PyApplication {id: $app_id})-[r:PY_READS_CONFIG_UNRESOLVED|PY_UNRESOLVED_IMPORT]->() "
105+ "CALL { WITH r DELETE r } IN TRANSACTIONS OF 1000 ROWS"
106+ )
107+ # Dependency → import edges into this app's ghosts; scoped by the ghost's app id.
108+ PURGE_PROVIDES = (
109+ "MATCH (:Package)-[r:PY_PROVIDES]->(g:PyExternal) WHERE g.id STARTS WITH $app_pre "
110+ "CALL { WITH r DELETE r } IN TRANSACTIONS OF 1000 ROWS"
111+ )
112+ # Release python's claim on artifacts/config keys the run no longer emits. An artifact is
113+ # deleted once no HAS_ARTIFACT is left (every analyzer inventories with one, marked or not);
114+ # a config key once no other analyzer's *CanNode claims it. Pre-1.5.0 artifacts carry no
115+ # marker; legacy roots are outside the selected identity and are left alone.
116+ RELEASE_VANISHED_ARTIFACTS = (
117+ "UNWIND $pres AS pre "
118+ "MATCH (n) WHERE (n:Artifact OR n:ConfigKey) AND n.id STARTS WITH pre AND NOT n.id IN $keep "
119+ f"AND n:{ CAN_NODE } "
120+ f"CALL {{ WITH n REMOVE n:{ CAN_NODE } "
121+ "WITH n WHERE (n:Artifact AND NOT ()-[:HAS_ARTIFACT]->(n)) "
122+ "OR (n:ConfigKey AND none(l IN labels(n) WHERE l ENDS WITH 'CanNode')) "
123+ "DETACH DELETE n } IN TRANSACTIONS OF 1000 ROWS "
124+ "RETURN count(*) AS released"
125+ )
126+
127+
128+ def _artifact_prefixes (app_name : str , namespace : str | None = None ) -> List [str ]:
129+ """Artifacts in the selected namespace/application only."""
130+ return [f"{ application_prefix (app_name , namespace )} artifact/" ]
131+
132+
133+ def _reconcile_artifacts (session , app_name : str , keep : List [str ], namespace : str | None = None ) -> None :
134+ """Drop python-owned artifact edges and release vanished artifacts/config keys.
135+ Auto-commit and batched; a partial failure is finished by the next eager push."""
136+ pres = _artifact_prefixes (app_name , namespace )
137+ app_pre = application_prefix (app_name , namespace )
138+ params = {"pres" : pres , "keep" : keep , "app_id" : app_pre [:- 1 ], "app_pre" : app_pre }
139+ with session () as s :
140+ s .run (PURGE_ARTIFACT_EDGES , ** params ).consume ()
141+ s .run (PURGE_UNRESOLVED_CONFIG_READS , ** params ).consume ()
142+ s .run (PURGE_PROVIDES , ** params ).consume ()
143+ rec = s .run (RELEASE_VANISHED_ARTIFACTS , ** params ).single ()
144+ logger .info (f"neo4j(bolt): released { rec ['released' ] if rec else 0 } vanished artifact/config node(s)" )
145+
146+
147+ def _artifact_keep (app ) -> List [str ]:
148+ """Every artifact and config-key id this run emits."""
149+ return [i for a in (app .artifacts or {}).values () for i in (a .id , * (k .id for k in a .config_keys ))]
150+
88151
89152@dataclass
90153class BoltConfig :
@@ -168,10 +231,12 @@ def finish(self):
168231class _EdgeBatches (RowBuilder ):
169232 """Writes the relationships owned by changed modules (or by no module) in batches."""
170233
171- def __init__ (self , session , neo4j , changed , keys , module_by_id ):
234+ def __init__ (self , session , neo4j , changed , keys , module_by_id , reconcile = False ):
172235 super ().__init__ ()
173236 self ._session , self ._neo4j , self ._changed = session , neo4j , changed
174237 self ._keys , self ._module_by_id = keys , module_by_id
238+ # After artifact reconciliation, edges into artifacts are rebuilt even from unchanged modules.
239+ self ._reconcile = reconcile
175240 self ._pending , self ._waiting = [], set ()
176241
177242 def _owner (self , value ):
@@ -189,7 +254,8 @@ def node(self, labels, key_prop, value, props):
189254
190255 def edge (self , type_ , from_ref , to_ref , props = None , key = None ):
191256 owner = self ._owner (from_ref .value )
192- if owner is not None and owner not in self ._changed :
257+ into_artifact = self ._reconcile and to_ref .label in ("Artifact" , "ConfigKey" )
258+ if owner is not None and owner not in self ._changed and not into_artifact :
193259 return
194260 identity = (type_ , from_ref , to_ref , key )
195261 if identity in self ._waiting :
@@ -212,8 +278,28 @@ def finish(self):
212278 return GraphRows ([], [])
213279
214280
281+ def _validate_stored_identity (session , app_name , namespace ):
282+ """Read the selected anchor before schema or graph mutations; absence is fresh."""
283+ from codeanalyzer .schema .ids import application_id
284+
285+ app_id = application_id (app_name , namespace )
286+ record = session .run (
287+ "MATCH (a:PyApplication {id: $id}) RETURN properties(a) AS identity" ,
288+ id = app_id ,
289+ ).single ()
290+ if record is None :
291+ return
292+ metadata = record ["identity" ]
293+ expected_namespace = app_name if namespace is None else namespace
294+ if (type (metadata .get ("identity_version" )) is not int
295+ or metadata .get ("identity_version" ) != 2
296+ or metadata .get ("application_name" ) != app_name
297+ or metadata .get ("identity_namespace" ) != expected_namespace ):
298+ raise ValueError ("Stored application identity is missing or incompatible" )
299+
300+
215301def bolt_writer_streaming (app , app_name , sig_to_id , analyzer , cfg : BoltConfig , full_run : bool ,
216- eager : bool = False ) -> None :
302+ eager : bool = False , namespace : str | None = None ) -> None :
217303 """:func:`bolt_writer` without materializing :class:`GraphRows`: the projection
218304 runs twice (nodes, then relationships) and is written in bounded batches."""
219305 try :
@@ -225,7 +311,7 @@ def bolt_writer_streaming(app, app_name, sig_to_id, analyzer, cfg: BoltConfig, f
225311 ) from exc
226312 from codeanalyzer .neo4j .project import project
227313
228- app_prefix = application_prefix (app_name )
314+ app_prefix = application_prefix (app_name , namespace )
229315 driver = neo4j .GraphDatabase .driver (cfg .uri , auth = (cfg .user , cfg .password ))
230316
231317 session_kwargs = {"database" : cfg .database } if cfg .database else {}
@@ -235,6 +321,7 @@ def session():
235321
236322 try :
237323 with session () as s :
324+ _validate_stored_identity (s , app_name , namespace )
238325 for stmt in CONSTRAINTS + INDEXES :
239326 s .run (stmt ).consume ()
240327 db_hash = {rec ["k" ]: rec ["h" ] for rec in s .run (
@@ -246,11 +333,14 @@ def session():
246333 or mod .content_hash != db_hash [mod .id ]}
247334 logger .info (f"neo4j(bolt): { len (app .symbol_table )} modules ({ len (changed )} changed)" )
248335
336+ reconcile = full_run and eager
337+ if reconcile :
338+ _reconcile_artifacts (session , app_name , _artifact_keep (app ), namespace )
249339 nodes = _NodeBatches (session , neo4j , changed , app_prefix , eager )
250- project (app , app_name , sig_to_id , analyzer = analyzer , builder = nodes , module_scoped = True )
340+ project (app , app_name , sig_to_id , analyzer = analyzer , namespace = namespace , builder = nodes , module_scoped = True )
251341 module_by_id = {mod .id : key for key , mod in app .symbol_table .items ()}
252- edges = _EdgeBatches (session , neo4j , changed , nodes ._keys , module_by_id )
253- project (app , app_name , sig_to_id , analyzer = analyzer , builder = edges )
342+ edges = _EdgeBatches (session , neo4j , changed , nodes ._keys , module_by_id , reconcile )
343+ project (app , app_name , sig_to_id , analyzer = analyzer , namespace = namespace , builder = edges )
254344
255345 if full_run and eager :
256346 with session () as s :
@@ -281,11 +371,6 @@ def session():
281371 return driver .session (** session_kwargs )
282372
283373 try :
284- # 1. schema (DDL runs in its own autocommit transactions).
285- with session () as s :
286- for stmt in [* CONSTRAINTS , * INDEXES ]:
287- s .run (stmt )
288-
289374 # The application anchor. Every destructive statement below is scoped to
290375 # ``can://<app>/``; an empty application is refused up front rather than
291376 # becoming ``STARTS WITH ''`` (every node in the database). The root row is
@@ -299,7 +384,17 @@ def session():
299384 ),
300385 None ,
301386 )
302- app_prefix = application_prefix (app_name )
387+ namespace = next ((n .props .get ("identity_namespace" ) for n in rows .nodes
388+ if n .labels and n .labels [0 ] == "PyApplication" ), None )
389+ app_prefix = application_prefix (app_name , namespace )
390+
391+ with session () as s :
392+ _validate_stored_identity (s , app_name , namespace )
393+
394+ # 1. schema (DDL runs in its own autocommit transactions).
395+ with session () as s :
396+ for stmt in [* CONSTRAINTS , * INDEXES ]:
397+ s .run (stmt )
303398
304399 # Partition nodes by owning module (an in-memory field, never emitted, #173);
305400 # shared nodes have none.
@@ -336,7 +431,11 @@ def session():
336431 f"{ len (shared )} shared nodes, { len (rows .edges )} edges"
337432 )
338433
339- # 3. shared nodes are always upserted (MERGE-only).
434+ # 3. eager full run: reconcile artifacts; then shared nodes are upserted (MERGE-only).
435+ reconcile = full_run and eager
436+ if reconcile :
437+ _reconcile_artifacts (session , app_name , [
438+ n .value for n in rows .nodes if n .labels and n .labels [0 ] in ("Artifact" , "ConfigKey" )], namespace )
340439 _upsert_nodes (session , neo4j , shared )
341440
342441 # 4. per changed module: purge owned edges + vanished decls, then upsert its nodes.
@@ -369,6 +468,7 @@ def _purge(tx, mid=module_id, node_keys=keys):
369468 e
370469 for e in rows .edges
371470 if module_of .get (e .from_ref .value ) is None or module_of .get (e .from_ref .value ) in changed
471+ or (reconcile and e .to_ref .label in ("Artifact" , "ConfigKey" ))
372472 ]
373473 _upsert_edges (session , neo4j , edges )
374474
0 commit comments