diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..70877e3b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## [3.4.0] - 2026-10-31 + +- Add `--namespace` and namespace-qualified application IDs. +- Scope embedded references, graph reconciliation and caches to the selected application. +- Validate stored identity metadata before connected writes. + +Regenerate existing analyses and graphs from source for the new ID roots. diff --git a/README.md b/README.md index 261f1cb5..82d210af 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ Usage: codeanalyzer [-hvV] [--no-build] [--no-clean-dependencies] [-a=] [-b=] [-f=] [-i=] [-o=] [-s=] [--emit=] [--app-name=] + [--namespace=] [--neo4j-uri=] [--neo4j-user=] [--neo4j-password=] [--neo4j-database=] [-t=]... @@ -145,6 +146,7 @@ Analyze java application. schema (the Neo4j schema.neo4j.json contract). --app-name= Logical application name for the graph :JApplication anchor (default: input dir name). + --namespace= Identity namespace (defaults to application name). --neo4j-uri= Push the graph to a live Neo4j over Bolt (incremental); omit to write graph.cypher. Falls back to the NEO4J_URI environment variable. @@ -157,6 +159,24 @@ Analyze java application. -V, --version Print version information and exit. ``` +**Application identity.** `--app-name` defaults to the basename of `--input` and `--namespace` +defaults to the resolved application name. The canonical root always contains both segments: +`can:///`, including `can://inventory/inventory` for a default monolith. +Names are case-sensitive raw strings encoded as UTF-8 URI segments without trimming or Unicode +normalization. Explicit empty/whitespace-only names, controls, unpaired surrogates, slashes, +backslashes, `.` and `..` are rejected. `--namespace` is unavailable with `--schema v1`. + +```bash +codeanalyzer -i monorepo/services/orders --app-name orders --namespace commerce +# root: can://commerce/orders +# symbol_table key: src/main/java/com/acme/Order.java +``` + +File and artifact keys remain relative to `--input`. The draft `--workspace-root` option has +been removed. Analysis envelopes and graph application anchors carry `identity_version: 2`; +anchors also retain raw `application_name` and `identity_namespace`. Namespace changes invalidate +the L1 cache. Legacy outputs must be regenerated from source; retire legacy graph data explicitly. + ## 3. Installing `codeanalyzer` as a native binary (once built, no JVM will be required for running `codeanalyzer`) @@ -241,15 +261,14 @@ build manifest or a configuration key is not a Java concept — a sibling-langua the same repository lands on the same nodes instead of a per-language duplicate. `SCHEMA_VERSION` is stamped onto the `:JApplication` node of every emitted graph. -Every node id starts with `can:///` — `can:///java/…` for code, `can:///artifact/…` -for build manifests and config keys, `can:///@external/…` for library symbols — so -`can://` is a prefix of every node the application emits, which is what the destructive +Every node id starts with `can:////` — `can:////java/…` for code, `can:////artifact/…` +for build manifests and config keys, `can:////@external/…` for library symbols — so +`can:///` is a prefix of every node the application emits, which is what the destructive statements scope on. (`:Package` is the exception: it is keyed on a `pkg:` purl, sits under no application, and no wipe reaches it.) `:JApplication` is keyed on that id, not on the free-text `--app-name`, so the root is addressable by the same id its descendants are prefixed with. The id -is derived from `--app-name` (`can://`), so it does -**not** disambiguate two services analyzed under the same name — those still merge onto one root. -Give each service its own `--app-name` if they share a database. +is derived from the pair (`--namespace`, `--app-name`). Services with the same raw application name +remain distinct in different namespaces. Services selecting the same pair merge onto the same root. **Polyglot ids.** The `/java/` segment is what makes a code node this analyzer's. Three id families deliberately omit it and are therefore **language-neutral, shared merge targets**: `:Artifact`, @@ -257,7 +276,7 @@ deliberately omit it and are therefore **language-neutral, shared merge targets* ids for them, so a `pom.xml`, a config key or `java.util.Map#get` is *one* node in a merged graph rather than a per-language duplicate. That sharing is the point, and its cost is accepted: two analyzers' notions of a library symbol are not necessarily the same thing, and merging them says -they are. The consequence for writes: because these shared nodes sit inside `can:///`, a Cypher +they are. The consequence for writes: because these shared nodes sit inside `can:////`, a Cypher snapshot's prefix wipe rebuilds them, so a cross-language edge into a shared `:Artifact` is dropped by one analyzer's snapshot and restored on the other's next push. @@ -297,14 +316,13 @@ codeanalyzer -i /path/to/project -a 2 --emit neo4j -o ./out cypher-shell -u neo4j -p < ./out/graph.cypher ``` -The snapshot is **not** incremental: it constraints, scopes-wipes this application's prior subgraph, -then `UNWIND … MERGE`-loads the full truth. The wipe is everything under `can:///`, plus a -containment traversal that also replaces a prior v1 (pre-`3.0.0`) graph of the same application. +Offline snapshots validate incoming identity metadata; they do not inspect an existing database. Execute them only as part of explicit regeneration. The snapshot is **not** incremental: it constraints, scopes-wipes this application's prior subgraph, +then `UNWIND … MERGE`-loads the full truth. The wipe is everything under `can:////`, plus a +containment traversal scoped to the selected canonical application. Legacy roots are not retired automatically. That scope **includes** this application's `:Artifact` and `:ConfigKey` nodes — the snapshot rebuilds them, rather than leaving stale ones to accumulate with nothing ever cleaning them up. The one consequence: a sibling-language analyzer's edge into a shared `:Artifact` is dropped by a Java -snapshot and restored on that analyzer's next push. The Bolt writer does **not** do this — its -deletions are scoped per module, not per application. +snapshot and restored on that analyzer's next push. The Bolt writer is narrower; see §4.2. ### 4.2. Live incremental push over Bolt @@ -313,11 +331,23 @@ codeanalyzer -i /path/to/project -a 2 --emit neo4j \ --neo4j-uri bolt://localhost:7687 --neo4j-user neo4j --neo4j-password ``` -The Bolt writer reads the database's current state and updates **only what changed**: it diffs each +The Bolt writer checks the selected existing anchor's identity marker and raw names before DDL or writes, then updates **only what changed**: it diffs each compilation unit's `content_hash`, replaces just the changed units' subgraphs (idempotent `MERGE` upserts), and — on a full run — prunes units whose source file vanished. Combine with `--target-files` for a targeted, partial re-push (orphan pruning is then skipped). +On a full run it also reconciles the application's artifacts, so a deleted or renamed file, a +or removed dependency leaves nothing stale within the selected namespace/application: + +- Java-owned artifact edges are rebuilt from the run: `HAS_ARTIFACT`, `DECLARES_DEPENDENCY`/`LOCKS` + to `pkg:maven/…`, and code reads of them (`J_USES_CONFIG`, `J_DISPATCHES_TO`, + `J_READS_CONFIG_UNRESOLVED`), so a config-only change updates them too. +- An `:Artifact`/`:ConfigKey` the run no longer emits loses its `:JCanNode` label. An artifact is + then deleted only if no other analyzer's `HAS_ARTIFACT` still points at it; a config key only if + no other analyzer's marker (`:PyCanNode`, `:TSCanNode`) still claims it. A sibling's edges are + never removed. +- Legacy artifacts require explicit retirement; reconciliation never falls back to a legacy root. + ### 4.3. Schema contract ```sh diff --git a/docs/design/specs/2026-10-24-workspace-root.md b/docs/design/specs/2026-10-24-workspace-root.md new file mode 100644 index 00000000..6dbf1244 --- /dev/null +++ b/docs/design/specs/2026-10-24-workspace-root.md @@ -0,0 +1,28 @@ +# Application identity namespaces (supersedes draft workspace-root design) + +The unmerged `--workspace-root` proposal is withdrawn. File keys, module suffixes and artifact +paths remain relative to the selected `--input` directory. No path prefix rekeying or URI +translation is performed. + +`application_name` is the supplied `--app-name`, otherwise the input basename. `identity_namespace` +is the supplied `--namespace`, otherwise the resolved application name. Both segments are always +present: `can:///`, including `can://inventory/inventory`. + +Names retain case and Unicode spelling. UTF-8 bytes outside ASCII letters/digits/`-._~` are +percent-encoded with uppercase escapes. Empty or whitespace-only names, controls, unpaired +surrogates, slashes/backslashes and whole-name `.`/`..` are invalid. Explicit invalid flags never +fall back to defaults. Legacy schema v1 rejects `--namespace`. + +The existing schema family remains `2.0.0`. The analysis envelope and graph application anchors +carry `identity_version: 2`; anchors retain raw `application_name` and `identity_namespace`. +Missing persisted metadata is unsupported, not defaulted. Regenerate from source and retire legacy +outputs explicitly. Cache identity includes the canonical namespace/application root and marker. + +Java keeps its existing `/java///` suffixes and body ordinals. External +symbols and artifacts share the canonical application root; `pkg:` identities remain unchanged. +Canonical-parent builders forward their root unchanged. No mutable global namespace is used. + +Bolt checks the selected existing application anchor before DDL or data writes. Incremental hashes, +module purges, orphan pruning and artifact reconciliation stay within that selected application. +Shared-artifact retention rules remain intact. Offline snapshots validate incoming metadata but +cannot inspect the target database; execution is an explicit regeneration operation. diff --git a/gradle.properties b/gradle.properties index 427a07a3..e6640dcd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=3.3.4 +version=3.4.0 diff --git a/packaging/python/src/codeanalyzer_java/__init__.py b/packaging/python/src/codeanalyzer_java/__init__.py index 65375429..2069b67d 100644 --- a/packaging/python/src/codeanalyzer_java/__init__.py +++ b/packaging/python/src/codeanalyzer_java/__init__.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import List -__version__ = "0.0.0" +__version__ = "3.4.0" __all__ = ["jar_path", "java_path", "command", "__version__"] diff --git a/schema.neo4j.json b/schema.neo4j.json index 54a5897d..c0130713 100644 --- a/schema.neo4j.json +++ b/schema.neo4j.json @@ -13,6 +13,9 @@ "properties": { "id": "string", "name": "string", + "application_name": "string", + "identity_namespace": "string", + "identity_version": "integer", "schema_version": "string", "analyzer_name": "string", "analyzer_version": "string", diff --git a/src/main/java/com/ibm/cldk/CodeAnalyzer.java b/src/main/java/com/ibm/cldk/CodeAnalyzer.java index bc57b87b..698160fd 100644 --- a/src/main/java/com/ibm/cldk/CodeAnalyzer.java +++ b/src/main/java/com/ibm/cldk/CodeAnalyzer.java @@ -31,6 +31,7 @@ import com.ibm.cldk.neo4j.BoltConfig; import com.ibm.cldk.neo4j.Neo4jEmitter; import com.ibm.cldk.schema.Analysis; +import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JArtifact; import com.ibm.cldk.schema.JDependency; import com.ibm.cldk.schema.JEntrypointReport; @@ -146,6 +147,9 @@ public class CodeAnalyzer implements Runnable { // values between CommandLine instances in the same JVM. New flags do not add to that. private String schema = "v2"; + @Option(names = "--namespace", description = "Identity namespace (defaults to application name)") + private String namespace; + @Option(names = {"-c", "--cache-dir" }, description = "Directory holding the incremental analysis cache. When set, " + "unchanged files are reused from analysis_cache.json instead of being reparsed.") @@ -261,6 +265,11 @@ private void analyze() throws Exception { analyzeV2(); return; } + if (namespace != null) { + // v1 keys are absolute paths, so there is nothing for a workspace root to relativize. + throw new ParameterException(spec.commandLine(), + "error: --namespace is not supported with --schema v1"); + } JsonObject combinedJsonObject = new JsonObject(); Map symbolTable; @@ -399,9 +408,11 @@ private boolean isV2Schema() { * which {@code --artifact-text}/{@code --artifact-text-max-bytes} may have emptied or truncated — * delegates to {@link DependencyView#readFromDisk} rather than re-reading the file itself, so that * from-disk logic exists exactly once in this codebase. + * + * @param keyRoot the directory artifact paths are relative to (the workspace root, else --input) */ - private static String readFully(JArtifact artifact) { - return DependencyView.readFromDisk(Paths.get(input), artifact.getPath()); + private static String readFully(Path keyRoot, JArtifact artifact) { + return DependencyView.readFromDisk(keyRoot, artifact.getPath()); } /** @@ -455,10 +466,16 @@ private void analyzeV2() throws Exception { if (input == null) { throw new ParameterException(spec.commandLine(), "error: --input is required"); } - - String application = appName != null && !appName.isBlank() - ? appName - : Paths.get(input).toAbsolutePath().normalize().getFileName().toString(); + Path inputRoot = Paths.get(input).toAbsolutePath().normalize(); + Path keyRoot = inputRoot; + String application = spec.commandLine().getParseResult().hasMatchedOption("--app-name") + ? appName : inputRoot.getFileName().toString(); + String applicationId; + try { + applicationId = CanId.applicationId(application, namespace); + } catch (IllegalArgumentException e) { + throw new ParameterException(spec.commandLine(), "error: invalid --app-name or --namespace: " + e.getMessage()); + } // Always attempt library type resolution: without the dependency jars on the solver's path, // third-party types degrade to bare spellings (`Model` rather than `org.springframework.ui.Model`) @@ -485,7 +502,7 @@ private void analyzeV2() throws Exception { // L1 artifact that must not carry L3 overlays. So bypass the cache entirely at level >= 3. Map cached = (eager || analysisLevel >= 3) ? new java.util.LinkedHashMap<>() - : L1Cache.load(cache, application, version); + : L1Cache.load(cache, applicationId, version); Map modules; List rtaEndpoints = null; @@ -496,8 +513,8 @@ private void analyzeV2() throws Exception { // is what turns them into a non-zero exit for a caller that cannot read stderr. List degraded = new ArrayList<>(); try { - modules = L1Extractor.extractAll( - Paths.get(input), application, dependencyDir, cached, + modules = L1Extractor.extractAllForApplication( + Paths.get(input), applicationId, dependencyDir, cached, analysisLevel, graphFieldDepth, l3Engine, entrypointReport); // The WALA L3 engine needs the RTA build; --no-rta suppresses it, so a level-3 wala run with // --no-rta would silently carry no overlays. Warn rather than degrade without a signal. @@ -559,7 +576,7 @@ private void analyzeV2() throws Exception { // (an L2 refinement) must not be persisted into it. At level >= 3 the tree carries L3 overlays, // which are not an L1 artifact, so the cache is not written. if (analysisLevel < 3) { - L1Cache.save(cache, application, version, modules); + L1Cache.save(cache, applicationId, version, modules); } // The repository-artifact layer (build manifests, config files, declared dependencies) sits @@ -567,13 +584,13 @@ private void analyzeV2() throws Exception { // -- but unlike them it is L1 data and runs at EVERY analysis level, so it is computed here, // ahead of the level gate, rather than inside either branch of it. Map artifacts = - ArtifactDiscovery.discover(Paths.get(input), application, artifactText, artifactTextMaxBytes); - List dependencies = DependencyView.build(Paths.get(input), artifacts); + ArtifactDiscovery.discover(inputRoot, application, artifactText, artifactTextMaxBytes, namespace); + List dependencies = DependencyView.build(keyRoot, artifacts); for (JArtifact a : artifacts.values()) { if (ConfigKeys.isEligible(a)) { // Re-read from disk: `source` may be truncated or suppressed, and extraction // must not silently degrade with a capture flag. - ConfigKeys.Result r = ConfigKeys.extract(a, readFully(a), artifactText); + ConfigKeys.Result r = ConfigKeys.extract(a, readFully(keyRoot, a), artifactText); a.setConfigKeys(r.keys); // A pre-existing "partial" from the dependency pass is never overwritten. if (!r.ok) { @@ -589,7 +606,7 @@ private void analyzeV2() throws Exception { // engine-free, so they run whenever analysisLevel >= 4 regardless of the WALA build's fate. Analysis analysis; if (analysisLevel >= 2) { - L2CallGraph.Result l2 = L2CallGraph.build(application, modules, rtaEndpoints, externalCalls); + L2CallGraph.Result l2 = L2CallGraph.buildForApplication(applicationId, modules, rtaEndpoints, externalCalls); // SdgVertices needs the callee ids L2CallGraph.build just backfilled onto call body nodes, // and no dependency jars, so it runs here rather than inside the try block above. SdgVertices.Result sdg = null; @@ -604,10 +621,10 @@ private void analyzeV2() throws Exception { analysis = V2Emitter.emit(application, analysisLevel, modules, version, l2.callGraph(), l2.externalSymbols(), sdg == null ? null : sdg.paramIn, sdg == null ? null : sdg.paramOut, - artifacts, dependencies); + artifacts, dependencies, namespace); } else { analysis = V2Emitter.emit(application, analysisLevel, modules, version, - null, null, null, null, artifacts, dependencies); + null, null, null, null, artifacts, dependencies, namespace); } // Enforced here, after every overlay pass has had its chance and before anything is written: @@ -629,7 +646,7 @@ private void analyzeV2() throws Exception { // Config reads join the L1 tree to the artifact layer's declared keys, so it runs after both // exist and at every analysis level -- the literal tier needs no call graph. Absence means // "no fact", so an empty result leaves both keys off the envelope. - ConfigUses.Result configReads = ConfigUses.detect(application, modules, artifacts, + ConfigUses.Result configReads = ConfigUses.detectForApplication(applicationId, modules, artifacts, analysisLevel, analysis.getApplication().getCallGraph()); if (!configReads.uses.isEmpty()) { analysis.getApplication().setConfigUses(configReads.uses); @@ -640,7 +657,7 @@ private void analyzeV2() throws Exception { // View dispatches (#259) join dispatch sites to the artifact layer's view templates -- the // same shape as config reads, at every level, with the dataflow tiers widening from -a 3. - ViewDispatches.Result views = ViewDispatches.detect(application, modules, artifacts, + ViewDispatches.Result views = ViewDispatches.detectForApplication(applicationId, modules, artifacts, analysisLevel, analysis.getApplication().getCallGraph()); if (!views.dispatches.isEmpty()) { analysis.getApplication().setViewDispatches(views.dispatches); diff --git a/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java b/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java index 755977b4..565cc579 100644 --- a/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java +++ b/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java @@ -136,7 +136,15 @@ private static final class Rule { */ public static Map discover( Path projectDir, String appName, boolean captureText, int textMaxBytes) throws IOException { + return discover(projectDir, appName, captureText, textMaxBytes, null); + } + + /** Discover application-relative artifacts in the selected identity namespace. */ + public static Map discover( + Path projectDir, String appName, boolean captureText, int textMaxBytes, String namespace) + throws IOException { Path root = projectDir.toAbsolutePath().normalize(); + Path keyBase = projectDir.toAbsolutePath().normalize(); List files; try (Stream walk = Files.walk(root)) { files = walk.filter(Files::isRegularFile).sorted().collect(Collectors.toList()); @@ -172,9 +180,10 @@ public static Map discover( roles = (!name.contains(".") && text.startsWith("#!")) ? List.of("script") : List.of("unknown"); } + String keyPosix = keyBase.relativize(file).toString().replace('\\', '/'); JArtifact artifact = new JArtifact(); - artifact.setId(CanId.artifactId(appName, relPosix)); - artifact.setPath(relPosix); + artifact.setId(CanId.artifactId(appName, keyPosix, namespace)); + artifact.setPath(keyPosix); artifact.setFormat(format); artifact.setRoles(roles); // sha256/sizeBytes always describe the whole file, independent of capture/truncation @@ -192,7 +201,7 @@ public static Map discover( } } - artifacts.put(relPosix, artifact); + artifacts.put(keyPosix, artifact); } return artifacts; } diff --git a/src/main/java/com/ibm/cldk/artifacts/ConfigUses.java b/src/main/java/com/ibm/cldk/artifacts/ConfigUses.java index cca7b1ed..0f976b2a 100644 --- a/src/main/java/com/ibm/cldk/artifacts/ConfigUses.java +++ b/src/main/java/com/ibm/cldk/artifacts/ConfigUses.java @@ -188,6 +188,11 @@ public static Result detect(String appName, Map modules, */ public static Result detect(String appName, Map modules, Map artifacts, int analysisLevel, List callGraph) { + return detectForApplication(CanId.applicationId(appName), modules, artifacts, analysisLevel, callGraph); + } + + public static Result detectForApplication(String appName, Map modules, + Map artifacts, int analysisLevel, List callGraph) { Map> keysByNamespace = keysByNamespace(artifacts); List reads = new ArrayList<>(); @@ -291,7 +296,7 @@ private static void collectCallSite(String appName, String callableId, String lo ? node.getCalleeSignature() : node.getMethodName(); reads.add(new Read(CanId.ordinalId(callableId, localId), - CanId.externalId(appName, rule.declaringType, signature), + CanId.externalIdForApplication(appName, rule.declaringType, signature), literal, rule.namespaces, false, callableId, localId, keyName)); return; } @@ -486,6 +491,6 @@ private static String annotationMember(JDecorator decorator, String... names) { } private static String ghost(String appName, String annotationType) { - return CanId.externalId(appName, annotationType, "value()"); + return CanId.externalIdForApplication(appName, annotationType, "value()"); } } diff --git a/src/main/java/com/ibm/cldk/artifacts/ViewDispatches.java b/src/main/java/com/ibm/cldk/artifacts/ViewDispatches.java index b32711e2..ddde7a96 100644 --- a/src/main/java/com/ibm/cldk/artifacts/ViewDispatches.java +++ b/src/main/java/com/ibm/cldk/artifacts/ViewDispatches.java @@ -110,6 +110,11 @@ public static Result detect(String appName, Map modules, public static Result detect(String appName, Map modules, Map artifacts, int analysisLevel, List callGraph) { + return detectForApplication(CanId.applicationId(appName), modules, artifacts, analysisLevel, callGraph); + } + + public static Result detectForApplication(String appName, Map modules, + Map artifacts, int analysisLevel, List callGraph) { List sites = new ArrayList<>(); if (modules != null) { for (JModule module : modules.values()) { @@ -195,7 +200,7 @@ private static Site siteOf(String appName, JCallable callable, String localId, J } String signature = node.getCalleeSignature() != null ? node.getCalleeSignature() : method; return new Site(CanId.ordinalId(callableId, localId), - CanId.externalId(appName, receiver, signature), via, target, callableId, localId); + CanId.externalIdForApplication(appName, receiver, signature), via, target, callableId, localId); } /** diff --git a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java index 9ade4914..631fc659 100644 --- a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java @@ -44,6 +44,8 @@ *
  • on a FULL run only, prune units whose source file vanished.
    • * * + *

    A full run also reconciles the app's artifacts; see {@link #RELEASE_VANISHED_ARTIFACTS}. + * *

    Nodes are MERGE-upserted, never blindly deleted, so a declaration another (unchanged) unit * still references survives and its incoming edges stay valid. {@code :JPackage}/{@code :JAnnotation} * are shared (no {@code _module}) and are MERGE-only. @@ -74,28 +76,71 @@ public final class BoltWriter implements BoltSink { "MATCH (x:" + RowBuilder.CAN_NODE + ") WHERE (x.id = $mid OR x.id STARTS WITH $pre) " + "AND NOT x.id IN $keys DETACH DELETE x"; - /** - * The orphan prune: matches the application root by its {@code can://} {@code id} when it is a - * v2 root (exact, unique-constrained), OR by {@code name} when it is a legacy v1 root -- - * identifiable because v1 never writes {@code id} (see {@code GraphProjector.java}). An - * id-only match would orphan a prior v1 graph instead of pruning it; an ungated - * {@code a.name =} disjunct would reach every v2 root carrying that display name -- including - * roots this analyzer never minted -- since {@code name} stopped being unique-constrained once - * the root was re-keyed. Hence the {@code a.id IS NULL} guard, which confines the name branch - * to legacy roots. - * - *

    This does not separate two distinct applications sharing an {@code --app-name}: - * {@link com.ibm.cldk.schema.CanId#applicationId} is {@code "can://" + appName}, so they share - * an id byte-for-byte and are one node. Distinct services need distinct {@code --app-name} - * values; no predicate here can recover a distinction the id never encoded. - */ + /** Hashes and orphan pruning are confined to the selected canonical application anchor. */ + static final String MODULE_HASHES_V2 = "MATCH (a:JApplication {id: $appId})-[:J_HAS_MODULE]->(c:JModule) " + + "RETURN c.file_key AS k, c.content_hash AS h"; + static final String PRUNE_VANISHED_UNITS_V2 = "MATCH (a:JApplication)-[:J_HAS_UNIT|J_HAS_MODULE]->(c) " - + "WHERE (a.id = $appId OR (a.id IS NULL AND a.name = $appName)) " + + "WHERE a.id = $appId " + "AND NOT c.file_key IN $present " + "OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x) " + "DETACH DELETE x, c RETURN count(c) AS pruned"; + static final String PRUNE_VANISHED_UNITS_V1 = + "MATCH (a:JApplication)-[:J_HAS_UNIT]->(c) " + + "WHERE a.id IS NULL AND a.name = $appName " + + "AND NOT c.file_key IN $present " + + "OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x) " + + "DETACH DELETE x, c RETURN count(c) AS pruned"; + + static String orphanPruneQuery(GraphRows rows) { + return CypherWriter.legacyApplicationName(rows) != null + ? PRUNE_VANISHED_UNITS_V1 : PRUNE_VANISHED_UNITS_V2; + } + + /** + * Drop every Java-owned edge on this app's artifacts and config keys so the run rebuilds them: + * {@code HAS_ARTIFACT} from a {@code :JApplication}, {@code DECLARES_DEPENDENCY}/{@code LOCKS} + * to {@code pkg:maven/}, and incoming {@code J_} edges ({@code J_USES_CONFIG}, + * {@code J_DISPATCHES_TO}). Not sha-diffed: siblings overwrite the shared {@code sha256}, and a + * config change alters code edges while every module's hash stays put. + */ + static final String PURGE_ARTIFACT_EDGES = + "UNWIND $pres AS pre " + + "MATCH (t:" + RowBuilder.CAN_NODE + ") WHERE t.id STARTS WITH pre AND (t:Artifact OR t:ConfigKey) " + + "CALL { WITH t " + + "WITH [(t)<-[r]-(s) WHERE (type(r) = 'HAS_ARTIFACT' AND s:JApplication) " + + "OR type(r) STARTS WITH 'J_' | r] " + + "+ [(t)-[d:DECLARES_DEPENDENCY|LOCKS]->(p:Package) WHERE p.id STARTS WITH 'pkg:maven/' | d] AS rs " + + "FOREACH (r IN rs | DELETE r) } IN TRANSACTIONS OF 1000 ROWS"; + + /** @see #PURGE_ARTIFACT_EDGES */ + static final String PURGE_UNRESOLVED_CONFIG_READS = + "MATCH (:JApplication {id: $appId})-[r:J_READS_CONFIG_UNRESOLVED]->() " + + "CALL { WITH r DELETE r } IN TRANSACTIONS OF 1000 ROWS"; + + /** + * Release Java's claim ({@code :JCanNode}) on artifacts/config keys the run no longer emits, + * after {@link #PURGE_ARTIFACT_EDGES}. An artifact is deleted once no {@code HAS_ARTIFACT} is + * left: every analyzer inventories its artifacts with one, marked or not. A config key is + * deleted once no other analyzer's {@code *CanNode} claims it. + */ + static final String RELEASE_VANISHED_ARTIFACTS = + "UNWIND $pres AS pre " + + "MATCH (n:" + RowBuilder.CAN_NODE + ") WHERE n.id STARTS WITH pre AND (n:Artifact OR n:ConfigKey) " + + "AND NOT n.id IN $keep " + + "CALL { WITH n REMOVE n:" + RowBuilder.CAN_NODE + " " + + "WITH n WHERE (n:Artifact AND NOT ()-[:HAS_ARTIFACT]->(n)) " + + "OR (n:ConfigKey AND none(l IN labels(n) WHERE l ENDS WITH 'CanNode')) " + + "DETACH DELETE n } IN TRANSACTIONS OF 1000 ROWS " + + "RETURN count(*) AS released"; + + /** Only the selected application's artifacts; legacy output requires explicit retirement. */ + static List artifactPrefixes(String appId, String appName) { + return List.of(appId + "/artifact/"); + } + /** * The prefix that matches a module's declarations but not a sibling module whose path merely * starts the same way. The separator is the whole point: {@code can://app/java/src/Foo.java} @@ -124,6 +169,21 @@ private static String canModuleIdOf(List nodes) { private static final int BATCH = 1000; + static void requireCompatibleApplicationIdentity(String id, Map incoming, Map stored) { + Object name = incoming.get("application_name"); + Object namespace = incoming.get("identity_namespace"); + Object version = incoming.get("identity_version"); + Object storedVersion = stored.get("identity_version"); + if (!(name instanceof String) || !(namespace instanceof String) + || (!(version instanceof Integer) && !(version instanceof Long)) || ((Number) version).longValue() != 2 + || !id.equals(CanId.applicationId((String) name, (String) namespace)) + || (!(storedVersion instanceof Integer) && !(storedVersion instanceof Long)) || ((Number) storedVersion).longValue() != 2 + || !name.equals(stored.get("application_name")) + || !namespace.equals(stored.get("identity_namespace"))) { + throw new IllegalArgumentException("Incompatible existing application identity; regenerate graph explicitly"); + } + } + /** Public no-arg constructor: {@link Neo4jEmitter} instantiates this reflectively via {@link BoltSink}. */ public BoltWriter() {} @@ -150,6 +210,19 @@ private Session session() { } void run(GraphRows rows, boolean fullRun) { + // Validate the selected canonical anchor before any schema or data mutation. + for (NodeRow node : rows.nodes) { + if (!node.labels.contains("JApplication") || !RowBuilder.isCanId(node.value)) continue; + requireCompatibleApplicationIdentity(node.value, node.props, node.props); + try (Session s = session()) { + var records = s.run("MATCH (a:JApplication {id: $id}) RETURN a.identity_version AS identity_version, " + + "a.application_name AS application_name, a.identity_namespace AS identity_namespace", + Values.parameters("id", node.value)).list(); + for (var record : records) { + requireCompatibleApplicationIdentity(node.value, node.props, record.asMap()); + } + } + } // 1. schema (DDL runs in its own autocommit transactions). Migrations run FIRST: a // constraint an older release created on a property this generation no longer keys on // is still live and enforcing, and would fail the very first push (see Schema.MIGRATIONS). @@ -182,8 +255,9 @@ void run(GraphRows rows, boolean fullRun) { // :JCompilationUnit and v2 :JModule. Map dbHash = new HashMap<>(); try (Session s = session()) { - s.run("MATCH (c) WHERE c:JCompilationUnit OR c:JModule " - + "RETURN c.file_key AS k, c.content_hash AS h").list() + boolean v2 = rows.nodes.stream().anyMatch(n -> n.labels.contains("JApplication") && RowBuilder.isCanId(n.value)); + String query = v2 ? MODULE_HASHES_V2 : "MATCH (c:JCompilationUnit) RETURN c.file_key AS k, c.content_hash AS h"; + s.run(query, Values.parameters("appId", applicationIdOf(rows))).list() .forEach(rec -> dbHash.put(rec.get("k").asString(null), rec.get("h").asString(null))); } Set changed = new HashSet<>(); @@ -197,7 +271,10 @@ void run(GraphRows rows, boolean fullRun) { Log.info("neo4j(bolt): " + byUnit.size() + " units (" + changed.size() + " changed), " + shared.size() + " shared nodes, " + rows.edges.size() + " edges"); - // 3. shared nodes are always upserted (MERGE-only). + // 3. full run: reconcile this app's artifacts, then upsert shared nodes (MERGE-only). + if (fullRun) { + reconcileArtifacts(rows); + } upsertNodes(shared); // 4. per changed unit: purge owned edges + vanished decls, then upsert its nodes. @@ -236,44 +313,28 @@ void run(GraphRows rows, boolean fullRun) { upsertNodes(nodes); } - // 5. upsert edges owned by a changed unit (owner = source node's unit) or shared. + // 5. upsert edges owned by a changed unit (owner = source node's unit) or shared; on a + // full run also every edge into an artifact/config key, which step 3 purged. List edges = new ArrayList<>(); for (EdgeRow e : rows.edges) { String owner = unitOf.get(e.from.value); - if (owner == null || changed.contains(owner)) { + boolean intoArtifact = e.to.label.equals("Artifact") || e.to.label.equals("ConfigKey"); + if (owner == null || changed.contains(owner) || (fullRun && intoArtifact)) { edges.add(e); } } upsertEdges(edges); - // 6. orphan prune — only safe on a full run. Reaches both generations' per-file nodes - // through the shared application anchor, so a v2 push also prunes a prior v1 graph's - // units (and other applications in the database are never touched). + // 6. Full-run orphan pruning uses the actual incoming schema family. A v1 request + // selects only an id-less legacy anchor; v2 selects only its exact canonical anchor. if (fullRun) { List present = new ArrayList<>(byUnit.keySet()); String appName = appNameOf(rows); - String appId = CanId.applicationId(appName); - // Checked against the same hazard as the _module purges above (#213) and found - // sound, so deliberately left alone: this traversal never leaves java's own graph. - // It enters at :JApplication (java-owned, and scoped to this app by its unique - // can:// id when the root is v2, or by name when it is a legacy v1 root that never - // got an id -- see PRUNE_VANISHED_UNITS_V2's javadoc), hops a java-owned - // relationship type, and expands only through DESCENDANTS -- every member of which - // is J_-prefixed containment. `c` and `x` are unlabelled but unreachable except - // along those edges, and DESCENDANTS deliberately excludes HAS_ARTIFACT, so no - // cross-language :Artifact/:ConfigKey and no shared :JPackage/:JAnnotation can be - // reached. Do not add a non-containment type to DESCENDANTS without re-checking that. - // - // The snapshot path (CypherWriter.wipe) deliberately went the other way and now - // deletes this app's :Artifact/:ConfigKey via a `can:///` prefix sweep, so a - // full snapshot rebuilds them instead of letting them accumulate. That widening is - // NOT mirrored here, on purpose: this writer is incremental. Its unit of - // replacement is one module (descendantPrefix(moduleId)) or one vanished unit, and - // an app-level artifact belongs to neither -- an app-wide artifact sweep would - // delete artifacts a targeted `--target-files` run never re-pushes. The snapshot can - // afford the widening only because it always writes the full truth back. + String appId = applicationIdOf(rows); + // Only Java containment under this exact anchor is traversed. Shared artifacts + // are reconciled separately and sibling namespaces are never fallback matches. try (Session s = session()) { - long pruned = s.run(PRUNE_VANISHED_UNITS_V2, + long pruned = s.run(orphanPruneQuery(rows), Values.parameters("present", present, "appId", appId, "appName", appName)) .single().get("pruned").asLong(0); Log.info("neo4j(bolt): pruned " + pruned + " vanished unit(s)"); @@ -283,6 +344,35 @@ void run(GraphRows rows, boolean fullRun) { } } + /** v2 only: v1 emits no artifacts, so it would release every one. */ + private void reconcileArtifacts(GraphRows rows) { + String appId = null; + List keep = new ArrayList<>(); + for (NodeRow n : rows.nodes) { + String label = n.labels.get(0); + if (label.equals("JApplication") && RowBuilder.isCanId(n.value)) { + appId = n.value; + } else if (label.equals("Artifact") || label.equals("ConfigKey")) { + keep.add(n.value); + } + } + if (appId == null) { + return; + } + Map args = new LinkedHashMap<>(); + args.put("appId", appId); + args.put("pres", artifactPrefixes(appId, appNameOf(rows))); + args.put("keep", keep); + // Auto-commit and batched to bound full-run artifact reconciliation. + // A partial failure is safe; the next full push finishes it. + try (Session s = session()) { + s.run(PURGE_ARTIFACT_EDGES, args).consume(); + s.run(PURGE_UNRESOLVED_CONFIG_READS, args).consume(); + long released = s.run(RELEASE_VANISHED_ARTIFACTS, args).single().get("released").asLong(0); + Log.info("neo4j(bolt): released " + released + " vanished artifact/config node(s)"); + } + } + private void upsertNodes(List nodes) { Map> groups = new LinkedHashMap<>(); for (NodeRow n : nodes) { @@ -362,6 +452,13 @@ private static String hashOf(List nodes, String fileKey) { * identically by both generations, so this always yields the true app name to re-derive * the id from. */ + private static String applicationIdOf(GraphRows rows) { + for (NodeRow n : rows.nodes) { + if (n.labels.contains("JApplication") && RowBuilder.isCanId(n.value)) return n.value; + } + return CanId.applicationId(appNameOf(rows)); + } + private static String appNameOf(GraphRows rows) { for (NodeRow n : rows.nodes) { if (n.labels.get(0).equals("JApplication")) { diff --git a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java index 1cb55f64..2e469aa8 100644 --- a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java @@ -34,29 +34,7 @@ public final class CypherWriter { private static final int BATCH = 500; - /** - * Every containment relationship either graph generation emits — v1 (unit-rooted) and v2 - * (module-rooted) together, so a v2 push wipes/prunes a prior v1 graph of the same app and vice - * versa (spec: one app name = one graph, latest push wins). - * - *

    This traversal is the only thing that reaches a v1 graph, and that is why - * {@link #wipe}'s {@code can://} prefix sweep cannot replace it: v1 ids are fully-qualified - * names carrying no application segment, and {@code GraphProjector.java} keys - * {@code :JApplication} on {@code name}, so no prefix predicate sees a single v1 node. Folding - * the two into one prefix sweep was tried on this branch and reverted; it orphans every v1 - * graph. - * - *

    {@code HAS_ARTIFACT} (and the {@code :Package} edges {@code DECLARES_DEPENDENCY} / - * {@code LOCKS}) stay out of this list — but no longer because the artifact subtree is - * untouchable. {@link #wipe}'s prefix sweep now does reach this application's - * {@code :Artifact}/{@code :ConfigKey} nodes, deliberately; see its javadoc. They stay out of - * {@code DESCENDANTS} because this constant is shared with - * {@link BoltWriter#PRUNE_VANISHED_UNITS_V2}, whose unit of deletion is one vanished - * compilation unit, not the application: an app-level {@code :Artifact} is not a descendant of - * any one unit and must not die with one. {@code :Package} ({@code pkg:} purls) is outside both - * — it is not under the app prefix either — as are the shared {@code :JPackage}/ - * {@code :JAnnotation} merge targets. - */ + /** Java containment relationships; traversed only below the selected application anchor. */ static final String DESCENDANTS = "[:J_DECLARES_TYPE|J_HAS_NESTED_TYPE|J_HAS_CALLABLE|J_HAS_FIELD|J_HAS_PARAMETER" + "|J_HAS_CALLSITE|J_DECLARES_VAR|J_HAS_ENUM_CONSTANT|J_HAS_RECORD_COMPONENT|J_HAS_INIT_BLOCK" + "|J_HAS_CRUD_OPERATION|J_HAS_CRUD_QUERY|J_HAS_COMMENT" @@ -114,7 +92,8 @@ public static void writeCypher(Appendable out, GraphRows rows, String appName) t s.add(""); s.add("// ── wipe this project's prior subgraph (packages/annotations are shared and survive; " + "artifacts/config keys are inside the app prefix and get rebuilt) ──"); - s.add(wipe(appName)); + String legacyName = legacyApplicationName(rows); + s.add(legacyName != null ? wipeLegacy(legacyName) : wipe(applicationIdOf(rows, appName))); s.add(""); s.add("// ── nodes ──"); @@ -165,58 +144,41 @@ static String applicationPrefix(String appName) { return CanId.applicationId(appName) + "/"; } - /** - * The application root by equality plus everything under {@code can:///}, then the same - * application's prior v1 graph by containment, then the orphan sweep. - * - *

    Statement 1, the prefix sweep. Scoped by id, so it is one application by - * construction: a second java app sharing a source path, and a sibling analyzer's own - * {@code can://} graph under a different app, are outside it. {@code :Package} nodes - * ({@code pkg:} purls) stay outside too — they are not under the app prefix. Two things moved - * with the app-outermost grammar: the root is matched by its {@code can://} id rather than - * by the free-text {@code --app-name}, so the id the graph is actually keyed on is the id the - * wipe uses; and this application's {@code :Artifact}/{@code :ConfigKey} nodes are now - * inside the prefix, so the snapshot rebuilds them instead of leaving them to accumulate - * forever with nothing ever cleaning them up. That is deliberate, and it is the one behavioural - * widening here: a cross-language edge into a shared {@code :Artifact} is dropped by a java - * snapshot and restored on that analyzer's next push. - * - *

    Statement 2, the containment traversal. Not redundant with the sweep above and not - * foldable into it — see {@link #DESCENDANTS}. v1 ids are fully-qualified names, so the prefix - * predicate cannot reach a v1 node; this hop is what makes "one app name = one graph, latest - * push wins" hold across generations. The unit hop is unlabeled and lists both generations' rel - * types (v1 {@code J_HAS_UNIT} → {@code :JCompilationUnit}, v2 {@code J_HAS_MODULE} → - * {@code :JModule}) so either generation's push replaces whichever the DB currently holds. The - * root is matched by id when it is v2 (exact, unique-constrained) OR by name when it is a legacy - * v1 root — identifiable because v1 never writes {@code id}. An id-only match would orphan a - * prior v1 graph instead of replacing it, breaking the very cross-generation guarantee this - * exists for. The {@code a.id IS NULL} guard confines the name branch to those legacy roots; an - * ungated {@code a.name =} disjunct would additionally reach every v2 root carrying this display - * name, including roots this analyzer never minted, so do not widen it. - * - *

    Statement 3, the orphan sweep. Fully-isolated {@code :JSymbol} nodes the containment - * traversal cannot reach — v1's import-materialized bodyless {@code :JType} stubs hang off units - * via {@code J_IMPORTS} only, so the deletes above orphan them. Degree-0 symbols are - * unreferencable junk in any generation, and a symbol another application still uses keeps its - * edges and survives. - * - *

    What none of this does — and must not be documented as doing — is separate two distinct - * applications sharing an {@code --app-name}. {@code CanId.applicationId} is - * {@code "can://" + appName}, so they share an id byte-for-byte and are ONE node; there is - * nothing here to tell apart. Distinct services need distinct {@code --app-name} values, not a - * wider predicate. - */ - private static String wipe(String appName) { - String appId = cypherValue(CanId.applicationId(appName)); + /** Actual name-keyed v1 rows select legacy cleanup; v2 never falls back by display name. */ + static String legacyApplicationName(GraphRows rows) { + for (NodeRow node : rows.nodes) { + if (node.labels.contains("JApplication") && "name".equals(node.keyProp)) { + return node.value; + } + } + return null; + } + + private static String wipeLegacy(String appName) { + return "MATCH (a:JApplication) WHERE a.id IS NULL AND a.name = " + cypherValue(appName) + "\n" + + "OPTIONAL MATCH (a)-[:J_HAS_UNIT]->(c)\n" + + "OPTIONAL MATCH (c)-" + DESCENDANTS + "->(x)\n" + + "DETACH DELETE x, c, a;\n"; + } + + /** Canonical roots come from projected rows, preserving the selected namespace. */ + private static String applicationIdOf(GraphRows rows, String appName) { + for (NodeRow n : rows.nodes) { + if (n.labels.contains("JApplication") && RowBuilder.isCanId(n.value)) return n.value; + } + return CanId.applicationId(appName); + } + + private static String wipe(String applicationId) { + String appId = cypherValue(applicationId); return "MATCH (x:" + RowBuilder.CAN_NODE + ") WHERE x.id = " + appId - + " OR x.id STARTS WITH " + cypherValue(applicationPrefix(appName)) + "\n" + + " OR x.id STARTS WITH " + cypherValue(applicationId + "/") + "\n" + "CALL { WITH x DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS;\n" + "MATCH (a:JApplication) WHERE a.id = " + appId - + " OR (a.id IS NULL AND a.name = " + cypherValue(appName) + ")\n" + + "\n" + "OPTIONAL MATCH (a)-[:J_HAS_UNIT|J_HAS_MODULE]->(c)\n" + "OPTIONAL MATCH (c)-" + DESCENDANTS + "->(x)\n" - + "DETACH DELETE x, c, a;\n" - + "MATCH (s:JSymbol) WHERE NOT (s)--() DELETE s;"; + + "DETACH DELETE x, c, a;\n"; } // ---------------------------------------------------------------------------------------------- diff --git a/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java b/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java index 9f39bfb9..92d0eeae 100644 --- a/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java +++ b/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java @@ -28,7 +28,9 @@ * a target that never materialized are dropped (the "edge-only-when-resolved" rule). * *

    This is the in-memory analog of {@code MERGE (n:Label {key}) SET n += props}: re-seeing the - * same (mergeLabel, value) merges props (last write wins) and unions labels. + * same canonical ID merges compatible properties and unions labels. A differing value for any + * existing node property, including an ordinary display property, is rejected rather than refined. + * Noncanonical rows retain their label/key identity. */ public final class RowBuilder { @@ -67,13 +69,24 @@ public static Map prune(Map in) { } /** - * Upsert a node. Re-seeing the same {@code (labels[0], value)} merges props (last write wins) - * and unions labels. + * Upsert a node. Canonical ids deduplicate across labels, unioning labels and compatible + * properties. Conflicting canonical properties are rejected. Legacy keys retain label scope. */ public NodeRef node(List labels, String keyProp, String value, Map props) { - String id = labels.get(0) + " " + value; + String id = isCanId(value) ? value : labels.get(0) + " " + value; NodeRow existing = nodes.get(id); if (existing != null) { + if (isCanId(value)) { + if (!existing.keyProp.equals(keyProp)) { + throw new IllegalArgumentException("Conflicting canonical entity key: " + value); + } + for (var entry : props.entrySet()) { + if (existing.props.containsKey(entry.getKey()) + && !java.util.Objects.equals(existing.props.get(entry.getKey()), entry.getValue())) { + throw new IllegalArgumentException("Conflicting canonical entity property " + entry.getKey() + ": " + value); + } + } + } existing.props.putAll(props); for (String l : labels) { if (!existing.labels.contains(l)) { @@ -165,6 +178,10 @@ public void edgeIfBothResolved(String type, NodeRef from, NodeRef to, Map(b) SET r += p`). Map uniqueEdges = new LinkedHashMap<>(); for (EdgeRow e : edges) { - uniqueEdges.put(e.type + "|" + e.from.label + ":" + e.from.value - + "|" + e.to.label + ":" + e.to.value + uniqueEdges.put(e.type + "|" + endpointKey(e.from) + + "|" + endpointKey(e.to) + (e.key == null ? "" : "|" + e.key), e); } diff --git a/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java b/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java index ecfa9bfd..22094a72 100644 --- a/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java +++ b/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java @@ -73,6 +73,15 @@ private V2GraphProjector() {} private static final List SYMBOL_EXTERNAL = Arrays.asList("JSymbol", "JExternal"); public static GraphRows project(Analysis analysis, String appName) { + if (!Integer.valueOf(2).equals(analysis.getIdentityVersion())) { + throw new IllegalArgumentException("Unsupported identity_version; regenerate analysis from source"); + } + String rawName = analysis.getApplication().getApplicationName(); + String namespace = analysis.getApplication().getIdentityNamespace(); + if (rawName == null || namespace == null + || !CanId.applicationId(rawName, namespace).equals(analysis.getApplication().getId())) { + throw new IllegalArgumentException("Missing or incompatible application identity; regenerate from source"); + } RowBuilder b = new RowBuilder(); Map symbolTable = analysis.getApplication().getSymbolTable(); @@ -83,7 +92,10 @@ public static GraphRows project(Analysis analysis, String appName) { "application.id is null -- the can:// root id must be stamped by V2Emitter before projection"); Map appProps = RowBuilder.props(); appProps.put("id", appId); - appProps.put("name", appName); + appProps.put("name", rawName); + appProps.put("application_name", rawName); + appProps.put("identity_namespace", namespace); + appProps.put("identity_version", 2); appProps.put("schema_version", V2SchemaCatalog.SCHEMA_VERSION); if (analysis.getAnalyzer() != null) { appProps.put("analyzer_name", analysis.getAnalyzer().getName()); diff --git a/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java b/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java index f863d83a..75f59065 100644 --- a/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java +++ b/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java @@ -129,7 +129,8 @@ private static List buildNodeLabels() { // one reading the same is the failure this record exists to prevent. Mirrors what // codeanalyzer-python projects onto :PyApplication. n.add(node("JApplication", "JApplication", "id", - new P().put("id", "string").put("name", "string").put("schema_version", "string") + new P().put("id", "string").put("name", "string").put("application_name", "string").put("identity_namespace", "string") + .put("identity_version", "integer").put("schema_version", "string") .put("analyzer_name", "string").put("analyzer_version", "string") .put("entrypoint_frameworks", "string[]") .put("entrypoint_report_json", "string").done())); diff --git a/src/main/java/com/ibm/cldk/schema/Analysis.java b/src/main/java/com/ibm/cldk/schema/Analysis.java index dd3aaa40..d7716d5f 100644 --- a/src/main/java/com/ibm/cldk/schema/Analysis.java +++ b/src/main/java/com/ibm/cldk/schema/Analysis.java @@ -9,6 +9,7 @@ */ @Data public class Analysis { + private Integer identityVersion; private String schemaVersion; private String language; private int maxLevel; diff --git a/src/main/java/com/ibm/cldk/schema/CanId.java b/src/main/java/com/ibm/cldk/schema/CanId.java index e29acfdd..33f570ab 100644 --- a/src/main/java/com/ibm/cldk/schema/CanId.java +++ b/src/main/java/com/ibm/cldk/schema/CanId.java @@ -4,7 +4,7 @@ * Canonical {@code can://} id construction for schema v2. * *

    Durable ids (≥ callable) are containment paths - * {@code can:///java///}; ordinal ids (< callable) are + * {@code can:////java///}; ordinal ids (< callable) are * {@code @} where {@code } is a source position {@code line:col} (real * nodes) or a synthetic tag (e.g. {@code entry}). Pure functions; ids are opaque handles (the * {@code } segment itself may contain {@code /}). @@ -19,9 +19,44 @@ private CanId() {} /** This analyzer's language segment, which now sits INSIDE the app rather than above it. */ public static final String LANG = "java"; - /** {@code can://} — the application root, and the prefix every id below it shares. */ + /** {@code can:///} — the application root, and the prefix every id below it shares. */ public static String applicationId(String appName) { - return SCHEME + appName; + return applicationId(appName, null); + } + + public static String applicationId(String appName, String namespace) { + return SCHEME + encodeName(namespace == null ? appName : namespace) + "/" + encodeName(appName); + } + + private static String encodeName(String value) { + if (value == null || value.isEmpty() || value.equals(".") || value.equals("..")) { + throw new IllegalArgumentException("Invalid identity name"); + } + boolean whitespace = true; + for (int i = 0; i < value.length();) { + int cp = value.codePointAt(i); + if (Character.isISOControl(cp) || cp == '/' || cp == '\\' + || cp >= 0xD800 && cp <= 0xDFFF) { + throw new IllegalArgumentException("Invalid identity name"); + } + whitespace &= Character.isWhitespace(cp) || Character.isSpaceChar(cp); + i += Character.charCount(cp); + } + if (whitespace) { + throw new IllegalArgumentException("Identity name must not be whitespace-only"); + } + StringBuilder out = new StringBuilder(); + for (byte b : value.getBytes(java.nio.charset.StandardCharsets.UTF_8)) { + int c = b & 255; + if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' + || c == '-' || c == '.' || c == '_' || c == '~') { + out.append((char) c); + } else { + out.append('%').append("0123456789ABCDEF".charAt(c >>> 4)) + .append("0123456789ABCDEF".charAt(c & 15)); + } + } + return out.toString(); } /** {@code /java/} (separators normalized to {@code /}). */ @@ -41,7 +76,7 @@ public static String ordinalId(String callableId, String tag) { } /** - * {@code can:///@external//} — a callable outside the project. The + * {@code can:////@external//} — a callable outside the project. The * type is a binary name ({@code java.util.Map$Entry}) so the id is unambiguous and joins * WALA natively. * @@ -53,23 +88,35 @@ public static String ordinalId(String callableId, String tag) { * codeanalyzer-python follows it, and so does this. * *

    Consequently an external id does not match the {@code canId} shape - * ({@code ^can://[^/]+/java/}): the analysis schema's {@code externalCanId} is a separate + * ({@code ^can://[^/]+/[^/]+/java/}): the analysis schema's {@code externalCanId} is a separate * definition, and the two sites that hold either — a call edge's {@code dst} and a body node's * {@code callee} — {@code $ref} the union, not a widened {@code canId}. */ public static String externalId(String appName, String binaryType, String signature) { - return applicationId(appName) + "/@external/" + binaryType + "/" + signature; + return externalId(appName, binaryType, signature, null); + } + + public static String externalId(String appName, String binaryType, String signature, String namespace) { + return externalIdForApplication(applicationId(appName, namespace), binaryType, signature); + } + + public static String externalIdForApplication(String applicationId, String binaryType, String signature) { + return applicationId + "/@external/" + binaryType + "/" + signature; } /** - * {@code can:///artifact/} — a language-neutral artifact id, nested under the app + * {@code can:////artifact/} — a language-neutral artifact id, nested under the app * rather than under a language. The {@code artifact} segment is deliberately chosen over * {@code java} so a sibling-language analyzer scanning the same repository lands on the same * node rather than a duplicate. {@code CypherWriter.DESCENDANTS} cites that sharing by name as * the reason the artifact/config-key subtree stays outside every wipe. */ public static String artifactId(String appName, String relPath) { - return applicationId(appName) + "/artifact/" + relPath; + return artifactId(appName, relPath, null); + } + + public static String artifactId(String appName, String relPath, String namespace) { + return applicationId(appName, namespace) + "/artifact/" + relPath; } /** diff --git a/src/main/java/com/ibm/cldk/schema/JApplication.java b/src/main/java/com/ibm/cldk/schema/JApplication.java index ceb1d491..85b61aa6 100644 --- a/src/main/java/com/ibm/cldk/schema/JApplication.java +++ b/src/main/java/com/ibm/cldk/schema/JApplication.java @@ -9,6 +9,8 @@ @Data public class JApplication { private String id; + private String applicationName; + private String identityNamespace; private String kind = "application"; private Map symbolTable = new LinkedHashMap<>(); diff --git a/src/main/java/com/ibm/cldk/schema/V2Emitter.java b/src/main/java/com/ibm/cldk/schema/V2Emitter.java index 34401be1..df6af7ae 100644 --- a/src/main/java/com/ibm/cldk/schema/V2Emitter.java +++ b/src/main/java/com/ibm/cldk/schema/V2Emitter.java @@ -72,8 +72,18 @@ public static Analysis emit( List paramOut, Map artifacts, List dependencies) { + return emit(appName, maxLevel, modules, analyzerVersion, callGraph, externalSymbols, + paramIn, paramOut, artifacts, dependencies, null); + } + + public static Analysis emit(String appName, int maxLevel, Map modules, + String analyzerVersion, List callGraph, Map externalSymbols, + List paramIn, List paramOut, Map artifacts, + List dependencies, String namespace) { JApplication application = new JApplication(); - application.setId(CanId.applicationId(appName)); + application.setId(CanId.applicationId(appName, namespace)); + application.setApplicationName(appName); + application.setIdentityNamespace(namespace == null ? appName : namespace); // Sort file keys so output is deterministic (the -j gate). Map sorted = new LinkedHashMap<>(); @@ -103,6 +113,7 @@ public static Analysis emit( } Analysis analysis = new Analysis(); + analysis.setIdentityVersion(2); analysis.setSchemaVersion("2.0.0"); analysis.setLanguage("java"); analysis.setMaxLevel(maxLevel); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java index 8e7eb498..68c51ace 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java @@ -34,6 +34,7 @@ private L1Cache() {} /** What is persisted: the modules plus the settings they were built under. */ @Data static class Envelope { + private Integer identityVersion; private String schemaVersion; private String analyzerVersion; private String appName; @@ -62,7 +63,7 @@ public static Map load(Path cacheDir, String appName, String an if (envelope == null || envelope.getModules() == null) { return new LinkedHashMap<>(); } - boolean sameSettings = "2.0.0".equals(envelope.getSchemaVersion()) + boolean sameSettings = Integer.valueOf(2).equals(envelope.getIdentityVersion()) && "2.0.0".equals(envelope.getSchemaVersion()) && java.util.Objects.equals(appName, envelope.getAppName()) && java.util.Objects.equals(analyzerVersion, envelope.getAnalyzerVersion()); if (!sameSettings) { @@ -83,6 +84,7 @@ public static void save( return; } Envelope envelope = new Envelope(); + envelope.setIdentityVersion(2); envelope.setSchemaVersion("2.0.0"); envelope.setAnalyzerVersion(analyzerVersion); envelope.setAppName(appName); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java index ecf72e8e..975d901c 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java @@ -121,6 +121,15 @@ public static Map extractAll( Path projectRoot, String appName, Path dependencyDir, Map cached, int analysisLevel, int graphFieldDepth, String l3Engine, JEntrypointReport entrypointReport) throws IOException { + return extractAllForApplication(projectRoot, CanId.applicationId(appName), dependencyDir, cached, analysisLevel, + graphFieldDepth, l3Engine, entrypointReport); + } + + /** Extract application-relative modules under an already validated canonical application root. */ + public static Map extractAllForApplication( + Path projectRoot, String applicationId, Path dependencyDir, Map cached, + int analysisLevel, int graphFieldDepth, String l3Engine, JEntrypointReport entrypointReport) + throws IOException { // JavaParser's collection strategy silently finds zero source roots for a path with a `.` // element (`-i .` arrives here as exactly that), so normalize before discovery. projectRoot = projectRoot.toAbsolutePath().normalize(); @@ -140,7 +149,6 @@ public static Map extractAll( // Collect into a sorted map first: directory listings are not ordered, and output must not // depend on traversal order. Map modules = new TreeMap<>(); - String applicationId = CanId.applicationId(appName); JavaParser parser = new JavaParser(config); int reused = 0; for (SourceRoot sourceRoot : sourceRoots) { @@ -249,9 +257,9 @@ private static CombinedTypeSolver typeSolver( return solver; } - /** The {@code symbol_table} key: path relative to the project root, always {@code /}-separated. */ - private static String fileKey(Path projectRoot, Path file) { - Path relative = projectRoot.toAbsolutePath().normalize().relativize(file.toAbsolutePath().normalize()); + /** The {@code symbol_table} key: path relative to the key root, always {@code /}-separated. */ + private static String fileKey(Path keyRoot, Path file) { + Path relative = keyRoot.toAbsolutePath().normalize().relativize(file.toAbsolutePath().normalize()); return relative.toString().replace('\\', '/'); } diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L2CallGraph.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L2CallGraph.java index c1f2644d..135535c0 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L2CallGraph.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L2CallGraph.java @@ -133,6 +133,10 @@ public static Result build(String appName, Map modules, List modules, List rtaEdges, boolean includeExternal) { + return buildForApplication(CanId.applicationId(appName), modules, rtaEdges, includeExternal); + } + + public static Result buildForApplication(String appName, Map modules, List rtaEdges, boolean includeExternal) { // The type index (named types only) and the set of every callable id in the tree. The latter // turns no-dangling, the signature-miss case, and the RTA drop rules into O(1) membership checks. Map typeIdByBinaryName = new HashMap<>(); @@ -279,7 +283,7 @@ private static String resolveCallee( if (!includeExternal) { return null; // case 4 gated off: match v1's application-only graph } - String externalId = CanId.externalId(appName, hint, signature); // case 4 + String externalId = CanId.externalIdForApplication(appName, hint, signature); // case 4 externalSymbols.computeIfAbsent(externalId, k -> externalSymbol(signature, hint, node.isConstructorCall())); return externalId; } @@ -314,7 +318,7 @@ private static void joinRtaEdge( if (!includeExternal) { return; // external target gated off: match v1's application-only graph } - dst = CanId.externalId(appName, edge.dstType, edge.dstSignature); + dst = CanId.externalIdForApplication(appName, edge.dstType, edge.dstSignature); externalSymbols.computeIfAbsent( dst, k -> externalSymbol(edge.dstSignature, edge.dstType, edge.dstSignature.startsWith(""))); } diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerIntegrationTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerIntegrationTest.java index cd933cc4..1fe38caf 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerIntegrationTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerIntegrationTest.java @@ -134,7 +134,7 @@ void callGraphShouldHaveKnownEdges() throws Exception { var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/call-graph-test --analysis-level=2", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/call-graph-test --analysis-level=2", javaHomePath, codeanalyzerVersion ) ); @@ -169,7 +169,7 @@ void corruptMavenShouldProduceAnalysisArtifactsWhenMVNCommandIsInPath() throws I var corruptMavenProjectBuild = mavenContainer.withWorkingDirectory("/test-applications/mvnw-corrupt-test").execInContainer("mvn", "-f", "/test-applications/mvnw-corrupt-test/pom.xml", "clean", "compile"); Assertions.assertEquals(0, corruptMavenProjectBuild.getExitCode(), "Failed to build the project with system's default Maven."); // NOw run codeanalyzer and assert if analysis.json is generated. - var runCodeAnalyzer = mavenContainer.execInContainer("java", "-jar", String.format("/opt/jars/codeanalyzer-%s.jar", codeanalyzerVersion), "--input=/test-applications/mvnw-corrupt-test", "--output=/tmp/", "--analysis-level=2", "--verbose", "--no-build"); + var runCodeAnalyzer = mavenContainer.execInContainer("java", "-jar", String.format("/opt/jars/codeanalyzer-%s.jar", codeanalyzerVersion), "--schema", "v1", "--input=/test-applications/mvnw-corrupt-test", "--output=/tmp/", "--analysis-level=2", "--verbose", "--no-build"); var codeAnalyzerOutputDirContents = mavenContainer.execInContainer("ls", "/tmp/analysis.json"); String codeAnalyzerOutputDirContentsStdOut = codeAnalyzerOutputDirContents.getStdout(); Assertions.assertTrue(codeAnalyzerOutputDirContentsStdOut.length() > 0, "Could not find 'analysis.json'."); @@ -185,7 +185,7 @@ void corruptMavenShouldNotTerminateWithErrorWhenMavenIsNotPresentUnlessAnalysisL var runCodeAnalyzer = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/mvnw-corrupt-test --output=/tmp/ --analysis-level=2", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/mvnw-corrupt-test --output=/tmp/ --analysis-level=2", javaHomePath, codeanalyzerVersion ) ); @@ -199,7 +199,7 @@ void shouldBeAbleToGenerateAnalysisArtifactForDaytrader8() throws Exception { var runCodeAnalyzerOnDaytrader8 = mavenContainer.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/daytrader8 --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/daytrader8 --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -214,7 +214,7 @@ void shouldBeAbleToDetectCRUDOperationsAndQueriesForPlantByWebsphere() throws Ex var runCodeAnalyzerOnPlantsByWebsphere = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/plantsbywebsphere --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/plantsbywebsphere --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -307,7 +307,7 @@ void symbolTableShouldHaveRecords() throws IOException, InterruptedException { var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/record-class-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/record-class-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -324,7 +324,7 @@ void symbolTableShouldHaveDefaultRecordComponents() throws IOException, Interrup var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/record-class-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/record-class-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -354,7 +354,7 @@ void parametersInCallableMustHaveStartAndEndLineAndColumns() throws IOException, var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/record-class-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/record-class-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -389,7 +389,7 @@ void mustBeAbleToResolveInitializationBlocks() throws IOException, InterruptedEx var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/init-blocks-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/init-blocks-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -420,7 +420,7 @@ void mustBeAbleToExtractCommentBlocks() throws IOException, InterruptedException var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/init-blocks-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/init-blocks-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java index 8a82ff18..eddbe4dc 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java @@ -791,4 +791,190 @@ void viewDispatchesReachBothProjections(@TempDir Path root) throws Exception { assertEquals(1, rows, "one J_DISPATCHES_TO row: the resolved forward, and not the unresolved one"); assertTrue(script.contains("via: 'forward'"), "the row carries its mechanism"); } + + // ---- --workspace-root ----------------------------------------------------------------------- + + /** Every key in {@code keys} is repo-relative under {@code prefix}: never absolute, never {@code ..}. */ + private static void assertRepoRelative(Set keys, String prefix) { + assertFalse(keys.isEmpty(), "precondition: something was emitted"); + for (String key : keys) { + assertTrue(key.startsWith(prefix), "expected the " + prefix + " prefix, got: " + key); + assertFalse(key.startsWith("/") || key.contains(".."), "keys stay relative: " + key); + } + } + + @Test + void namespaceKeepsSymbolTableKeysApplicationRelative(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path in = project(repo.resolve("services/app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "-a", "2", + "--namespace", "team", "--app-name", "widgets")); + + JsonObject app = application(out); + assertEquals(CanId.applicationId("widgets", "team"), app.get("id").getAsString(), + "the app segment does not move with the workspace root"); + JsonObject symbolTable = app.getAsJsonObject("symbol_table"); + assertRepoRelative(symbolTable.keySet(), "src/"); + String key = "src/main/java/com/example/Widget.java"; + assertTrue(symbolTable.has(key), "got: " + symbolTable.keySet()); + assertEquals(CanId.moduleId(CanId.applicationId("widgets", "team"), key), + symbolTable.getAsJsonObject(key).get("id").getAsString()); + + String moduleId = CanId.moduleId(CanId.applicationId("widgets", "team"), key); + JsonArray callGraph = app.getAsJsonArray("call_graph"); + assertFalse(callGraph.isEmpty(), "precondition: size() -> count() is an in-project edge"); + for (JsonElement e : callGraph) { + assertTrue(e.getAsJsonObject().get("src").getAsString().startsWith(moduleId + "/"), + "call-graph ids carry the repo-relative module id: " + e); + } + assertConformsToV2Schema(out); + } + + @Test + void namespaceCarriesIntoLevelThreeOverlayIds(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path in = project(repo.resolve("services/app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "-a", "3", + "--namespace", "team", "--app-name", "widgets")); + + String moduleId = CanId.moduleId(CanId.applicationId("widgets", "team"), + "src/main/java/com/example/Widget.java"); + String json = Files.readString(out.resolve("analysis.json")); + assertTrue(json.contains("\"cfg\""), "precondition: level 3 lays the cfg overlay"); + java.util.regex.Matcher ids = java.util.regex.Pattern.compile("\"(can://team/widgets/java/[^\"]*)\"").matcher(json); + int seen = 0; + while (ids.find()) { + assertTrue(ids.group(1).startsWith(moduleId), + "every code id, overlay endpoints included, is under the repo-relative module: " + ids.group(1)); + seen++; + } + assertTrue(seen > 0, "precondition: ids were emitted"); + assertConformsToV2Schema(out); + } + + @Test + void namespaceKeepsArtifactKeysAndClassification(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path in = artifactProject(repo.resolve("services/app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), + "--namespace", "team", "--app-name", "widgets")); + + JsonObject app = application(out); + JsonObject artifacts = app.getAsJsonObject("artifacts"); + assertRepoRelative(artifacts.keySet(), ""); + + JsonObject pom = artifacts.getAsJsonObject("pom.xml"); + assertNotNull(pom, "got: " + artifacts.keySet()); + assertEquals("pom.xml", pom.get("path").getAsString()); + assertEquals(CanId.artifactId("widgets", "pom.xml", "team"), pom.get("id").getAsString()); + assertEquals("full", pom.get("extraction").getAsString(), + "the manifest is still read from disk under the repo-relative key"); + assertFalse(app.getAsJsonArray("dependencies").isEmpty(), "and its dependency still extracted"); + + JsonArray propsKeys = artifacts.getAsJsonObject("application.properties") + .getAsJsonArray("config_keys"); + assertEquals(1, propsKeys.size(), "config keys are still extracted"); + assertConformsToV2Schema(out); + } + + @Test + void explicitDefaultNamespaceChangesNothing(@TempDir Path tmp) throws IOException { + Path in = artifactProject(tmp.resolve("app")); + Path without = tmp.resolve("without"); + Path with = tmp.resolve("with"); + assertEquals(0, run("-i", in.toString(), "-o", without.toString(), "--app-name", "widgets")); + assertEquals(0, run("-i", in.toString(), "-o", with.toString(), "--app-name", "widgets", + "--namespace", "widgets")); + assertEquals(application(without).get("symbol_table").toString(), + application(with).get("symbol_table").toString()); + assertEquals(application(without).get("artifacts").toString(), + application(with).get("artifacts").toString()); + } + + @Test + void removedWorkspaceRootIsRejected(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path elsewhere = Files.createDirectories(tmp.resolve("elsewhere")); + assertNotEquals(0, run("-i", in.toString(), "-o", tmp.resolve("out").toString(), + "--workspace-root", elsewhere.toString()), + "an input outside the workspace root would need absolute or .. keys"); + assertFalse(Files.exists(tmp.resolve("out/analysis.json")), "and nothing is written"); + } + + @Test + void namespaceIsRejectedWithSchemaV1(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + assertNotEquals(0, run("-i", in.toString(), "-o", tmp.resolve("out").toString(), + "--schema", "v1", "--namespace", "team"), + "v1 keys are absolute: the flag has nothing to relativize"); + } + @Test + void omittedIdentityFlagsDoNotReusePreviousInvocation(@TempDir Path tmp) throws Exception { + Path in = project(tmp.resolve("inventory")); + assertEquals(0, run("-i", in.toString(), "-o", tmp.resolve("first").toString(), "--app-name", "other", "--namespace", "team")); + assertEquals(0, run("-i", in.toString(), "-o", tmp.resolve("second").toString())); + assertEquals("can://inventory/inventory", application(tmp.resolve("second")).get("id").getAsString()); + } + + @Test + void namespaceScopesReferencesAndCache(@TempDir Path tmp) throws Exception { + Path in = project(tmp.resolve("repo/services/app")); + Files.writeString(in.resolve("application.properties"), "port=8080\n"); + for (String ns : new String[] {"team one", "team two"}) { + Path out = tmp.resolve(ns); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "-a", "2", "--no-build", "--no-rta", + "--external-calls", "--namespace", ns, "--app-name", "widgets", "--cache-dir", tmp.resolve("cache").toString())); + JsonObject app = application(out); + assertEquals(CanId.applicationId("widgets", ns), app.get("id").getAsString()); + assertEquals("widgets", app.get("application_name").getAsString()); + assertEquals(ns, app.get("identity_namespace").getAsString()); + assertTrue(app.getAsJsonObject("symbol_table").has("src/main/java/com/example/Widget.java")); + assertTrue(app.getAsJsonObject("artifacts").has("application.properties")); + assertTrue(app.has("external_symbols")); + assertOwnedReferences(app, CanId.applicationId("widgets", ns)); + var analysis = com.ibm.cldk.schema.V2Json.compact().fromJson( + Files.readString(out.resolve("analysis.json")), com.ibm.cldk.schema.Analysis.class); + var rows = com.ibm.cldk.neo4j.V2GraphProjector.project(analysis, "widgets"); + String root = CanId.applicationId("widgets", ns); + for (var node : rows.nodes) { + if (node.value.startsWith("can://")) assertTrue(node.value.equals(root) || node.value.startsWith(root + "/")); + } + for (var edge : rows.edges) { + for (String id : new String[] {edge.from.value, edge.to.value}) { + if (id.startsWith("can://")) assertTrue(id.equals(root) || id.startsWith(root + "/")); + } + } + + assertConformsToV2Schema(out); + } + } + + private static void assertOwnedReferences(JsonElement element, String root) { + if (element.isJsonObject()) { + for (var e : element.getAsJsonObject().entrySet()) { + if (e.getKey().startsWith("can://")) assertTrue(e.getKey().startsWith(root + "/")); + assertOwnedReferences(e.getValue(), root); + } + } else if (element.isJsonArray()) { + for (var e : element.getAsJsonArray()) assertOwnedReferences(e, root); + } else if (element.isJsonPrimitive() && element.getAsJsonPrimitive().isString()) { + String value = element.getAsString(); + if (value.startsWith("can://")) assertTrue(value.equals(root) || value.startsWith(root + "/"), value); + } + } + + @Test + void invalidExplicitIdentityNeverDefaults(@TempDir Path tmp) throws Exception { + Path in = project(tmp.resolve("app")); + for (String flag : new String[] {"--app-name", "--namespace"}) { + for (String value : new String[] {"", " ", "a/b", ".."}) { + assertNotEquals(0, run("-i", in.toString(), "-o", tmp.resolve("out").toString(), flag, value)); + assertFalse(Files.exists(tmp.resolve("out/analysis.json"))); + } + } + } + } diff --git a/src/test/java/com/ibm/cldk/L3WalaOverlaysTest.java b/src/test/java/com/ibm/cldk/L3WalaOverlaysTest.java index f3f78f53..586646ee 100644 --- a/src/test/java/com/ibm/cldk/L3WalaOverlaysTest.java +++ b/src/test/java/com/ibm/cldk/L3WalaOverlaysTest.java @@ -28,6 +28,6 @@ public void deriveApplicationIdFallsBackToCanUnknownWhenTheModuleHasNoId() throw String applicationId = invokeDeriveApplicationId(modules); - assertEquals("can://unknown", applicationId); + assertEquals("can://unknown/unknown", applicationId); } } diff --git a/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java b/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java new file mode 100644 index 00000000..fdc8d8c3 --- /dev/null +++ b/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java @@ -0,0 +1,475 @@ +package com.ibm.cldk; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ibm.cldk.neo4j.BoltConfig; +import com.ibm.cldk.neo4j.BoltWriter; +import com.ibm.cldk.neo4j.GraphRows; +import com.ibm.cldk.schema.CanId; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.api.io.TempDir; +import org.neo4j.driver.AuthTokens; +import org.neo4j.driver.Driver; +import org.neo4j.driver.GraphDatabase; +import org.neo4j.driver.Session; +import org.neo4j.driver.Values; +import org.testcontainers.containers.Neo4jContainer; + +/** + * Namespace identity against a live Neo4j: the incremental push's wipe and orphan prune are + * scoped by the app's {@code can:///} prefix and each module's own id, so repo-relative module + * ids must still be pruned when their file vanishes, and must not reach another application's graph. + * + *

    Needs a container runtime, so it is opt-in like {@code Neo4jBoltWriterTest}: it runs only when + * {@code RUN_CONTAINER_TESTS} is set. + */ +@EnabledIfEnvironmentVariable(named = "RUN_CONTAINER_TESTS", matches = ".+") +class WorkspaceRootNeo4jPushTest { + + private static final String PASSWORD = "testpassword123"; + + @SuppressWarnings("resource") + private static final Neo4jContainer CONTAINER = + new Neo4jContainer<>("neo4j:5").withAdminPassword(PASSWORD); + + private static Driver driver; + + @BeforeAll + static void startup() { + CONTAINER.start(); + driver = GraphDatabase.driver(CONTAINER.getBoltUrl(), AuthTokens.basic("neo4j", PASSWORD)); + } + + @AfterAll + static void teardown() { + if (driver != null) { + driver.close(); + } + CONTAINER.stop(); + } + + /** The CLI's pre-existing options are static; reset the ones this test sets, before and after. */ + @BeforeEach + @AfterEach + void resetStaticOptions() throws Exception { + set("emit", "json"); + set("analysisLevel", 1); + set("output", null); + set("input", null); + set("appName", null); + set("neo4jUri", null); + set("neo4jUser", null); + set("neo4jPassword", null); + CodeAnalyzer.projectRootPom = null; + } + + private static void set(String field, Object value) throws Exception { + Field f = CodeAnalyzer.class.getDeclaredField(field); + f.setAccessible(true); + f.set(null, value); + } + + private static void writeClass(Path srcRoot, String name) throws IOException { + Path pkg = Files.createDirectories(srcRoot.resolve("src/main/java/com/example")); + Files.writeString(pkg.resolve(name + ".java"), + "package com.example;\npublic class " + name + " {\n public int m() { return 1; }\n}\n", + StandardCharsets.UTF_8); + } + + private static int push(Path in, Path repo, String app, Path out) { + return new picocli.CommandLine(new CodeAnalyzer()).execute( + "-i", in.toString(), "-o", out.toString(), "--emit", "neo4j", "--no-build", + "--namespace", repo.getFileName().toString(), "--app-name", app, + "--neo4j-uri", CONTAINER.getBoltUrl(), "--neo4j-user", "neo4j", "--neo4j-password", PASSWORD); + } + + /** As {@link #push}, with the default namespace: keys remain relative to {@code in}. */ + private static int pushUnrooted(Path in, String app, Path out) { + return new picocli.CommandLine(new CodeAnalyzer()).execute( + "-i", in.toString(), "-o", out.toString(), "--emit", "neo4j", "--no-build", + "--app-name", app, + "--neo4j-uri", CONTAINER.getBoltUrl(), "--neo4j-user", "neo4j", "--neo4j-password", PASSWORD); + } + + private static long count(String cypher, Object... params) { + try (Session s = driver.session()) { + return s.run(cypher, Values.parameters(params)).single().get(0).asLong(); + } + } + + private static void run(String cypher, Object... params) { + try (Session s = driver.session()) { + s.run(cypher, Values.parameters(params)).consume(); + } + } + + private static String artifactPrefix(String app) { + return CanId.applicationId(app) + "/artifact/"; + } + + /** A project with one dependency manifest and one config file that mints a key. */ + private static void writeArtifacts(Path project, String... deps) throws IOException { + StringBuilder pom = new StringBuilder( + "4.0.0com.example" + + "demo1"); + for (String dep : deps) { + String[] ga = dep.split(":"); + pom.append("").append(ga[0]).append("") + .append(ga[1]).append("1.0"); + } + pom.append("\n"); + Files.writeString(project.resolve("pom.xml"), pom.toString(), StandardCharsets.UTF_8); + } + + private static void writeConfig(Path project, String... keys) throws IOException { + Path res = Files.createDirectories(project.resolve("src/main/resources")); + StringBuilder props = new StringBuilder(); + for (String k : keys) { + props.append(k).append("=v\n"); + } + Files.writeString(res.resolve("application.properties"), props.toString(), StandardCharsets.UTF_8); + } + + @Test + void changingNamespacePreservesPreviousApplicationAndUsesInputRelativeKeys(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path app = repo.resolve("services/app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + writeConfig(app, "server.port"); + assertEquals(0, pushUnrooted(app, "toggle", tmp.resolve("o1"))); + String oldPom = CanId.artifactId("toggle", "pom.xml"); + assertEquals(1, count("MATCH (a:Artifact {id: $id}) RETURN count(a)", "id", oldPom)); + assertEquals(0, push(app, repo, "toggle", tmp.resolve("o2"))); + assertEquals(1, count("MATCH (a:Artifact {id: $id}) RETURN count(a)", "id", oldPom), + "the prior namespace is outside reconciliation scope"); + String root = CanId.applicationId("toggle", "repo"); + String pom = CanId.artifactId("toggle", "pom.xml", "repo"); + assertEquals(1, count("MATCH (:JApplication {id: $app})-[:HAS_ARTIFACT]->(:Artifact {id: $pom}) RETURN count(*)", + "app", root, "pom", pom)); + String module = CanId.moduleId(root, "src/main/java/com/example/Main.java"); + assertEquals(1, count("MATCH (:JModule {id: $id}) RETURN count(*)", "id", module), + "same-file content hashes in another namespace cannot suppress module insertion"); + assertEquals(1, count("MATCH (:Artifact {id: $id})-[:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) RETURN count(*)", + "id", pom, "pkg", CanId.purlMaven("org.example", "lib"))); + } + + @Test + void aChangedManifestDropsItsRemovedDependencyAndConfigKey(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:kept", "org.example:dropped"); + writeConfig(app, "kept.key", "dropped.key"); + assertEquals(0, pushUnrooted(app, "edit", tmp.resolve("o1"))); + + String pom = CanId.artifactId("edit", "pom.xml"); + String props = CanId.artifactId("edit", "src/main/resources/application.properties"); + assertEquals(1, count("MATCH (:Artifact {id: $a})-[r:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) " + + "RETURN count(r)", "a", pom, "pkg", CanId.purlMaven("org.example", "dropped")), + "precondition: the dependency is declared"); + + writeArtifacts(app, "org.example:kept"); + writeConfig(app, "kept.key"); + assertEquals(0, pushUnrooted(app, "edit", tmp.resolve("o2"))); + + assertEquals(0, count("MATCH (:Artifact {id: $a})-[r:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) " + + "RETURN count(r)", "a", pom, "pkg", CanId.purlMaven("org.example", "dropped")), + "the removed dependency's edge is gone"); + assertEquals(1, count("MATCH (:Artifact {id: $a})-[r:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) " + + "RETURN count(r)", "a", pom, "pkg", CanId.purlMaven("org.example", "kept")), + "the kept dependency's edge stays"); + assertEquals(0, count("MATCH (k:ConfigKey {id: $id}) RETURN count(k)", + "id", CanId.configKeyId(props, "dropped.key")), "the removed config key is gone"); + assertEquals(1, count("MATCH (k:ConfigKey {id: $id}) RETURN count(k)", + "id", CanId.configKeyId(props, "kept.key")), "the kept config key stays"); + } + + @Test + void aSiblingAnalyzersArtifactsAndEdgesSurviveAJavaPush(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + assertEquals(0, pushUnrooted(app, "poly", tmp.resolve("o1"))); + + // Seed what codeanalyzer-python would have written: a claim on the shared pom.xml (with a + // pypi dependency edge), plus an artifact and config key Java never emits. + String pom = CanId.artifactId("poly", "pom.xml"); + String pyOnly = CanId.artifactId("poly", "pyproject.toml"); + String pyKey = CanId.configKeyId(pyOnly, "tool.name"); + run("MATCH (a:Artifact {id: $pom}) SET a:PyCanNode " + + "MERGE (app:PyApplication {id: 'can://poly/poly'}) MERGE (app)-[:HAS_ARTIFACT]->(a) " + + "MERGE (p:Package {id: 'pkg:pypi/requests'}) MERGE (a)-[:DECLARES_DEPENDENCY {_k: 'main'}]->(p) " + + "MERGE (o:Artifact:PyCanNode {id: $py}) MERGE (app)-[:HAS_ARTIFACT]->(o) " + + "MERGE (k:ConfigKey:PyCanNode {id: $key}) MERGE (o)-[:DEFINES_CONFIG]->(k) " + + "MERGE (o)-[:DECLARES_DEPENDENCY {_k: 'main'}]->(p)", + "pom", pom, "py", pyOnly, "key", pyKey); + + // Java now stops emitting pom.xml: it releases its claim but must not delete a node Python holds. + Files.delete(app.resolve("pom.xml")); + assertEquals(0, pushUnrooted(app, "poly", tmp.resolve("o2"))); + + assertEquals(1, count("MATCH (a:Artifact:PyCanNode {id: $id}) WHERE NOT a:JCanNode RETURN count(a)", + "id", pom), "the shared artifact survives with Python's claim and without Java's"); + assertEquals(0, count("MATCH (:JApplication)-[r:HAS_ARTIFACT]->(:Artifact {id: $id}) RETURN count(r)", + "id", pom), "Java's HAS_ARTIFACT to the released artifact is gone"); + assertEquals(0, count("MATCH (:Artifact {id: $id})-[r:DECLARES_DEPENDENCY]->(p:Package) " + + "WHERE p.id STARTS WITH 'pkg:maven/' RETURN count(r)", "id", pom), + "Java's maven dependency edge from the released artifact is gone"); + assertEquals(1, count("MATCH (:Artifact {id: $id})-[r:DECLARES_DEPENDENCY]->(:Package {id: 'pkg:pypi/requests'}) " + + "RETURN count(r)", "id", pom), "Python's dependency edge on the shared artifact stays"); + assertEquals(1, count("MATCH (:Artifact {id: $id})-[:DEFINES_CONFIG]->(:ConfigKey {id: $key}) " + + "RETURN count(*)", "id", pyOnly, "key", pyKey), + "an artifact and config key only Python wrote are untouched"); + } + + @Test + void anAppWhoseNameExtendsAnothersIsNotReconciledByIt(@TempDir Path tmp) throws IOException { + Path a = tmp.resolve("a"); + Path b = tmp.resolve("b"); + for (Path p : new Path[] {a, b}) { + writeClass(p, "Main"); + writeArtifacts(p, "org.example:lib"); + } + assertEquals(0, pushUnrooted(b, "apps", tmp.resolve("o1"))); + long apps = count("MATCH (n:Artifact) WHERE n.id STARTS WITH $p RETURN count(n)", "p", artifactPrefix("apps")); + assertTrue(apps > 0, "precondition"); + + assertEquals(0, pushUnrooted(a, "app", tmp.resolve("o2"))); + + assertEquals(apps, count("MATCH (n:Artifact:JCanNode) WHERE n.id STARTS WITH $p RETURN count(n)", + "p", artifactPrefix("apps")), "can://app's reconciliation never reaches can://apps"); + assertEquals(1, count("MATCH (:Artifact {id: $id})-[r:DECLARES_DEPENDENCY]->() RETURN count(r)", + "id", CanId.artifactId("apps", "pom.xml")), "can://apps keeps its dependency edge"); + } + + @Test + void aConfigKeyAnotherAnalyzerClaimsSurvivesJavaDroppingIt(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeConfig(app, "shared.key", "other"); + assertEquals(0, pushUnrooted(app, "ckshare", tmp.resolve("o1"))); + + String props = CanId.artifactId("ckshare", "src/main/resources/application.properties"); + String key = CanId.configKeyId(props, "shared.key"); + run("MATCH (k:ConfigKey {id: $key}) SET k:PyCanNode " + + "MERGE (b:PyBodyNode {id: 'can://ckshare/ckshare/python/x.py/f@1:1'}) MERGE (b)-[:PY_USES_CONFIG]->(k)", + "key", key); + + writeConfig(app, "other"); + assertEquals(0, pushUnrooted(app, "ckshare", tmp.resolve("o2"))); + + assertEquals(1, count("MATCH (k:ConfigKey:PyCanNode {id: $id}) WHERE NOT k:JCanNode RETURN count(k)", + "id", key), "the key survives with Python's claim and without Java's"); + assertEquals(1, count("MATCH (:Artifact {id: $a})-[:DEFINES_CONFIG]->(:ConfigKey {id: $k}) RETURN count(*)", + "a", props, "k", key), "its DEFINES_CONFIG stays"); + assertEquals(1, count("MATCH ()-[r:PY_USES_CONFIG]->(:ConfigKey {id: $k}) RETURN count(r)", "k", key), + "Python's read of it stays"); + } + + @Test + void anArtifactAnUnmarkedSiblingInventoriesSurvivesJavaDroppingIt(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + assertEquals(0, pushUnrooted(app, "oldsib", tmp.resolve("o1"))); + + // A pre-1.5.0 codeanalyzer-python: inventories the file via HAS_ARTIFACT, puts no marker on it. + String pom = CanId.artifactId("oldsib", "pom.xml"); + run("MATCH (a:Artifact {id: $pom}) MERGE (p:PyApplication {id: 'can://oldsib/oldsib'}) " + + "MERGE (p)-[:HAS_ARTIFACT]->(a)", "pom", pom); + + Files.delete(app.resolve("pom.xml")); + assertEquals(0, pushUnrooted(app, "oldsib", tmp.resolve("o2"))); + + assertEquals(1, count("MATCH (:PyApplication)-[:HAS_ARTIFACT]->(a:Artifact {id: $id}) " + + "WHERE NOT a:JCanNode RETURN count(a)", "id", pom), + "the artifact and the sibling's HAS_ARTIFACT survive; only Java's claim is released"); + } + + @Test + void codeEdgesIntoConfigFollowAConfigOnlyChange(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + Path pkg = Files.createDirectories(app.resolve("src/main/java/com/example")); + Files.writeString(pkg.resolve("Main.java"), "package com.example;\npublic class Main {\n" + + " public String port() { return System.getenv(\"PORT\"); }\n}\n", StandardCharsets.UTF_8); + Files.writeString(app.resolve(".env"), "OTHER=1\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "cfguse", tmp.resolve("o1"))); + String env = CanId.artifactId("cfguse", ".env"); + assertEquals(0, count("MATCH ()-[r:J_USES_CONFIG]->(k) WHERE k.id STARTS WITH $p RETURN count(r)", + "p", env), "precondition: PORT is undeclared"); + assertEquals(1, count("MATCH (:JApplication {id: 'can://cfguse/cfguse'})-[r:J_READS_CONFIG_UNRESOLVED {key: 'PORT'}]->() " + + "RETURN count(r)"), "precondition: the read is unresolved"); + + // Only the config changes; Main.java's content_hash is identical. + Files.writeString(app.resolve(".env"), "PORT=8080\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "cfguse", tmp.resolve("o2"))); + + assertEquals(1, count("MATCH ()-[r:J_USES_CONFIG]->(:ConfigKey {id: $k}) RETURN count(r)", + "k", CanId.configKeyId(env, "PORT")), "the newly declared key gains its read"); + assertEquals(0, count("MATCH (:JApplication {id: 'can://cfguse/cfguse'})-[r:J_READS_CONFIG_UNRESOLVED {key: 'PORT'}]->() " + + "RETURN count(r)"), "the stale unresolved read is gone"); + + Files.writeString(app.resolve(".env"), "OTHER=1\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "cfguse", tmp.resolve("o3"))); + assertEquals(0, count("MATCH ()-[r:J_USES_CONFIG]->(k) WHERE k.id STARTS WITH $p RETURN count(r)", + "p", env), "removing the key drops its read"); + } + + @Test + void aLockEdgeFollowsAManifestChangeWithAnUnchangedLockfile(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + Files.writeString(app.resolve("build.gradle"), + "dependencies {\n implementation 'org.example:a:1.0'\n implementation 'org.example:b:1.0'\n}\n", + StandardCharsets.UTF_8); + Files.writeString(app.resolve("gradle.lockfile"), + "org.example:a:1.0=runtimeClasspath\nempty=\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "locks", tmp.resolve("o1"))); + String lock = CanId.artifactId("locks", "gradle.lockfile"); + assertEquals(1, count("MATCH (:Artifact {id: $l})-[r:LOCKS]->(:Package {id: $p}) RETURN count(r)", + "l", lock, "p", CanId.purlMaven("org.example", "a")), "precondition: a is locked"); + + // Drop `a` from the manifest only. The lockfile is byte-identical but `a` is now lock-only. + Files.writeString(app.resolve("build.gradle"), + "dependencies {\n implementation 'org.example:b:1.0'\n}\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "locks", tmp.resolve("o2"))); + + assertEquals(0, count("MATCH (:Artifact {id: $m})-[r:DECLARES_DEPENDENCY]->(:Package {id: $p}) RETURN count(r)", + "m", CanId.artifactId("locks", "build.gradle"), "p", CanId.purlMaven("org.example", "a")), + "the manifest no longer declares a"); + assertEquals(1, count("MATCH (:Artifact {id: $l})-[r:LOCKS]->(:Package {id: $p}) RETURN count(r)", + "l", lock, "p", CanId.purlMaven("org.example", "a")), "the lock still pins a, once"); + } + + @Test + void anUnchangedRepushIsIdempotent(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + writeConfig(app, "k"); + assertEquals(0, pushUnrooted(app, "idem", tmp.resolve("o1"))); + String snapshot = "MATCH (n) WHERE n.id STARTS WITH 'can://idem' OPTIONAL MATCH (n)-[r]-() " + + "RETURN count(DISTINCT n) * 1000000 + count(DISTINCT r)"; + long before = count(snapshot); + + assertEquals(0, pushUnrooted(app, "idem", tmp.resolve("o2"))); + + assertEquals(before, count(snapshot), "a re-push with no change leaves node and edge counts unchanged"); + assertEquals(1, count("MATCH (:Artifact {id: $a})-[r:DECLARES_DEPENDENCY]->() RETURN count(r)", + "a", CanId.artifactId("idem", "pom.xml")), "no keyed edge is doubled"); + } + + @Test + void legacyArtifactsRequireExplicitRetirement(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + // What a 3.0.x-3.1.0 push left behind: the artifact keyed `can://artifact//...`. + String legacy = "can://artifact/legacy/pom.xml"; + run("MERGE (a:Artifact:JCanNode {id: $id}) MERGE (k:ConfigKey:JCanNode {id: $key}) " + + "MERGE (a)-[:DEFINES_CONFIG]->(k) " + + "MERGE (p:Package {id: 'pkg:maven/org.example/lib'}) MERGE (a)-[:DECLARES_DEPENDENCY {_k: 'main'}]->(p)", + "id", legacy, "key", legacy + "@key/x"); + + assertEquals(0, pushUnrooted(app, "legacy", tmp.resolve("o1"))); + + assertEquals(2, count("MATCH (n) WHERE n.id STARTS WITH 'can://artifact/legacy/' RETURN count(n)"), + "legacy data is outside the selected canonical application"); + assertEquals(1, count("MATCH (n:Artifact {id: $id}) RETURN count(n)", + "id", CanId.artifactId("legacy", "pom.xml")), "the current artifact is written"); + } + + @Test + void aV1PushLeavesV2ArtifactsAlone(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + assertEquals(0, pushUnrooted(app, "gen", tmp.resolve("o1"))); + long artifacts = count("MATCH (n:Artifact) WHERE n.id STARTS WITH $p RETURN count(n)", "p", artifactPrefix("gen")); + assertTrue(artifacts > 0, "precondition"); + + // A v1 full run's root is keyed on its bare name and it emits no artifacts. Driven through + // the writer directly: the v1 CLI path needs compiled classes, which --no-build cannot give. + GraphRows.NodeRow v1Root = new GraphRows.NodeRow(List.of("JApplication"), "name", "gen", + new java.util.HashMap<>(Map.of("name", "gen"))); + new BoltWriter().write(new GraphRows(List.of(v1Root), List.of()), + new BoltConfig(CONTAINER.getBoltUrl(), "neo4j", PASSWORD, null), true); + + assertEquals(artifacts, count("MATCH (n:Artifact:JCanNode) WHERE n.id STARTS WITH $p RETURN count(n)", + "p", artifactPrefix("gen")), "v1 emits no artifacts, so it must not release v2's"); + } + + @Test + void reconcilingOneAppLeavesAnotherAppsArtifactsAlone(@TempDir Path tmp) throws IOException { + Path a = tmp.resolve("a"); + Path b = tmp.resolve("b"); + for (Path p : new Path[] {a, b}) { + writeClass(p, "Main"); + writeArtifacts(p, "org.example:lib"); + writeConfig(p, "k"); + } + assertEquals(0, pushUnrooted(a, "appa", tmp.resolve("o1"))); + assertEquals(0, pushUnrooted(b, "appb", tmp.resolve("o2"))); + long bArtifacts = count("MATCH (n) WHERE (n:Artifact OR n:ConfigKey) AND n.id STARTS WITH $p " + + "RETURN count(n)", "p", artifactPrefix("appb")); + assertTrue(bArtifacts > 0, "precondition: the other app has artifacts"); + + Files.delete(a.resolve("pom.xml")); + Files.delete(a.resolve("src/main/resources/application.properties")); + assertEquals(0, pushUnrooted(a, "appa", tmp.resolve("o3"))); + + assertEquals(0, count("MATCH (n:Artifact {id: $id}) RETURN count(n)", + "id", CanId.artifactId("appa", "pom.xml")), "this app's vanished artifact is deleted"); + assertEquals(bArtifacts, count("MATCH (n) WHERE (n:Artifact OR n:ConfigKey) AND n.id STARTS WITH $p " + + "RETURN count(n)", "p", artifactPrefix("appb")), "the other app's artifacts are unchanged"); + } + + @Test + void aVanishedRepoRelativeModuleIsPrunedAndOtherAppsAreUntouched(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path orders = repo.resolve("services/orders"); + Path billing = repo.resolve("services/billing"); + writeClass(orders, "Keep"); + writeClass(orders, "Gone"); + writeClass(billing, "Gone"); + + assertEquals(0, push(orders, repo, "orders", tmp.resolve("o1"))); + assertEquals(0, push(billing, repo, "billing", tmp.resolve("o2"))); + + String key = "src/main/java/com/example/"; + String gone = CanId.moduleId(CanId.applicationId("orders", "repo"), key + "Gone.java"); + String keep = CanId.moduleId(CanId.applicationId("orders", "repo"), key + "Keep.java"); + String billingGone = CanId.moduleId(CanId.applicationId("billing", "repo"), + "src/main/java/com/example/Gone.java"); + assertEquals(1, count("MATCH (m:JModule {id: $id}) RETURN count(m)", "id", gone), + "precondition: the module is pushed under its repo-relative id"); + long billingNodes = count("MATCH (n) WHERE n.id STARTS WITH $p RETURN count(n)", + "p", CanId.applicationId("billing", "repo") + "/"); + assertTrue(billingNodes > 0, "precondition: the other app is in the same database"); + + Files.delete(orders.resolve("src/main/java/com/example/Gone.java")); + assertEquals(0, push(orders, repo, "orders", tmp.resolve("o3"))); + + assertEquals(0, count("MATCH (n) WHERE n.id = $id OR n.id STARTS WITH $p RETURN count(n)", + "id", gone, "p", gone + "/"), "the vanished module and its declarations are pruned"); + assertEquals(1, count("MATCH (m:JModule {id: $id}) RETURN count(m)", "id", keep), + "the surviving module stays"); + assertEquals(1, count("MATCH (m:JModule {id: $id}) RETURN count(m)", "id", billingGone), + "a same-named file in another app is a different node and is not touched"); + assertEquals(billingNodes, count("MATCH (n) WHERE n.id STARTS WITH $p RETURN count(n)", + "p", CanId.applicationId("billing", "repo") + "/"), "the other app's graph is unchanged"); + } +} diff --git a/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java b/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java index 097010e3..a331fd6f 100644 --- a/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java @@ -25,6 +25,44 @@ */ class BoltWriterPurgeTest { + @Test + void fullRunPruneSelectsActualSchemaFamilyAndProtectsSharedDisplayNames() { + var legacy = GraphProjector.project(java.util.Map.of(), null, "shared"); + String query = BoltWriter.orphanPruneQuery(legacy); + assertTrue(query.contains("a.id IS NULL AND a.name = $appName")); + assertTrue(query.contains("[:J_HAS_UNIT]")); + assertTrue(query.contains("AND NOT c.file_key IN $present")); + assertTrue(query.contains("DETACH DELETE x, c")); + assertFalse(query.contains("a.id = $appId")); + var analysis = com.ibm.cldk.schema.V2Emitter.emit("shared", 1, java.util.Map.of(), null, + null, null, null, null, null, null, "team"); + String canonical = BoltWriter.orphanPruneQuery(V2GraphProjector.project(analysis, "shared")); + assertTrue(canonical.contains("a.id = $appId")); + assertFalse(canonical.contains("a.name")); + } + + @Test + void existingAnchorMustHaveExactMarkerAndRawIdentity() { + var identity = java.util.Map.of("identity_version", 2, + "application_name", "app", "identity_namespace", "team"); + BoltWriter.requireCompatibleApplicationIdentity("can://team/app", identity, identity); + for (var legacy : java.util.List.of(java.util.Map.of(), + java.util.Map.of("identity_version", 2.5, "application_name", "app", "identity_namespace", "team"), + java.util.Map.of("identity_version", 1, "application_name", "app", "identity_namespace", "team"), + java.util.Map.of("identity_version", 2, "application_name", "app", "identity_namespace", "other"))) { + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> BoltWriter.requireCompatibleApplicationIdentity("can://team/app", identity, legacy)); + } + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> BoltWriter.requireCompatibleApplicationIdentity("can://team/app", java.util.Map.of(), identity)); + } + + @Test + void incrementalHashesAreScopedToSelectedApplication() { + assertTrue(BoltWriter.MODULE_HASHES_V2.contains("{id: $appId}")); + assertTrue(BoltWriter.MODULE_HASHES_V2.contains("J_HAS_MODULE")); + } + @Test void thePurgeIsScopedByCanIdPrefixAndNotByTheRetiredProperty() { for (String stmt : new String[] {BoltWriter.PURGE_MODULE_EDGES_V2, BoltWriter.PURGE_VANISHED_NODES_V2}) { @@ -63,26 +101,12 @@ void theDescendantPrefixEndsWithTheSeparator() { } @Test - void theOrphanPruneMatchesAV2RootByIdAndALegacyV1RootByNameButNotAnotherApplication() { - // v2 roots are keyed on `id` -- unique-constrained and exact -- but legacy v1 roots - // (GraphProjector.java) never write `id`, only `name`. An id-only match orphans a prior v1 - // graph instead of pruning it; an ungated `a.name =` disjunct reaches every v2 root - // carrying that display name, including roots this analyzer never minted, since `name` - // stopped being unique once the id re-key landed. Note this is scoping hygiene, not - // disambiguation: CanId.applicationId is "can://" + appName, so two services sharing an - // --app-name share an id and are one node either way. + void orphanPruneNeverFallsBackToLegacyName() { String stmt = BoltWriter.PRUNE_VANISHED_UNITS_V2; - assertTrue(stmt.contains("a.id = $appId"), - "must reach a v2 root of THIS app by its unique id: " + stmt); - assertTrue(stmt.contains("a.id IS NULL AND a.name = $appName"), - "must reach a legacy v1 root (no id) by name, guarded by id IS NULL: " + stmt); - // Every `a.name =` must sit inside the `a.id IS NULL` guard: an unguarded one reaches v2 - // roots this analyzer never minted that merely carry the same display name. - assertFalse(stmt.replace("a.id IS NULL AND a.name = $appName", "").contains("a.name ="), - "a.name must never be matched outside the a.id IS NULL guard -- that would reach " - + "any v2 root carrying this display name, not just this app's: " + stmt); - assertFalse(stmt.contains("{name:"), - "the prune must not property-match :JApplication by its no-longer-unique name: " + stmt); + assertTrue(stmt.contains("a.id = $appId")); + assertFalse(stmt.contains("a.name")); + org.junit.jupiter.api.Assertions.assertEquals(java.util.List.of("can://team/app/artifact/"), + BoltWriter.artifactPrefixes("can://team/app", "app")); } @Test diff --git a/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java b/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java new file mode 100644 index 00000000..4b130388 --- /dev/null +++ b/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java @@ -0,0 +1,58 @@ +package com.ibm.cldk.neo4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.gson.JsonParser; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class CanonicalIdentityRowsTest { + @Test + void conflictingCanonicalEntityPropertiesAreRejected() { + RowBuilder builder = new RowBuilder(); + String id = "can://team/app/java/Foo.java/Foo"; + builder.node(List.of("JSymbol"), "id", id, Map.of("kind", "class")); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> builder.node(List.of("JType"), "id", id, Map.of("kind", "method"))); + } + + @Test + void ordinaryCanonicalDisplayPropertyRefinementIsRejected() { + RowBuilder builder = new RowBuilder(); + String id = "can://team/app/java/Foo.java/Foo"; + builder.node(List.of("JSymbol"), "id", id, Map.of("name", "Foo")); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> builder.node(List.of("JSymbol"), "id", id, Map.of("name", "refined Foo"))); + } + + @Test + void canonicalIdsDeduplicateAcrossLabelsWithoutCollapsingEdgeKindsOrSites() throws Exception { + var fixture = JsonParser.parseReader(new InputStreamReader( + getClass().getResourceAsStream("/identity/namespaces.json"), StandardCharsets.UTF_8)) + .getAsJsonObject().getAsJsonObject("duplicate_module_rows"); + RowBuilder builder = new RowBuilder(); + GraphRows.NodeRef first = null; + GraphRows.NodeRef second = null; + for (var row : fixture.getAsJsonArray("rows")) { + String id = row.getAsJsonObject().get("module_id").getAsString(); + var ref = builder.node(List.of("JModule"), "id", id, Map.of("id", id)); + if (first == null) first = ref; + else if (!first.value.equals(id)) second = ref; + } + assertEquals(fixture.get("expected_unique_id_count").getAsInt(), builder.finish().nodes.size()); + var alternate = builder.node(List.of("JSymbol"), "id", first.value, Map.of("id", first.value)); + builder.keyedEdge("J_CALLS", first, second, Map.of(), "site1"); + builder.keyedEdge("J_CALLS", alternate, second, Map.of(), "site1"); + builder.keyedEdge("J_CALLS", first, second, Map.of(), "site2"); + builder.keyedEdge("J_RESOLVES_TO", first, second, Map.of(), "site1"); + var rows = builder.finish(); + assertEquals(2, rows.nodes.size()); + assertEquals(3, rows.edges.size()); + org.junit.jupiter.api.Assertions.assertTrue(rows.nodes.stream() + .filter(n -> n.value.equals(alternate.value)).findFirst().orElseThrow() + .labels.containsAll(List.of("JModule", "JSymbol", "JCanNode"))); + } +} diff --git a/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java b/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java index 84964c45..cbb22946 100644 --- a/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java @@ -61,6 +61,24 @@ public void flush() {} public void close() {} } + @Test + void actualV1SnapshotsReplaceOnlyTheNameKeyedLegacyApplication() { + GraphRows legacy = GraphProjector.project(Map.of(), null, "shared"); + String script = CypherWriter.renderCypher(legacy, "shared"); + assertTrue(script.contains("a.id IS NULL AND a.name = 'shared'"), script); + assertTrue(script.contains("OPTIONAL MATCH (a)-[:J_HAS_UNIT]->(c)"), script); + assertTrue(script.contains("OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x)"), script); + assertTrue(script.contains("DETACH DELETE x, c, a;"), script); + assertFalse(script.contains("STARTS WITH"), "v1 must not sweep a synthesized canonical root"); + assertTrue(script.indexOf("DETACH DELETE x, c, a;") < script.indexOf("// ── nodes")); + var analysis = com.ibm.cldk.schema.V2Emitter.emit("shared", 1, Map.of(), null, + null, null, null, null, null, null, "team"); + String canonical = CypherWriter.renderCypher(V2GraphProjector.project(analysis, "shared"), "shared"); + assertTrue(canonical.contains("a.id = 'can://team/shared'")); + assertFalse(canonical.contains("a.name ="), "the same display name cannot activate v1 cleanup for v2"); + assertFalse(canonical.contains("can://shared/shared")); + } + @Test void streamedOutputIsIdenticalToTheRenderedString() throws IOException { GraphRows rows = sampleRows(1200); // > 2 batches of 500 @@ -148,15 +166,9 @@ void theWipePreambleMatchesAV2RootByIdAndALegacyV1RootByNameButNotAnotherApplica // "can://" + appName, so those share an id and are one node regardless. String out = CypherWriter.renderCypher(new RowBuilder().finish(), "app"); - assertTrue(out.contains("a.id = 'can://app'"), + assertTrue(out.contains("a.id = 'can://app/app'"), "must reach a v2 root of THIS app by its unique id, got: " + out); - assertTrue(out.contains("a.id IS NULL AND a.name = 'app'"), - "must reach a legacy v1 root (no id) by name, guarded by id IS NULL, got: " + out); - // Every `a.name =` in the predicate must be inside the `a.id IS NULL AND ...` guard: an - // unguarded one reaches v2 roots this analyzer never minted that carry the same name. - assertFalse(out.replace("a.id IS NULL AND a.name = 'app'", "").contains("a.name ="), - "a.name must never be matched outside the a.id IS NULL guard -- that would reach " - + "any v2 root carrying this display name, not just this app's, got: " + out); + assertFalse(out.contains("a.name ="), "legacy roots require explicit retirement"); assertFalse(out.contains("JApplication {name:"), "the wipe must not property-match :JApplication by its no-longer-unique name: " + out); } @@ -173,7 +185,7 @@ void anEmptyApplicationNameIsRefusedRatherThanEmittingAnUnscopedWipe() { "an unscoped destructive statement must never be rendered, app name: " + name); } // ... and the guard lives on the prefix itself, so every future caller inherits it. - assertEquals("can://app/", CypherWriter.applicationPrefix("app"), + assertEquals("can://app/app/", CypherWriter.applicationPrefix("app"), "the swept prefix must end in the separator, or `can://appX` is in scope too"); } } diff --git a/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java b/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java index ec93318d..5d674a13 100644 --- a/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java @@ -258,7 +258,7 @@ public void wipeCoversBothGenerationsSoV2ReplacesAPriorV1Graph() { String cypher = CypherWriter.renderCypher(rows, APP_NAME); assertTrue(cypher.contains("J_HAS_UNIT|J_HAS_MODULE"), "the wipe must traverse both generations' unit relationship"); - assertTrue(cypher.contains("MATCH (s:JSymbol) WHERE NOT (s)--() DELETE s"), + assertFalse(cypher.contains("MATCH (s:JSymbol) WHERE NOT (s)--() DELETE s"), "the wipe must sweep orphaned symbols (v1 import-materialized type stubs)"); // The v1 path is NOT reachable by the can:// prefix sweep -- v1 ids are FQNs and v1 roots // carry no `id` at all -- so replacing the containment traversal with the sweep orphans @@ -267,7 +267,7 @@ public void wipeCoversBothGenerationsSoV2ReplacesAPriorV1Graph() { && cypher.contains("OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x)") && cypher.contains("DETACH DELETE x, c, a;"), "the v1-reaching containment traversal must survive alongside the prefix sweep: " + cypher); - assertTrue(cypher.contains("a.id IS NULL AND a.name = '" + APP_NAME + "'"), + assertFalse(cypher.contains("a.id IS NULL AND a.name = '" + APP_NAME + "'"), "a legacy v1 root has no id and is reachable only by the guarded name branch: " + cypher); for (String rel : new String[] {"J_HAS_CALLSITE", "J_HAS_COMMENT", "J_HAS_PARAMETER", "J_DECLARES", "J_HAS_METHOD", "J_HAS_BODY_NODE"}) { @@ -470,9 +470,9 @@ void theWipeReachesThisAppsArtifactsAndConfigKeysButNeverAPackage() { // analyzer's next push. :Package (a `pkg:` purl) sits under no application, so it stays out // by construction -- assert that, because it is the only thing left holding the line. String cypher = CypherWriter.renderCypher(rows, APP_NAME); - String prefix = "can://" + APP_NAME + "/"; + String prefix = CanId.applicationId(APP_NAME) + "/"; - assertTrue(cypher.contains("MATCH (x:" + RowBuilder.CAN_NODE + ") WHERE x.id = 'can://" + APP_NAME + assertTrue(cypher.contains("MATCH (x:" + RowBuilder.CAN_NODE + ") WHERE x.id = '" + CanId.applicationId(APP_NAME) + "' OR x.id STARTS WITH '" + prefix + "'"), "the wipe must sweep the whole app prefix, rooted on the label-anchored index: " + cypher); assertTrue(cypher.contains("CALL { WITH x DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS;"), diff --git a/src/test/java/com/ibm/cldk/schema/CanIdTest.java b/src/test/java/com/ibm/cldk/schema/CanIdTest.java index 963a9d54..02d8f9a1 100644 --- a/src/test/java/com/ibm/cldk/schema/CanIdTest.java +++ b/src/test/java/com/ibm/cldk/schema/CanIdTest.java @@ -13,34 +13,66 @@ */ class CanIdTest { + @Test + void namespaceFixtureAndInvalidNames() throws Exception { + var fixture = com.google.gson.JsonParser.parseReader(new java.io.InputStreamReader( + getClass().getResourceAsStream("/identity/namespaces.json"), java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + for (var row : fixture.getAsJsonArray("roots")) { + var r = row.getAsJsonObject(); + String ns = r.get("namespace").isJsonNull() ? null : r.get("namespace").getAsString(); + assertEquals(r.get("expected_root").getAsString(), CanId.applicationId(r.get("application_name").getAsString(), ns)); + } + for (var row : fixture.getAsJsonArray("invalid")) { + String value = row.getAsJsonObject().get("value").getAsString(); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, () -> CanId.applicationId(value)); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, () -> CanId.applicationId("app", value)); + } + assertEquals("can://team/%21%27%28%29%2A~", CanId.applicationId("!'()*~", "team")); + for (String value : List.of("\u00a0", "\u2007", "\u202f", "\u0085")) { + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, () -> CanId.applicationId(value)); + } + assertEquals("can://team/%F0%9F%98%80", CanId.applicationId("😀", "team")); + } + @Test void applicationIdPutsTheAppOutermost() { - assertEquals("can://daytrader8", CanId.applicationId("daytrader8")); + assertEquals("can://daytrader8/daytrader8", CanId.applicationId("daytrader8")); } @Test void moduleIdNestsTheLanguageUnderTheApp() { String app = CanId.applicationId("daytrader8"); - assertEquals("can://daytrader8/java/src/Foo.java", CanId.moduleId(app, "src/Foo.java")); + assertEquals("can://daytrader8/daytrader8/java/src/Foo.java", CanId.moduleId(app, "src/Foo.java")); } @Test void moduleId_normalizesBackslashesAndLeadingDotSlash() { String app = CanId.applicationId("myapp"); assertEquals( - "can://myapp/java/a/b/C.java", CanId.moduleId(app, "./a\\b\\C.java")); + "can://myapp/myapp/java/a/b/C.java", CanId.moduleId(app, "./a\\b\\C.java")); + } + + @Test + void moduleId_keepsARepoRelativeKeyWhole() { + // --workspace-root keys carry the path from the repository root; the leading-[./] strip must + // only drop a `./`, never eat into the key's own first segment. + String app = CanId.applicationId("myapp"); + assertEquals("can://myapp/myapp/java/services/app/src/Foo.java", + CanId.moduleId(app, "services/app/src/Foo.java")); + assertEquals("can://myapp/myapp/java/services/app/src/Foo.java", + CanId.moduleId(app, "./services/app/src/Foo.java")); } @Test void childId_appendsSegmentWithSlash() { assertEquals( - "can://myapp/java/src/Foo.java/com.example.Foo", - CanId.childId("can://myapp/java/src/Foo.java", "com.example.Foo")); + "can://myapp/myapp/java/src/Foo.java/com.example.Foo", + CanId.childId("can://myapp/myapp/java/src/Foo.java", "com.example.Foo")); } @Test void ordinalId_appendsTagAfterAt() { - String callableId = "can://myapp/java/src/Foo.java/com.example.Foo/bar(int)"; + String callableId = "can://myapp/myapp/java/src/Foo.java/com.example.Foo/bar(int)"; assertEquals(callableId + "@15:2", CanId.ordinalId(callableId, "15:2")); assertEquals(callableId + "@entry", CanId.ordinalId(callableId, "entry")); } @@ -50,14 +82,14 @@ void externalIdIsLanguageNeutralUnderTheApp() { // No `/java/` segment: @external sits where the language sits for code nodes, so a sibling // analyzer over the same app mints the same id and merges onto the same node (#244). assertEquals( - "can://daytrader8/@external/java.util.Map/get(java.lang.Object)", + "can://daytrader8/daytrader8/@external/java.util.Map/get(java.lang.Object)", CanId.externalId("daytrader8", "java.util.Map", "get(java.lang.Object)")); } @Test void artifactIdNestsUnderTheAppInsteadOfAParallelScheme() { // Was can://artifact// — a third outermost shape. Now one rule. - assertEquals("can://daytrader8/artifact/pom.xml", CanId.artifactId("daytrader8", "pom.xml")); + assertEquals("can://daytrader8/daytrader8/artifact/pom.xml", CanId.artifactId("daytrader8", "pom.xml")); } @Test diff --git a/src/test/java/com/ibm/cldk/schema/L2SchemaOracleTest.java b/src/test/java/com/ibm/cldk/schema/L2SchemaOracleTest.java index c0eb77b1..0b61e752 100644 --- a/src/test/java/com/ibm/cldk/schema/L2SchemaOracleTest.java +++ b/src/test/java/com/ibm/cldk/schema/L2SchemaOracleTest.java @@ -44,11 +44,13 @@ class L2SchemaOracleTest { /** Wrap application-level members in an otherwise-minimal conformant envelope. */ private static String payload(String applicationMembers) { return "{" + + "\"identity_version\":2," + "\"schema_version\":\"2.0.0\"," + "\"language\":\"java\"," + "\"max_level\":2," + "\"application\":{" + "\"id\":\"" + APP_ID + "\"," + + "\"application_name\":\"myapp\",\"identity_namespace\":\"myapp\"," + "\"kind\":\"application\"," + "\"symbol_table\":{}" + (applicationMembers.isEmpty() ? "" : "," + applicationMembers) diff --git a/src/test/java/com/ibm/cldk/schema/L3SchemaOracleTest.java b/src/test/java/com/ibm/cldk/schema/L3SchemaOracleTest.java index 83be68ee..38c31b36 100644 --- a/src/test/java/com/ibm/cldk/schema/L3SchemaOracleTest.java +++ b/src/test/java/com/ibm/cldk/schema/L3SchemaOracleTest.java @@ -52,11 +52,13 @@ private static String payload(String overlays) { + "\"source\":\"\"," + "\"types\":{\"Foo\":" + type + "}}"; return "{" + + "\"identity_version\":2," + "\"schema_version\":\"2.0.0\"," + "\"language\":\"java\"," + "\"max_level\":3," + "\"application\":{" + "\"id\":\"" + APP_ID + "\"," + + "\"application_name\":\"myapp\",\"identity_namespace\":\"myapp\"," + "\"kind\":\"application\"," + "\"symbol_table\":{\"Foo.java\":" + module + "}}}"; } diff --git a/src/test/java/com/ibm/cldk/schema/L4SchemaOracleTest.java b/src/test/java/com/ibm/cldk/schema/L4SchemaOracleTest.java index ef26b345..3a199025 100644 --- a/src/test/java/com/ibm/cldk/schema/L4SchemaOracleTest.java +++ b/src/test/java/com/ibm/cldk/schema/L4SchemaOracleTest.java @@ -53,11 +53,13 @@ private static String payload(String localId, String kind) { + "\"source\":\"\"," + "\"types\":{\"Foo\":" + type + "}}"; return "{" + + "\"identity_version\":2," + "\"schema_version\":\"2.0.0\"," + "\"language\":\"java\"," + "\"max_level\":4," + "\"application\":{" + "\"id\":\"" + APP_ID + "\"," + + "\"application_name\":\"myapp\",\"identity_namespace\":\"myapp\"," + "\"kind\":\"application\"," + "\"symbol_table\":{\"Foo.java\":" + module + "}}}"; } diff --git a/src/test/java/com/ibm/cldk/schema/V2JsonTest.java b/src/test/java/com/ibm/cldk/schema/V2JsonTest.java index 4c644786..5f1a5db3 100644 --- a/src/test/java/com/ibm/cldk/schema/V2JsonTest.java +++ b/src/test/java/com/ibm/cldk/schema/V2JsonTest.java @@ -46,6 +46,19 @@ private static JsonObject payload() { return V2Json.compact().toJsonTree(V2Emitter.emit("myapp", 1, modules)).getAsJsonObject(); } + @Test + void identityMetadataIsRequiredAndNotDefaultedWhenReadingOldPayloads() { + JsonObject json = payload(); + assertEquals(2, json.get("identity_version").getAsInt()); + assertEquals("myapp", json.getAsJsonObject("application").get("application_name").getAsString()); + assertEquals("myapp", json.getAsJsonObject("application").get("identity_namespace").getAsString()); + json.remove("identity_version"); + Analysis legacy = V2Json.compact().fromJson(json, Analysis.class); + assertNull(legacy.getIdentityVersion()); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> com.ibm.cldk.neo4j.V2GraphProjector.project(legacy, "myapp")); + } + private static JsonObject module() { return payload().getAsJsonObject("application").getAsJsonObject("symbol_table").getAsJsonObject(FILE_KEY); } diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/L1CacheTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/L1CacheTest.java index 2631f2f4..46155a2f 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/L1CacheTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/L1CacheTest.java @@ -54,6 +54,16 @@ private static String hintOf(Map modules) { .getBody().get("4:5").getDeclaringTypeHint(); } + @Test + void missingIdentityMarkerCannotReuseOldCache(@TempDir Path cacheDir) throws Exception { + L1Cache.save(cacheDir, APP, VERSION, modulesWithHint("hint")); + Path path = L1Cache.fileIn(cacheDir); + String json = java.nio.file.Files.readString(path).replace("\"identity_version\":2,", ""); + java.nio.file.Files.writeString(path, json); + org.junit.jupiter.api.Assertions.assertTrue(L1Cache.load(cacheDir, APP, VERSION).isEmpty()); + assertEquals(json, java.nio.file.Files.readString(path), "incompatible persisted input remains read-only"); + } + @Test void save_thenLoad_preservesTheDeclaringTypeHint(@TempDir Path cacheDir) { L1Cache.save(cacheDir, APP, VERSION, modulesWithHint("java.util.Map$Entry")); diff --git a/src/test/resources/identity/namespaces.json b/src/test/resources/identity/namespaces.json new file mode 100644 index 00000000..8a892193 --- /dev/null +++ b/src/test/resources/identity/namespaces.json @@ -0,0 +1,139 @@ +{ + "identity_version": 2, + "roots": [ + { + "application_name": "inventory", + "namespace": null, + "expected_root": "can://inventory/inventory" + }, + { + "application_name": "currencyservice", + "namespace": "online-boutique", + "expected_root": "can://online-boutique/currencyservice" + }, + { + "application_name": "a%b", + "namespace": "team space", + "expected_root": "can://team%20space/a%25b" + }, + { + "application_name": "caf\u00e9", + "namespace": "\u56e2\u961f", + "expected_root": "can://%E5%9B%A2%E9%98%9F/caf%C3%A9" + }, + { + "application_name": "Inventory", + "namespace": "Team", + "expected_root": "can://Team/Inventory" + }, + { + "application_name": "inventory", + "namespace": "team", + "expected_root": "can://team/inventory" + }, + { + "application_name": " cafe\u0301 ", + "namespace": "team", + "expected_root": "can://team/%20cafe%CC%81%20" + } + ], + "invalid": [ + { + "value": "", + "reason": "empty" + }, + { + "value": " ", + "reason": "whitespace-only" + }, + { + "value": "a/b", + "reason": "slash" + }, + { + "value": "a\\b", + "reason": "backslash" + }, + { + "value": ".", + "reason": "dot" + }, + { + "value": "..", + "reason": "dot-dot" + }, + { + "value": "a\u0001b", + "reason": "control" + }, + { + "value": "\ud800", + "reason": "unpaired-high-surrogate" + }, + { + "value": "\udc00", + "reason": "unpaired-low-surrogate" + } + ], + "modules": [ + { + "application_name": "currencyservice", + "namespace": "online-boutique", + "language": "javascript", + "file_key": "server.js", + "expected_uri": "can://online-boutique/currencyservice/javascript/server.js" + }, + { + "application_name": "paymentservice", + "namespace": "online-boutique", + "language": "javascript", + "file_key": "server.js", + "expected_uri": "can://online-boutique/paymentservice/javascript/server.js" + }, + { + "application_name": "inventory", + "namespace": null, + "language": "python", + "file_key": "src/main.py", + "expected_uri": "can://inventory/inventory/python/src/main.py" + }, + { + "application_name": "inventory", + "namespace": null, + "language": "typescript", + "file_key": "src/main.ts", + "expected_uri": "can://inventory/inventory/typescript/src/main.ts" + }, + { + "application_name": "inventory", + "namespace": null, + "language": "java", + "file_key": "src/Main.java", + "expected_uri": "can://inventory/inventory/java/src/Main.java" + }, + { + "application_name": "inventory", + "namespace": null, + "language": "go", + "file_key": "main.go", + "expected_uri": "can://inventory/inventory/go/main.go" + } + ], + "duplicate_module_rows": { + "rows": [ + { + "module_id": "can://online-boutique/currencyservice/javascript/server.js", + "file_key": "server.js" + }, + { + "module_id": "can://online-boutique/paymentservice/javascript/server.js", + "file_key": "server.js" + }, + { + "module_id": "can://online-boutique/currencyservice/javascript/server.js", + "file_key": "server.js" + } + ], + "expected_unique_id_count": 2 + } +} diff --git a/src/test/resources/schema/analysis.v2.schema.json b/src/test/resources/schema/analysis.v2.schema.json index 18107ffa..efcd4e41 100644 --- a/src/test/resources/schema/analysis.v2.schema.json +++ b/src/test/resources/schema/analysis.v2.schema.json @@ -5,8 +5,9 @@ "description": "Conformance oracle for codeanalyzer-java's v2 output. Encodes the canonical CPG shape: an envelope carrying one application node, whose containment tree is named maps down to callable depth, with body nodes keyed by local id. Strict (additionalProperties: false) so an accidentally renamed or stray key fails the gate rather than silently reaching consumers. Replace with the SDK's v2 models once they exist.", "type": "object", "additionalProperties": false, - "required": ["schema_version", "language", "max_level", "application"], + "required": ["identity_version", "schema_version", "language", "max_level", "application"], "properties": { + "identity_version": { "const": 2 }, "schema_version": { "const": "2.0.0" }, "language": { "const": "java" }, "max_level": { "type": "integer", "minimum": 1, "maximum": 4 }, @@ -27,12 +28,12 @@ "canId": { "type": "string", - "pattern": "^can://[^/]+/java/", + "pattern": "^can://[^/]+/[^/]+/java/", "description": "Durable id for nodes at or above callable depth: can:///java/..." }, "applicationCanId": { "type": "string", - "pattern": "^can://[^/]+$", + "pattern": "^can://[^/]+/[^/]+$", "description": "Id of the application root node: can://, no trailing segment." }, @@ -105,9 +106,11 @@ "application": { "type": "object", "additionalProperties": false, - "required": ["id", "kind", "symbol_table"], + "required": ["id", "kind", "symbol_table", "application_name", "identity_namespace"], "properties": { "id": { "$ref": "#/$defs/applicationCanId" }, + "application_name": { "type": "string", "minLength": 1 }, + "identity_namespace": { "type": "string", "minLength": 1 }, "kind": { "const": "application" }, "symbol_table": { "type": "object", @@ -280,7 +283,7 @@ "externalCanId": { "type": "string", - "pattern": "^can://[^/]+/@external/", + "pattern": "^can://[^/]+/[^/]+/@external/", "description": "Id of a symbol outside the project, so an in-project id landing in the external map fails the gate instead of validating happily. Language-NEUTRAL like artifactCanId: @external sits where the language segment sits for code nodes, so sibling analyzers over the same name a library symbol identically. It therefore does NOT match canId." }, @@ -306,7 +309,7 @@ "artifactCanId": { "type": "string", - "pattern": "^can://[^/]+/artifact/", + "pattern": "^can://[^/]+/[^/]+/artifact/", "description": "Id of a repository-artifact node: nested under the app, language-neutral." },