From 8b57a22a692e2e3edafde94c71c2d4e5c55e4d58 Mon Sep 17 00:00:00 2001 From: George Safta Date: Sat, 24 Oct 2026 17:53:04 +0300 Subject: [PATCH 01/11] fix: existing integration tests --- .../ibm/cldk/CodeAnalyzerIntegrationTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerIntegrationTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerIntegrationTest.java index cd933cc4..1fe38caf 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerIntegrationTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerIntegrationTest.java @@ -134,7 +134,7 @@ void callGraphShouldHaveKnownEdges() throws Exception { var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/call-graph-test --analysis-level=2", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/call-graph-test --analysis-level=2", javaHomePath, codeanalyzerVersion ) ); @@ -169,7 +169,7 @@ void corruptMavenShouldProduceAnalysisArtifactsWhenMVNCommandIsInPath() throws I var corruptMavenProjectBuild = mavenContainer.withWorkingDirectory("/test-applications/mvnw-corrupt-test").execInContainer("mvn", "-f", "/test-applications/mvnw-corrupt-test/pom.xml", "clean", "compile"); Assertions.assertEquals(0, corruptMavenProjectBuild.getExitCode(), "Failed to build the project with system's default Maven."); // NOw run codeanalyzer and assert if analysis.json is generated. - var runCodeAnalyzer = mavenContainer.execInContainer("java", "-jar", String.format("/opt/jars/codeanalyzer-%s.jar", codeanalyzerVersion), "--input=/test-applications/mvnw-corrupt-test", "--output=/tmp/", "--analysis-level=2", "--verbose", "--no-build"); + var runCodeAnalyzer = mavenContainer.execInContainer("java", "-jar", String.format("/opt/jars/codeanalyzer-%s.jar", codeanalyzerVersion), "--schema", "v1", "--input=/test-applications/mvnw-corrupt-test", "--output=/tmp/", "--analysis-level=2", "--verbose", "--no-build"); var codeAnalyzerOutputDirContents = mavenContainer.execInContainer("ls", "/tmp/analysis.json"); String codeAnalyzerOutputDirContentsStdOut = codeAnalyzerOutputDirContents.getStdout(); Assertions.assertTrue(codeAnalyzerOutputDirContentsStdOut.length() > 0, "Could not find 'analysis.json'."); @@ -185,7 +185,7 @@ void corruptMavenShouldNotTerminateWithErrorWhenMavenIsNotPresentUnlessAnalysisL var runCodeAnalyzer = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/mvnw-corrupt-test --output=/tmp/ --analysis-level=2", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/mvnw-corrupt-test --output=/tmp/ --analysis-level=2", javaHomePath, codeanalyzerVersion ) ); @@ -199,7 +199,7 @@ void shouldBeAbleToGenerateAnalysisArtifactForDaytrader8() throws Exception { var runCodeAnalyzerOnDaytrader8 = mavenContainer.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/daytrader8 --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/daytrader8 --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -214,7 +214,7 @@ void shouldBeAbleToDetectCRUDOperationsAndQueriesForPlantByWebsphere() throws Ex var runCodeAnalyzerOnPlantsByWebsphere = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/plantsbywebsphere --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/plantsbywebsphere --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -307,7 +307,7 @@ void symbolTableShouldHaveRecords() throws IOException, InterruptedException { var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/record-class-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/record-class-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -324,7 +324,7 @@ void symbolTableShouldHaveDefaultRecordComponents() throws IOException, Interrup var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/record-class-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/record-class-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -354,7 +354,7 @@ void parametersInCallableMustHaveStartAndEndLineAndColumns() throws IOException, var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/record-class-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/record-class-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -389,7 +389,7 @@ void mustBeAbleToResolveInitializationBlocks() throws IOException, InterruptedEx var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/init-blocks-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/init-blocks-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); @@ -420,7 +420,7 @@ void mustBeAbleToExtractCommentBlocks() throws IOException, InterruptedException var runCodeAnalyzerOnCallGraphTest = container.execInContainer( "bash", "-c", String.format( - "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --input=/test-applications/init-blocks-test --analysis-level=1", + "export JAVA_HOME=%s && java -jar /opt/jars/codeanalyzer-%s.jar --schema v1 --input=/test-applications/init-blocks-test --analysis-level=1", javaHomePath, codeanalyzerVersion ) ); From 758ba70f65be61d32080d2b40683e070145324f7 Mon Sep 17 00:00:00 2001 From: George Safta Date: Sat, 24 Oct 2026 19:00:09 +0300 Subject: [PATCH 02/11] docs(design): spec for --workspace-root repo-relative file keys --- .../design/specs/2026-10-24-workspace-root.md | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 docs/design/specs/2026-10-24-workspace-root.md diff --git a/docs/design/specs/2026-10-24-workspace-root.md b/docs/design/specs/2026-10-24-workspace-root.md new file mode 100644 index 00000000..b896757f --- /dev/null +++ b/docs/design/specs/2026-10-24-workspace-root.md @@ -0,0 +1,75 @@ +# `--workspace-root`: repo-relative file keys + +**Status:** accepted · 2026-10-24 · tracks _(issue to be filed)_ + +## Problem + +Every v2 file key is relative to `--input`. When the analyzed project sits inside a larger +repository (`services/orders/` in a monorepo), keys read `src/main/java/.../App.java` rather than +the repository path `services/orders/src/main/java/.../App.java`. Tooling that speaks repository +paths — git, IDEs, PR diffs, scanners — cannot join on them without knowing and re-adding the +prefix. + +## Contract-impact triage + +| Question | Answer | +| --- | --- | +| Changes `analysis.json` v2 shape? | No. No node, edge or field is added, removed or renamed. | +| Changes v2 id/key meaning? | Yes, opt-in: with `--workspace-root` the `` segment is relative to the workspace root instead of `--input`. Default output is byte-identical. | +| Changes CLI? | Yes — one new optional flag. Additive → **minor, 3.4.0**. | +| Repos touched | `codeanalyzer-java` (this spec's PR). | + +## Decisions (user-approved 2026-10-24) + +1. **New optional flag `--workspace-root `.** When set, every path-derived v2 key is computed + relative to it rather than to `--input`: + - `application.symbol_table` keys (`L1Extractor.fileKey`); + - `module.id` and every durable id below it (`can:///java//…`), since they are built + from the file key; + - artifact keys and ids (`can:///artifact/`, `ArtifactDiscovery`). + + Unset, behavior is unchanged: the root is `--input`. +2. **The app segment does not move.** `can://` stays `--app-name`, else the `--input` + directory name. Only the `` segment changes, so the L1 cache key, the Neo4j `can:///` + prefix wipe/prune and `@external` ids keep their current scoping. +3. **`--input` must be inside the workspace root.** Both are resolved to absolute, normalized + paths; if `--input` is not the root or a descendant of it, exit non-zero at argument validation. + This keeps the schema invariant intact: keys are relative, never absolute, never `..`. +4. **v2 only.** v1 keys are absolute paths, so there is nothing to relativize. `--workspace-root` + with `--schema v1` is a hard argument error. +5. **No new schema field.** Consumers read source text from `module.source` and never resolve a + key against the filesystem, so the output need not record which root keys are relative to. +6. **Language-neutral semantics.** Artifact and `@external` ids are deliberately shared across + sibling analyzers over the same ``. Sibling analyzers that adopt a workspace root are + expected to use these same semantics (decisions 1–4), so a repository-relative artifact id is + the same node whichever analyzer emitted it. + +## Implementation notes + +- Thread one `keyRoot` path (workspace root if set, else `--input`) to the two relativization + sites: `L1Extractor.fileKey` (via `extractAll`) and `ArtifactDiscovery.discover`. Source + discovery, builds and dependency download still run against `--input`. +- `CanId.moduleId` strips leading `[./]+` from the key. Decision 3 makes `..` keys unreachable; + pin that with a test rather than relying on the strip. +- `L3WalaOverlays.deriveApplicationId` recovers the app id by stripping the file key off a module + id; it is key-shape agnostic, but cover it with a nested-root fixture. +- Incremental cache: keys change when the flag is toggled, so a cache written without the flag + simply misses for a run with it. No migration; document it. + +## Verification + +- Unit: nested fixture (`/repo/services/app/…`) run with `--workspace-root /repo` — + every `symbol_table` key, module id and artifact id starts with `services/app/`; no key is + absolute or contains `..`. +- Unit: without the flag, output is byte-identical to today's. +- CLI: `--input` outside the root → non-zero exit with a clear message; `--workspace-root` with + `--schema v1` → non-zero exit. +- L2/L3 on the nested fixture: call-graph `src`/`dst` and L3 overlay ids carry the same + repo-relative module ids. +- Neo4j: an incremental push with the flag wipes/prunes only that app's nodes. + +## Release plan + +Ships in **codeanalyzer-java 3.4.0** as an opt-in flag; default output is unchanged, so no +consumer needs to move in lockstep. README documents the flag, the repo-relative keys it produces, +and the cache miss on toggling it. From d16653b5ac6df8ff55cf2a4a177e568c768e04b2 Mon Sep 17 00:00:00 2001 From: George Safta Date: Sat, 24 Oct 2026 19:08:26 +0300 Subject: [PATCH 03/11] docs(design): link --workspace-root spec to #274 --- docs/design/specs/2026-10-24-workspace-root.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/specs/2026-10-24-workspace-root.md b/docs/design/specs/2026-10-24-workspace-root.md index b896757f..7233e26b 100644 --- a/docs/design/specs/2026-10-24-workspace-root.md +++ b/docs/design/specs/2026-10-24-workspace-root.md @@ -1,6 +1,6 @@ # `--workspace-root`: repo-relative file keys -**Status:** accepted · 2026-10-24 · tracks _(issue to be filed)_ +**Status:** accepted · 2026-10-24 · tracks [#274](https://github.com/codellm-devkit/codeanalyzer-java/issues/274) ## Problem From d41cea2ffa76dbed47b3ca50ebb189e575bddc80 Mon Sep 17 00:00:00 2001 From: George Safta Date: Sat, 24 Oct 2026 21:02:21 +0300 Subject: [PATCH 04/11] feat: --workspace-root makes file keys and ids repo-relative (#274) With --workspace-root, symbol_table keys, module ids and every id below them, and artifact keys/paths/ids are relative to the workspace root instead of --input. The app segment, discovery, builds and artifact classification are unchanged. --input outside the root, or the flag with --schema v1, is an argument error. --- README.md | 17 +++ src/main/java/com/ibm/cldk/CodeAnalyzer.java | 33 ++++- .../ibm/cldk/artifacts/ArtifactDiscovery.java | 20 ++- .../cldk/syntactic_analysis/L1Extractor.java | 21 ++- .../com/ibm/cldk/CodeAnalyzerV2CliTest.java | 120 ++++++++++++++++ .../ibm/cldk/WorkspaceRootNeo4jPushTest.java | 135 ++++++++++++++++++ .../java/com/ibm/cldk/schema/CanIdTest.java | 11 ++ 7 files changed, 344 insertions(+), 13 deletions(-) create mode 100644 src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java diff --git a/README.md b/README.md index 261f1cb5..4ca3ccbf 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ Usage: codeanalyzer [-hvV] [--no-build] [--no-clean-dependencies] [-a=] [-b=] [-f=] [-i=] [-o=] [-s=] [--emit=] [--app-name=] + [--workspace-root=] [--neo4j-uri=] [--neo4j-user=] [--neo4j-password=] [--neo4j-database=] [-t=]... @@ -145,6 +146,9 @@ Analyze java application. schema (the Neo4j schema.neo4j.json contract). --app-name= Logical application name for the graph :JApplication anchor (default: input dir name). + --workspace-root= Directory that file keys, module ids and artifact + ids are relative to (default: --input). Must + contain --input. Not supported with --schema v1. --neo4j-uri= Push the graph to a live Neo4j over Bolt (incremental); omit to write graph.cypher. Falls back to the NEO4J_URI environment variable. @@ -157,6 +161,19 @@ Analyze java application. -V, --version Print version information and exit. ``` +**Repository-relative paths.** By default, `symbol_table` keys and the ids built from them are relative to +`--input`. When the project sits inside a larger repository, pass `--workspace-root` to make them +repository-relative instead: + +```bash +codeanalyzer -i monorepo/services/orders --workspace-root monorepo +# symbol_table key: services/orders/src/main/java/com/acme/Order.java +``` + +Only the path part of ids changes. The application id (`can://`) stays the same. Keys +change when you toggle the flag, so a `--cache-dir` written without it is not reused by a run with +it, and the other way round. + ## 3. Installing `codeanalyzer` as a native binary (once built, no JVM will be required for running `codeanalyzer`) diff --git a/src/main/java/com/ibm/cldk/CodeAnalyzer.java b/src/main/java/com/ibm/cldk/CodeAnalyzer.java index bc57b87b..412f5a90 100644 --- a/src/main/java/com/ibm/cldk/CodeAnalyzer.java +++ b/src/main/java/com/ibm/cldk/CodeAnalyzer.java @@ -146,6 +146,11 @@ public class CodeAnalyzer implements Runnable { // values between CommandLine instances in the same JVM. New flags do not add to that. private String schema = "v2"; + @Option(names = { + "--workspace-root" }, description = "Directory that file keys, module ids and artifact ids are " + + "relative to (default: --input). Must contain --input. Not supported with --schema v1.") + private String workspaceRoot; + @Option(names = {"-c", "--cache-dir" }, description = "Directory holding the incremental analysis cache. When set, " + "unchanged files are reused from analysis_cache.json instead of being reparsed.") @@ -261,6 +266,11 @@ private void analyze() throws Exception { analyzeV2(); return; } + if (workspaceRoot != null) { + // v1 keys are absolute paths, so there is nothing for a workspace root to relativize. + throw new ParameterException(spec.commandLine(), + "error: --workspace-root is not supported with --schema v1"); + } JsonObject combinedJsonObject = new JsonObject(); Map symbolTable; @@ -399,9 +409,11 @@ private boolean isV2Schema() { * which {@code --artifact-text}/{@code --artifact-text-max-bytes} may have emptied or truncated — * delegates to {@link DependencyView#readFromDisk} rather than re-reading the file itself, so that * from-disk logic exists exactly once in this codebase. + * + * @param keyRoot the directory artifact paths are relative to (the workspace root, else --input) */ - private static String readFully(JArtifact artifact) { - return DependencyView.readFromDisk(Paths.get(input), artifact.getPath()); + private static String readFully(Path keyRoot, JArtifact artifact) { + return DependencyView.readFromDisk(keyRoot, artifact.getPath()); } /** @@ -455,6 +467,15 @@ private void analyzeV2() throws Exception { if (input == null) { throw new ParameterException(spec.commandLine(), "error: --input is required"); } + // What every emitted file key, module id and artifact id is relative to. Discovery, builds and + // dependency download still run against --input; only the key changes. + Path inputRoot = Paths.get(input).toAbsolutePath().normalize(); + Path keyRoot = workspaceRoot == null ? inputRoot : Paths.get(workspaceRoot).toAbsolutePath().normalize(); + if (!inputRoot.startsWith(keyRoot)) { + // Relativizing would yield `..` keys, which the schema forbids (relative, never absolute or `..`). + throw new ParameterException(spec.commandLine(), + "error: --input " + inputRoot + " is not inside --workspace-root " + keyRoot); + } String application = appName != null && !appName.isBlank() ? appName @@ -497,7 +518,7 @@ private void analyzeV2() throws Exception { List degraded = new ArrayList<>(); try { modules = L1Extractor.extractAll( - Paths.get(input), application, dependencyDir, cached, + Paths.get(input), keyRoot, application, dependencyDir, cached, analysisLevel, graphFieldDepth, l3Engine, entrypointReport); // The WALA L3 engine needs the RTA build; --no-rta suppresses it, so a level-3 wala run with // --no-rta would silently carry no overlays. Warn rather than degrade without a signal. @@ -567,13 +588,13 @@ private void analyzeV2() throws Exception { // -- but unlike them it is L1 data and runs at EVERY analysis level, so it is computed here, // ahead of the level gate, rather than inside either branch of it. Map artifacts = - ArtifactDiscovery.discover(Paths.get(input), application, artifactText, artifactTextMaxBytes); - List dependencies = DependencyView.build(Paths.get(input), artifacts); + ArtifactDiscovery.discover(inputRoot, keyRoot, application, artifactText, artifactTextMaxBytes); + List dependencies = DependencyView.build(keyRoot, artifacts); for (JArtifact a : artifacts.values()) { if (ConfigKeys.isEligible(a)) { // Re-read from disk: `source` may be truncated or suppressed, and extraction // must not silently degrade with a capture flag. - ConfigKeys.Result r = ConfigKeys.extract(a, readFully(a), artifactText); + ConfigKeys.Result r = ConfigKeys.extract(a, readFully(keyRoot, a), artifactText); a.setConfigKeys(r.keys); // A pre-existing "partial" from the dependency pass is never overwritten. if (!r.ok) { diff --git a/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java b/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java index 755977b4..909ae48b 100644 --- a/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java +++ b/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java @@ -136,7 +136,20 @@ private static final class Rule { */ public static Map discover( Path projectDir, String appName, boolean captureText, int textMaxBytes) throws IOException { + return discover(projectDir, projectDir, appName, captureText, textMaxBytes); + } + + /** + * As above, with each artifact's key, {@code path} and id relative to {@code keyRoot} rather than + * {@code projectDir} — {@code --workspace-root}. Ignore rules and classification still see the + * {@code projectDir}-relative path, so a rule like {@code gradle/libs.versions.toml} matches the + * same files whatever the key root is. {@code projectDir} must be {@code keyRoot} or inside it. + */ + public static Map discover( + Path projectDir, Path keyRoot, String appName, boolean captureText, int textMaxBytes) + throws IOException { Path root = projectDir.toAbsolutePath().normalize(); + Path keyBase = keyRoot.toAbsolutePath().normalize(); List files; try (Stream walk = Files.walk(root)) { files = walk.filter(Files::isRegularFile).sorted().collect(Collectors.toList()); @@ -172,9 +185,10 @@ public static Map discover( roles = (!name.contains(".") && text.startsWith("#!")) ? List.of("script") : List.of("unknown"); } + String keyPosix = keyBase.relativize(file).toString().replace('\\', '/'); JArtifact artifact = new JArtifact(); - artifact.setId(CanId.artifactId(appName, relPosix)); - artifact.setPath(relPosix); + artifact.setId(CanId.artifactId(appName, keyPosix)); + artifact.setPath(keyPosix); artifact.setFormat(format); artifact.setRoles(roles); // sha256/sizeBytes always describe the whole file, independent of capture/truncation @@ -192,7 +206,7 @@ public static Map discover( } } - artifacts.put(relPosix, artifact); + artifacts.put(keyPosix, artifact); } return artifacts; } diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java index ecf72e8e..cb431c9e 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java @@ -121,6 +121,19 @@ public static Map extractAll( Path projectRoot, String appName, Path dependencyDir, Map cached, int analysisLevel, int graphFieldDepth, String l3Engine, JEntrypointReport entrypointReport) throws IOException { + return extractAll(projectRoot, projectRoot, appName, dependencyDir, cached, analysisLevel, + graphFieldDepth, l3Engine, entrypointReport); + } + + /** + * As above, with file keys (and so every module id below them) relative to {@code keyRoot} rather + * than {@code projectRoot} — {@code --workspace-root}. Source discovery still runs over + * {@code projectRoot}, which the caller guarantees is {@code keyRoot} or inside it. + */ + public static Map extractAll( + Path projectRoot, Path keyRoot, String appName, Path dependencyDir, Map cached, + int analysisLevel, int graphFieldDepth, String l3Engine, JEntrypointReport entrypointReport) + throws IOException { // JavaParser's collection strategy silently finds zero source roots for a path with a `.` // element (`-i .` arrives here as exactly that), so normalize before discovery. projectRoot = projectRoot.toAbsolutePath().normalize(); @@ -145,7 +158,7 @@ public static Map extractAll( int reused = 0; for (SourceRoot sourceRoot : sourceRoots) { for (Path path : javaFilesUnder(sourceRoot.getRoot())) { - String fileKey = fileKey(projectRoot, path); + String fileKey = fileKey(keyRoot, path); // Read the file's own text rather than printing the AST: `span.bytes` must index the // real file, byte for byte. String source = Files.readString(path, StandardCharsets.UTF_8); @@ -249,9 +262,9 @@ private static CombinedTypeSolver typeSolver( return solver; } - /** The {@code symbol_table} key: path relative to the project root, always {@code /}-separated. */ - private static String fileKey(Path projectRoot, Path file) { - Path relative = projectRoot.toAbsolutePath().normalize().relativize(file.toAbsolutePath().normalize()); + /** The {@code symbol_table} key: path relative to the key root, always {@code /}-separated. */ + private static String fileKey(Path keyRoot, Path file) { + Path relative = keyRoot.toAbsolutePath().normalize().relativize(file.toAbsolutePath().normalize()); return relative.toString().replace('\\', '/'); } diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java index 8a82ff18..361b866c 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java @@ -791,4 +791,124 @@ void viewDispatchesReachBothProjections(@TempDir Path root) throws Exception { assertEquals(1, rows, "one J_DISPATCHES_TO row: the resolved forward, and not the unresolved one"); assertTrue(script.contains("via: 'forward'"), "the row carries its mechanism"); } + + // ---- --workspace-root ----------------------------------------------------------------------- + + /** Every key in {@code keys} is repo-relative under {@code prefix}: never absolute, never {@code ..}. */ + private static void assertRepoRelative(Set keys, String prefix) { + assertFalse(keys.isEmpty(), "precondition: something was emitted"); + for (String key : keys) { + assertTrue(key.startsWith(prefix), "expected the " + prefix + " prefix, got: " + key); + assertFalse(key.startsWith("/") || key.contains(".."), "keys stay relative: " + key); + } + } + + @Test + void workspaceRootMakesSymbolTableKeysAndIdsRepoRelative(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path in = project(repo.resolve("services/app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "-a", "2", + "--workspace-root", repo.toString(), "--app-name", "widgets")); + + JsonObject app = application(out); + assertEquals(CanId.applicationId("widgets"), app.get("id").getAsString(), + "the app segment does not move with the workspace root"); + JsonObject symbolTable = app.getAsJsonObject("symbol_table"); + assertRepoRelative(symbolTable.keySet(), "services/app/"); + String key = "services/app/src/main/java/com/example/Widget.java"; + assertTrue(symbolTable.has(key), "got: " + symbolTable.keySet()); + assertEquals(CanId.moduleId(CanId.applicationId("widgets"), key), + symbolTable.getAsJsonObject(key).get("id").getAsString()); + + String moduleId = CanId.moduleId(CanId.applicationId("widgets"), key); + JsonArray callGraph = app.getAsJsonArray("call_graph"); + assertFalse(callGraph.isEmpty(), "precondition: size() -> count() is an in-project edge"); + for (JsonElement e : callGraph) { + assertTrue(e.getAsJsonObject().get("src").getAsString().startsWith(moduleId + "/"), + "call-graph ids carry the repo-relative module id: " + e); + } + assertConformsToV2Schema(out); + } + + @Test + void workspaceRootCarriesIntoLevelThreeOverlayIds(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path in = project(repo.resolve("services/app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "-a", "3", + "--workspace-root", repo.toString(), "--app-name", "widgets")); + + String moduleId = CanId.moduleId(CanId.applicationId("widgets"), + "services/app/src/main/java/com/example/Widget.java"); + String json = Files.readString(out.resolve("analysis.json")); + assertTrue(json.contains("\"cfg\""), "precondition: level 3 lays the cfg overlay"); + java.util.regex.Matcher ids = java.util.regex.Pattern.compile("\"(can://widgets/java/[^\"]*)\"").matcher(json); + int seen = 0; + while (ids.find()) { + assertTrue(ids.group(1).startsWith(moduleId), + "every code id, overlay endpoints included, is under the repo-relative module: " + ids.group(1)); + seen++; + } + assertTrue(seen > 0, "precondition: ids were emitted"); + assertConformsToV2Schema(out); + } + + @Test + void workspaceRootPrefixesArtifactKeysWithoutChangingClassification(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path in = artifactProject(repo.resolve("services/app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), + "--workspace-root", repo.toString(), "--app-name", "widgets")); + + JsonObject app = application(out); + JsonObject artifacts = app.getAsJsonObject("artifacts"); + assertRepoRelative(artifacts.keySet(), "services/app/"); + + JsonObject pom = artifacts.getAsJsonObject("services/app/pom.xml"); + assertNotNull(pom, "got: " + artifacts.keySet()); + assertEquals("services/app/pom.xml", pom.get("path").getAsString()); + assertEquals(CanId.artifactId("widgets", "services/app/pom.xml"), pom.get("id").getAsString()); + assertEquals("full", pom.get("extraction").getAsString(), + "the manifest is still read from disk under the repo-relative key"); + assertFalse(app.getAsJsonArray("dependencies").isEmpty(), "and its dependency still extracted"); + + JsonArray propsKeys = artifacts.getAsJsonObject("services/app/application.properties") + .getAsJsonArray("config_keys"); + assertEquals(1, propsKeys.size(), "config keys are still extracted"); + assertConformsToV2Schema(out); + } + + @Test + void workspaceRootEqualToInputChangesNothing(@TempDir Path tmp) throws IOException { + Path in = artifactProject(tmp.resolve("app")); + Path without = tmp.resolve("without"); + Path with = tmp.resolve("with"); + assertEquals(0, run("-i", in.toString(), "-o", without.toString(), "--app-name", "widgets")); + assertEquals(0, run("-i", in.toString(), "-o", with.toString(), "--app-name", "widgets", + "--workspace-root", in.toString())); + assertEquals(application(without).get("symbol_table").toString(), + application(with).get("symbol_table").toString()); + assertEquals(application(without).get("artifacts").toString(), + application(with).get("artifacts").toString()); + } + + @Test + void workspaceRootMustContainTheInput(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path elsewhere = Files.createDirectories(tmp.resolve("elsewhere")); + assertNotEquals(0, run("-i", in.toString(), "-o", tmp.resolve("out").toString(), + "--workspace-root", elsewhere.toString()), + "an input outside the workspace root would need absolute or .. keys"); + assertFalse(Files.exists(tmp.resolve("out/analysis.json")), "and nothing is written"); + } + + @Test + void workspaceRootIsRejectedWithSchemaV1(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + assertNotEquals(0, run("-i", in.toString(), "-o", tmp.resolve("out").toString(), + "--schema", "v1", "--workspace-root", tmp.toString()), + "v1 keys are absolute: the flag has nothing to relativize"); + } } diff --git a/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java b/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java new file mode 100644 index 00000000..c08bb580 --- /dev/null +++ b/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java @@ -0,0 +1,135 @@ +package com.ibm.cldk; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ibm.cldk.schema.CanId; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.api.io.TempDir; +import org.neo4j.driver.AuthTokens; +import org.neo4j.driver.Driver; +import org.neo4j.driver.GraphDatabase; +import org.neo4j.driver.Session; +import org.neo4j.driver.Values; +import org.testcontainers.containers.Neo4jContainer; + +/** + * {@code --workspace-root} against a live Neo4j: the incremental push's wipe and orphan prune are + * scoped by the app's {@code can:///} prefix and each module's own id, so repo-relative module + * ids must still be pruned when their file vanishes, and must not reach another application's graph. + * + *

Needs a container runtime, so it is opt-in like {@code Neo4jBoltWriterTest}: it runs only when + * {@code RUN_CONTAINER_TESTS} is set. + */ +@EnabledIfEnvironmentVariable(named = "RUN_CONTAINER_TESTS", matches = ".+") +class WorkspaceRootNeo4jPushTest { + + private static final String PASSWORD = "testpassword123"; + + @SuppressWarnings("resource") + private static final Neo4jContainer CONTAINER = + new Neo4jContainer<>("neo4j:5").withAdminPassword(PASSWORD); + + private static Driver driver; + + @BeforeAll + static void startup() { + CONTAINER.start(); + driver = GraphDatabase.driver(CONTAINER.getBoltUrl(), AuthTokens.basic("neo4j", PASSWORD)); + } + + @AfterAll + static void teardown() { + if (driver != null) { + driver.close(); + } + CONTAINER.stop(); + } + + /** The CLI's pre-existing options are static; reset the ones this test sets, before and after. */ + @BeforeEach + @AfterEach + void resetStaticOptions() throws Exception { + set("emit", "json"); + set("analysisLevel", 1); + set("output", null); + set("input", null); + set("appName", null); + set("neo4jUri", null); + set("neo4jUser", null); + set("neo4jPassword", null); + CodeAnalyzer.projectRootPom = null; + } + + private static void set(String field, Object value) throws Exception { + Field f = CodeAnalyzer.class.getDeclaredField(field); + f.setAccessible(true); + f.set(null, value); + } + + private static void writeClass(Path srcRoot, String name) throws IOException { + Path pkg = Files.createDirectories(srcRoot.resolve("src/main/java/com/example")); + Files.writeString(pkg.resolve(name + ".java"), + "package com.example;\npublic class " + name + " {\n public int m() { return 1; }\n}\n", + StandardCharsets.UTF_8); + } + + private static int push(Path in, Path repo, String app, Path out) { + return new picocli.CommandLine(new CodeAnalyzer()).execute( + "-i", in.toString(), "-o", out.toString(), "--emit", "neo4j", "--no-build", + "--workspace-root", repo.toString(), "--app-name", app, + "--neo4j-uri", CONTAINER.getBoltUrl(), "--neo4j-user", "neo4j", "--neo4j-password", PASSWORD); + } + + private static long count(String cypher, Object... params) { + try (Session s = driver.session()) { + return s.run(cypher, Values.parameters(params)).single().get(0).asLong(); + } + } + + @Test + void aVanishedRepoRelativeModuleIsPrunedAndOtherAppsAreUntouched(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path orders = repo.resolve("services/orders"); + Path billing = repo.resolve("services/billing"); + writeClass(orders, "Keep"); + writeClass(orders, "Gone"); + writeClass(billing, "Gone"); + + assertEquals(0, push(orders, repo, "orders", tmp.resolve("o1"))); + assertEquals(0, push(billing, repo, "billing", tmp.resolve("o2"))); + + String key = "services/orders/src/main/java/com/example/"; + String gone = CanId.moduleId(CanId.applicationId("orders"), key + "Gone.java"); + String keep = CanId.moduleId(CanId.applicationId("orders"), key + "Keep.java"); + String billingGone = CanId.moduleId(CanId.applicationId("billing"), + "services/billing/src/main/java/com/example/Gone.java"); + assertEquals(1, count("MATCH (m:JModule {id: $id}) RETURN count(m)", "id", gone), + "precondition: the module is pushed under its repo-relative id"); + long billingNodes = count("MATCH (n) WHERE n.id STARTS WITH $p RETURN count(n)", + "p", CanId.applicationId("billing") + "/"); + assertTrue(billingNodes > 0, "precondition: the other app is in the same database"); + + Files.delete(orders.resolve("src/main/java/com/example/Gone.java")); + assertEquals(0, push(orders, repo, "orders", tmp.resolve("o3"))); + + assertEquals(0, count("MATCH (n) WHERE n.id = $id OR n.id STARTS WITH $p RETURN count(n)", + "id", gone, "p", gone + "/"), "the vanished module and its declarations are pruned"); + assertEquals(1, count("MATCH (m:JModule {id: $id}) RETURN count(m)", "id", keep), + "the surviving module stays"); + assertEquals(1, count("MATCH (m:JModule {id: $id}) RETURN count(m)", "id", billingGone), + "a same-named file in another app is a different node and is not touched"); + assertEquals(billingNodes, count("MATCH (n) WHERE n.id STARTS WITH $p RETURN count(n)", + "p", CanId.applicationId("billing") + "/"), "the other app's graph is unchanged"); + } +} diff --git a/src/test/java/com/ibm/cldk/schema/CanIdTest.java b/src/test/java/com/ibm/cldk/schema/CanIdTest.java index 963a9d54..38f0cb47 100644 --- a/src/test/java/com/ibm/cldk/schema/CanIdTest.java +++ b/src/test/java/com/ibm/cldk/schema/CanIdTest.java @@ -31,6 +31,17 @@ void moduleId_normalizesBackslashesAndLeadingDotSlash() { "can://myapp/java/a/b/C.java", CanId.moduleId(app, "./a\\b\\C.java")); } + @Test + void moduleId_keepsARepoRelativeKeyWhole() { + // --workspace-root keys carry the path from the repository root; the leading-[./] strip must + // only drop a `./`, never eat into the key's own first segment. + String app = CanId.applicationId("myapp"); + assertEquals("can://myapp/java/services/app/src/Foo.java", + CanId.moduleId(app, "services/app/src/Foo.java")); + assertEquals("can://myapp/java/services/app/src/Foo.java", + CanId.moduleId(app, "./services/app/src/Foo.java")); + } + @Test void childId_appendsSegmentWithSlash() { assertEquals( From ef3ac0e921b3cefdf148bb683b71a72ea245cdcd Mon Sep 17 00:00:00 2001 From: George Safta Date: Wed, 28 Oct 2026 13:26:58 +0200 Subject: [PATCH 05/11] docs(design): Bolt artifact reconciliation addendum to --workspace-root spec --- .../design/specs/2026-10-24-workspace-root.md | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/design/specs/2026-10-24-workspace-root.md b/docs/design/specs/2026-10-24-workspace-root.md index 7233e26b..91b6fde6 100644 --- a/docs/design/specs/2026-10-24-workspace-root.md +++ b/docs/design/specs/2026-10-24-workspace-root.md @@ -73,3 +73,92 @@ prefix. Ships in **codeanalyzer-java 3.4.0** as an opt-in flag; default output is unchanged, so no consumer needs to move in lockstep. README documents the flag, the repo-relative keys it produces, and the cache miss on toggling it. + +## Addendum: Bolt artifact reconciliation (user-approved 2026-10-28) + +### Problem + +The Bolt writer never deletes an `:Artifact` or `:ConfigKey`. They carry no `_module`, so they +are treated as shared nodes: MERGE-upserted, never purged. Toggling `--workspace-root` on an app +that already has a graph changes every artifact id (`can:///artifact/pom.xml` becomes +`can:///artifact/services/app/pom.xml`), so the old-keyed artifacts survive alongside the +new ones. Each still hangs off `:JApplication` via `HAS_ARTIFACT` and still declares its +`:Package`s. The same leak exists without the flag, for any deleted or renamed artifact, and +for the outgoing edges of an artifact whose content changed (a dependency removed from +`pom.xml` keeps its `DECLARES_DEPENDENCY` edge; a removed yaml key keeps its `:ConfigKey`). +The Cypher snapshot does not have this problem, because its `can:///` prefix wipe +rebuilds artifacts from scratch. + +### Triage + +| Question | Answer | +| --- | --- | +| Changes `analysis.json` / `schema.neo4j.json` shape? | No. Write semantics only. | +| Changes cross-analyzer graph semantics? | Yes, narrowly: a Java Bolt full run now removes Java's own claim on artifacts and config keys it no longer emits, and deletes a node only when no other analyzer claims it. Sibling-owned nodes and edges are never touched. | +| Repos touched | `codeanalyzer-java` now. `codeanalyzer-python` (`neo4j/bolt.py`) and `codeanalyzer-typescript` (`build/neo4j/bolt.ts`) have the same gap; tracked, filed when picked up. | + +### Ownership: what is Java's + +The artifact layer's vocabulary is shared (`:Artifact`, `:ConfigKey`, `:Package`, +`HAS_ARTIFACT`, `DEFINES_CONFIG`, `DECLARES_DEPENDENCY`, `LOCKS` are unprefixed and identical in +all three analyzers), but the facts each analyzer writes under it are not: Java parses Maven and +Gradle manifests and its own set of config formats; Python writes `pkg:pypi` dependencies and +toml/ini/json config keys; TypeScript writes `pkg:npm`. Ownership is recoverable without a new +property: + +| Fact | Java's when | +| --- | --- | +| an `:Artifact` / `:ConfigKey` node | it carries `:JCanNode` (every `can://` node Java writes gets it, `RowBuilder.node`) | +| `HAS_ARTIFACT` | its source is this app's `:JApplication` | +| `DECLARES_DEPENDENCY` / `LOCKS` | its target `:Package` id starts with `pkg:maven/` | +| `DEFINES_CONFIG` | shared fact; kept while any analyzer still claims the key | + +A node claimed by another analyzer carries that analyzer's marker (`:PyCanNode`, `:TSCanNode`), +so "claimed by someone else" is `any(l IN labels(x) WHERE l ENDS WITH 'CanNode' AND +l <> 'JCanNode')`. + +### Decisions + +1. **Release, don't delete.** On a full Bolt run, for every `:Artifact` under + `can:///artifact/` that carries `:JCanNode` and is not among this run's artifact ids, and + every `:ConfigKey` under that prefix that carries `:JCanNode` and is not among this run's + config-key ids: remove `:JCanNode`; if no other `*CanNode` marker remains, `DETACH DELETE` + it. A node another analyzer still claims survives with that analyzer's edges intact. An + artifact with no marker at all was not written by Java and is left alone. +2. **Java-owned artifact edges are rebuilt on every full run.** `sha256` is one property every + writer `SET`s, so a sibling's push can make a changed manifest look unchanged to Java; a sha + diff is therefore unsound on shared nodes. Instead, each full run deletes this app's + Java-owned artifact edges (`HAS_ARTIFACT` from its `:JApplication`; + `DECLARES_DEPENDENCY`/`LOCKS` from its artifacts to `pkg:maven/` packages) and re-upserts + them from the run. That also covers `LOCKS` going stale when a manifest changes and the lock + file does not. Artifact edges number tens to hundreds, so the cost is negligible. +3. **Full runs only.** Releasing and rebuilding run only when `fullRun`. v2 is always a full run + (`--target-files` is rejected), so this covers every v2 push. +4. **Ordering.** One write transaction runs release → delete → edge purge, then nodes and edges + are upserted as today. The ids to keep come from the run's rows, not the DB. +5. **Non-goals.** `:Package` nodes left with no edges are not deleted (they sit outside the app + prefix and are shared across apps). The Cypher snapshot keeps its broader wipe, which + `DETACH DELETE`s every `:JCanNode` under the app, sibling-shared artifacts included. Aligning + it with decision 1 is a recorded follow-up. + +### Verification + +- Bolt (Testcontainers), failing before the fix: push without `--workspace-root`, then push the + same project with it; assert every `:Artifact`/`:ConfigKey` under `can:///artifact/` + carries the repo-relative id, no old-keyed artifact remains, and `:JApplication` has exactly + as many `HAS_ARTIFACT` edges as the run emitted. +- Bolt: change a manifest (drop a dependency, drop a config key); re-push; assert the dropped + `DECLARES_DEPENDENCY` edge and `:ConfigKey` are gone. +- Bolt, polyglot: seed a sibling-shaped artifact (`:Artifact:PyCanNode`, a `pkg:pypi` + `DECLARES_DEPENDENCY`, a `:ConfigKey:PyCanNode`) on an id Java also emits and on one it does + not; after Java's re-push both survive with their sibling edges, and the shared one keeps + `:PyCanNode`. +- Bolt: a second application in the same database keeps all its artifacts. + +### Docs + +README §4.2: the Bolt writer reconciles this app's artifacts on a full run. It releases Java's +claim on artifacts and config keys it no longer emits, deletes them only when no other analyzer +claims them, and rebuilds Java-owned artifact edges. §4.1: state that the snapshot's wipe is +broader. Update the `BoltWriter` / `CypherWriter.DESCENDANTS` comments that say Bolt never +deletes artifacts. Releases with 3.4.0; no lockstep with siblings. From 90a48ca82a3a8011f9d58d9e167e60042df206b5 Mon Sep 17 00:00:00 2001 From: George Safta Date: Thu, 29 Oct 2026 01:15:41 +0200 Subject: [PATCH 06/11] add artifact removal/update strategy --- README.md | 15 +- .../design/specs/2026-10-24-workspace-root.md | 35 +- .../java/com/ibm/cldk/neo4j/BoltWriter.java | 97 ++++- .../java/com/ibm/cldk/neo4j/CypherWriter.java | 3 +- .../ibm/cldk/WorkspaceRootNeo4jPushTest.java | 351 ++++++++++++++++++ 5 files changed, 476 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 4ca3ccbf..ad332c9e 100644 --- a/README.md +++ b/README.md @@ -320,8 +320,7 @@ containment traversal that also replaces a prior v1 (pre-`3.0.0`) graph of the s That scope **includes** this application's `:Artifact` and `:ConfigKey` nodes — the snapshot rebuilds them, rather than leaving stale ones to accumulate with nothing ever cleaning them up. The one consequence: a sibling-language analyzer's edge into a shared `:Artifact` is dropped by a Java -snapshot and restored on that analyzer's next push. The Bolt writer does **not** do this — its -deletions are scoped per module, not per application. +snapshot and restored on that analyzer's next push. The Bolt writer is narrower; see §4.2. ### 4.2. Live incremental push over Bolt @@ -335,6 +334,18 @@ compilation unit's `content_hash`, replaces just the changed units' subgraphs (i `MERGE` upserts), and — on a full run — prunes units whose source file vanished. Combine with `--target-files` for a targeted, partial re-push (orphan pruning is then skipped). +On a full run it also reconciles the application's artifacts, so a deleted or renamed file, a +removed dependency, or re-keying under `--workspace-root` leaves nothing stale: + +- Java-owned artifact edges are rebuilt from the run: `HAS_ARTIFACT`, `DECLARES_DEPENDENCY`/`LOCKS` + to `pkg:maven/…`, and code reads of them (`J_USES_CONFIG`, `J_DISPATCHES_TO`, + `J_READS_CONFIG_UNRESOLVED`), so a config-only change updates them too. +- An `:Artifact`/`:ConfigKey` the run no longer emits loses its `:JCanNode` label. An artifact is + then deleted only if no other analyzer's `HAS_ARTIFACT` still points at it; a config key only if + no other analyzer's marker (`:PyCanNode`, `:TSCanNode`) still claims it. A sibling's edges are + never removed. +- Artifacts written before 3.1.1 (`can://artifact//…`) are reconciled the same way. + ### 4.3. Schema contract ```sh diff --git a/docs/design/specs/2026-10-24-workspace-root.md b/docs/design/specs/2026-10-24-workspace-root.md index 91b6fde6..8c9f792e 100644 --- a/docs/design/specs/2026-10-24-workspace-root.md +++ b/docs/design/specs/2026-10-24-workspace-root.md @@ -119,24 +119,37 @@ l <> 'JCanNode')`. ### Decisions -1. **Release, don't delete.** On a full Bolt run, for every `:Artifact` under - `can:///artifact/` that carries `:JCanNode` and is not among this run's artifact ids, and - every `:ConfigKey` under that prefix that carries `:JCanNode` and is not among this run's - config-key ids: remove `:JCanNode`; if no other `*CanNode` marker remains, `DETACH DELETE` - it. A node another analyzer still claims survives with that analyzer's edges intact. An - artifact with no marker at all was not written by Java and is left alone. +1. **Release, don't delete.** On a full Bolt run, Java first drops its own edges, then releases + `:JCanNode` from each `:JCanNode` artifact/config key under `can:///artifact/` (and the + pre-3.1.1 `can://artifact//`) that the run no longer emits. Deletion then depends on the + node type: + - **Artifact:** deleted when no `HAS_ARTIFACT` edge is left. Every analyzer emits `HAS_ARTIFACT` + from its own app root for every artifact it inventories, and has since it first wrote + artifacts, so a remaining edge means another analyzer still holds it. This works whether or + not that analyzer marks the node. + - **Config key:** deleted with its artifact, unless another `*CanNode` label still claims the + key. A vanished key on a surviving artifact is also deleted when no other `*CanNode` label + claims it. A sibling version that does not mark keys loses such a key until its next push. + + A node that never carried `:JCanNode` was not written by Java and is not matched. 2. **Java-owned artifact edges are rebuilt on every full run.** `sha256` is one property every writer `SET`s, so a sibling's push can make a changed manifest look unchanged to Java; a sha diff is therefore unsound on shared nodes. Instead, each full run deletes this app's Java-owned artifact edges (`HAS_ARTIFACT` from its `:JApplication`; `DECLARES_DEPENDENCY`/`LOCKS` from its artifacts to `pkg:maven/` packages) and re-upserts them from the run. That also covers `LOCKS` going stale when a manifest changes and the lock - file does not. Artifact edges number tens to hundreds, so the cost is negligible. -3. **Full runs only.** Releasing and rebuilding run only when `fullRun`. v2 is always a full run + file does not. Artifact edges number tens to hundreds, so the cost is negligible. Java's + code-to-artifact edges (`J_USES_CONFIG`, `J_DISPATCHES_TO`, `J_READS_CONFIG_UNRESOLVED`) are + rebuilt the same way, because a config or template change alters them while every module's + `content_hash` stays the same. +3. **Batched.** The release runs as `CALL { … } IN TRANSACTIONS`: a toggle on a large repository + releases one node per non-source file, which a single transaction cannot hold. Partial failure + is safe, since the next full push finishes the reconciliation. +4. **Full runs only.** Releasing and rebuilding run only when `fullRun`. v2 is always a full run (`--target-files` is rejected), so this covers every v2 push. -4. **Ordering.** One write transaction runs release → delete → edge purge, then nodes and edges - are upserted as today. The ids to keep come from the run's rows, not the DB. -5. **Non-goals.** `:Package` nodes left with no edges are not deleted (they sit outside the app +5. **Ordering.** Edge purge, then release/delete, then nodes and edges are upserted as today. + The ids to keep come from the run's rows, not the DB. +6. **Non-goals.** `:Package` nodes left with no edges are not deleted (they sit outside the app prefix and are shared across apps). The Cypher snapshot keeps its broader wipe, which `DETACH DELETE`s every `:JCanNode` under the app, sibling-shared artifacts included. Aligning it with decision 1 is a recorded follow-up. diff --git a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java index 9ade4914..63685f58 100644 --- a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java @@ -44,6 +44,8 @@ *

  • on a FULL run only, prune units whose source file vanished.
    • * * + *

    A full run also reconciles the app's artifacts; see {@link #RELEASE_VANISHED_ARTIFACTS}. + * *

    Nodes are MERGE-upserted, never blindly deleted, so a declaration another (unchanged) unit * still references survives and its incoming edges stay valid. {@code :JPackage}/{@code :JAnnotation} * are shared (no {@code _module}) and are MERGE-only. @@ -96,6 +98,52 @@ public final class BoltWriter implements BoltSink { + "OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x) " + "DETACH DELETE x, c RETURN count(c) AS pruned"; + /** + * Drop every Java-owned edge on this app's artifacts and config keys so the run rebuilds them: + * {@code HAS_ARTIFACT} from a {@code :JApplication}, {@code DECLARES_DEPENDENCY}/{@code LOCKS} + * to {@code pkg:maven/}, and incoming {@code J_} edges ({@code J_USES_CONFIG}, + * {@code J_DISPATCHES_TO}). Not sha-diffed: siblings overwrite the shared {@code sha256}, and a + * config change alters code edges while every module's hash stays put. + */ + static final String PURGE_ARTIFACT_EDGES = + "UNWIND $pres AS pre " + + "MATCH (t:" + RowBuilder.CAN_NODE + ") WHERE t.id STARTS WITH pre AND (t:Artifact OR t:ConfigKey) " + + "CALL { WITH t " + + "WITH [(t)<-[r]-(s) WHERE (type(r) = 'HAS_ARTIFACT' AND s:JApplication) " + + "OR type(r) STARTS WITH 'J_' | r] " + + "+ [(t)-[d:DECLARES_DEPENDENCY|LOCKS]->(p:Package) WHERE p.id STARTS WITH 'pkg:maven/' | d] AS rs " + + "FOREACH (r IN rs | DELETE r) } IN TRANSACTIONS OF 1000 ROWS"; + + /** @see #PURGE_ARTIFACT_EDGES */ + static final String PURGE_UNRESOLVED_CONFIG_READS = + "MATCH (:JApplication {id: $appId})-[r:J_READS_CONFIG_UNRESOLVED]->() " + + "CALL { WITH r DELETE r } IN TRANSACTIONS OF 1000 ROWS"; + + /** + * Release Java's claim ({@code :JCanNode}) on artifacts/config keys the run no longer emits, + * after {@link #PURGE_ARTIFACT_EDGES}. An artifact is deleted once no {@code HAS_ARTIFACT} is + * left: every analyzer inventories its artifacts with one, marked or not. A config key is + * deleted once no other analyzer's {@code *CanNode} claims it. + */ + static final String RELEASE_VANISHED_ARTIFACTS = + "UNWIND $pres AS pre " + + "MATCH (n:" + RowBuilder.CAN_NODE + ") WHERE n.id STARTS WITH pre AND (n:Artifact OR n:ConfigKey) " + + "AND NOT n.id IN $keep " + + "CALL { WITH n REMOVE n:" + RowBuilder.CAN_NODE + " " + + "WITH n WHERE (n:Artifact AND NOT ()-[:HAS_ARTIFACT]->(n)) " + + "OR (n:ConfigKey AND none(l IN labels(n) WHERE l ENDS WITH 'CanNode')) " + + "DETACH DELETE n } IN TRANSACTIONS OF 1000 ROWS " + + "RETURN count(*) AS released"; + + /** + * Where this app's artifacts live: {@code can:///artifact/}, and {@code + * can://artifact//} for those written before 3.1.1 moved the app outermost. Both are + * separator-terminated, so {@code app} never reaches {@code apps}. + */ + static List artifactPrefixes(String appId, String appName) { + return List.of(appId + "/artifact/", CanId.SCHEME + "artifact/" + appName + "/"); + } + /** * The prefix that matches a module's declarations but not a sibling module whose path merely * starts the same way. The separator is the whole point: {@code can://app/java/src/Foo.java} @@ -197,7 +245,10 @@ void run(GraphRows rows, boolean fullRun) { Log.info("neo4j(bolt): " + byUnit.size() + " units (" + changed.size() + " changed), " + shared.size() + " shared nodes, " + rows.edges.size() + " edges"); - // 3. shared nodes are always upserted (MERGE-only). + // 3. full run: reconcile this app's artifacts, then upsert shared nodes (MERGE-only). + if (fullRun) { + reconcileArtifacts(rows); + } upsertNodes(shared); // 4. per changed unit: purge owned edges + vanished decls, then upsert its nodes. @@ -236,11 +287,13 @@ void run(GraphRows rows, boolean fullRun) { upsertNodes(nodes); } - // 5. upsert edges owned by a changed unit (owner = source node's unit) or shared. + // 5. upsert edges owned by a changed unit (owner = source node's unit) or shared; on a + // full run also every edge into an artifact/config key, which step 3 purged. List edges = new ArrayList<>(); for (EdgeRow e : rows.edges) { String owner = unitOf.get(e.from.value); - if (owner == null || changed.contains(owner)) { + boolean intoArtifact = e.to.label.equals("Artifact") || e.to.label.equals("ConfigKey"); + if (owner == null || changed.contains(owner) || (fullRun && intoArtifact)) { edges.add(e); } } @@ -264,14 +317,7 @@ void run(GraphRows rows, boolean fullRun) { // cross-language :Artifact/:ConfigKey and no shared :JPackage/:JAnnotation can be // reached. Do not add a non-containment type to DESCENDANTS without re-checking that. // - // The snapshot path (CypherWriter.wipe) deliberately went the other way and now - // deletes this app's :Artifact/:ConfigKey via a `can:///` prefix sweep, so a - // full snapshot rebuilds them instead of letting them accumulate. That widening is - // NOT mirrored here, on purpose: this writer is incremental. Its unit of - // replacement is one module (descendantPrefix(moduleId)) or one vanished unit, and - // an app-level artifact belongs to neither -- an app-wide artifact sweep would - // delete artifacts a targeted `--target-files` run never re-pushes. The snapshot can - // afford the widening only because it always writes the full truth back. + // Artifacts belong to no unit; reconcileArtifacts (step 3) handles them. try (Session s = session()) { long pruned = s.run(PRUNE_VANISHED_UNITS_V2, Values.parameters("present", present, "appId", appId, "appName", appName)) @@ -283,6 +329,35 @@ void run(GraphRows rows, boolean fullRun) { } } + /** v2 only: v1 emits no artifacts, so it would release every one. */ + private void reconcileArtifacts(GraphRows rows) { + String appId = null; + List keep = new ArrayList<>(); + for (NodeRow n : rows.nodes) { + String label = n.labels.get(0); + if (label.equals("JApplication") && RowBuilder.isCanId(n.value)) { + appId = n.value; + } else if (label.equals("Artifact") || label.equals("ConfigKey")) { + keep.add(n.value); + } + } + if (appId == null) { + return; + } + Map args = new LinkedHashMap<>(); + args.put("appId", appId); + args.put("pres", artifactPrefixes(appId, appNameOf(rows))); + args.put("keep", keep); + // Auto-commit, batched: a --workspace-root toggle releases one node per non-source file. + // A partial failure is safe; the next full push finishes it. + try (Session s = session()) { + s.run(PURGE_ARTIFACT_EDGES, args).consume(); + s.run(PURGE_UNRESOLVED_CONFIG_READS, args).consume(); + long released = s.run(RELEASE_VANISHED_ARTIFACTS, args).single().get("released").asLong(0); + Log.info("neo4j(bolt): released " + released + " vanished artifact/config node(s)"); + } + } + private void upsertNodes(List nodes) { Map> groups = new LinkedHashMap<>(); for (NodeRow n : nodes) { diff --git a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java index 1cb55f64..274afbcc 100644 --- a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java @@ -53,7 +53,8 @@ public final class CypherWriter { * {@code DESCENDANTS} because this constant is shared with * {@link BoltWriter#PRUNE_VANISHED_UNITS_V2}, whose unit of deletion is one vanished * compilation unit, not the application: an app-level {@code :Artifact} is not a descendant of - * any one unit and must not die with one. {@code :Package} ({@code pkg:} purls) is outside both + * any one unit and must not die with one (Bolt reconciles artifacts separately, see + * {@link BoltWriter#RELEASE_VANISHED_ARTIFACTS}). {@code :Package} ({@code pkg:} purls) is outside both * — it is not under the app prefix either — as are the shared {@code :JPackage}/ * {@code :JAnnotation} merge targets. */ diff --git a/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java b/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java index c08bb580..0c487799 100644 --- a/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java +++ b/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java @@ -3,12 +3,17 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.ibm.cldk.neo4j.BoltConfig; +import com.ibm.cldk.neo4j.BoltWriter; +import com.ibm.cldk.neo4j.GraphRows; import com.ibm.cldk.schema.CanId; import java.io.IOException; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -91,12 +96,358 @@ private static int push(Path in, Path repo, String app, Path out) { "--neo4j-uri", CONTAINER.getBoltUrl(), "--neo4j-user", "neo4j", "--neo4j-password", PASSWORD); } + /** As {@link #push}, without {@code --workspace-root}: keys are relative to {@code in}. */ + private static int pushUnrooted(Path in, String app, Path out) { + return new picocli.CommandLine(new CodeAnalyzer()).execute( + "-i", in.toString(), "-o", out.toString(), "--emit", "neo4j", "--no-build", + "--app-name", app, + "--neo4j-uri", CONTAINER.getBoltUrl(), "--neo4j-user", "neo4j", "--neo4j-password", PASSWORD); + } + private static long count(String cypher, Object... params) { try (Session s = driver.session()) { return s.run(cypher, Values.parameters(params)).single().get(0).asLong(); } } + private static void run(String cypher, Object... params) { + try (Session s = driver.session()) { + s.run(cypher, Values.parameters(params)).consume(); + } + } + + private static String artifactPrefix(String app) { + return CanId.applicationId(app) + "/artifact/"; + } + + /** A project with one dependency manifest and one config file that mints a key. */ + private static void writeArtifacts(Path project, String... deps) throws IOException { + StringBuilder pom = new StringBuilder( + "4.0.0com.example" + + "demo1"); + for (String dep : deps) { + String[] ga = dep.split(":"); + pom.append("").append(ga[0]).append("") + .append(ga[1]).append("1.0"); + } + pom.append("\n"); + Files.writeString(project.resolve("pom.xml"), pom.toString(), StandardCharsets.UTF_8); + } + + private static void writeConfig(Path project, String... keys) throws IOException { + Path res = Files.createDirectories(project.resolve("src/main/resources")); + StringBuilder props = new StringBuilder(); + for (String k : keys) { + props.append(k).append("=v\n"); + } + Files.writeString(res.resolve("application.properties"), props.toString(), StandardCharsets.UTF_8); + } + + @Test + void togglingWorkspaceRootLeavesNoOldKeyedArtifacts(@TempDir Path tmp) throws IOException { + Path repo = tmp.resolve("repo"); + Path app = repo.resolve("services/app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + writeConfig(app, "server.port"); + + assertEquals(0, pushUnrooted(app, "toggle", tmp.resolve("o1"))); + String oldPom = CanId.artifactId("toggle", "pom.xml"); + assertEquals(1, count("MATCH (a:Artifact {id: $id}) RETURN count(a)", "id", oldPom), + "precondition: the unrooted push keys the artifact on its input-relative path"); + assertTrue(count("MATCH (k:ConfigKey) WHERE k.id STARTS WITH $p RETURN count(k)", + "p", oldPom.replace("pom.xml", "src/main/resources/application.properties")) > 0, + "precondition: the config file minted a key"); + + assertEquals(0, push(app, repo, "toggle", tmp.resolve("o2"))); + + String rooted = artifactPrefix("toggle") + "services/app/"; + assertEquals(0, count("MATCH (a:Artifact) WHERE a.id STARTS WITH $p AND NOT a.id STARTS WITH $r " + + "RETURN count(a)", "p", artifactPrefix("toggle"), "r", rooted), + "no artifact keyed on the old input-relative path survives"); + assertEquals(0, count("MATCH (k:ConfigKey) WHERE k.id STARTS WITH $p AND NOT k.id STARTS WITH $r " + + "RETURN count(k)", "p", artifactPrefix("toggle"), "r", rooted), + "no config key keyed on the old input-relative path survives"); + long artifacts = count("MATCH (a:Artifact) WHERE a.id STARTS WITH $r RETURN count(a)", "r", rooted); + assertTrue(artifacts > 0, "the rooted push wrote its artifacts"); + assertEquals(artifacts, count("MATCH (:JApplication {id: $app})-[r:HAS_ARTIFACT]->() RETURN count(r)", + "app", CanId.applicationId("toggle")), "one HAS_ARTIFACT per artifact the run emitted"); + assertEquals(1, count("MATCH (a:Artifact)-[r:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) " + + "WHERE a.id STARTS WITH $p RETURN count(r)", + "p", artifactPrefix("toggle"), "pkg", CanId.purlMaven("org.example", "lib")), + "the dependency is declared once, by the rooted manifest"); + } + + @Test + void aChangedManifestDropsItsRemovedDependencyAndConfigKey(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:kept", "org.example:dropped"); + writeConfig(app, "kept.key", "dropped.key"); + assertEquals(0, pushUnrooted(app, "edit", tmp.resolve("o1"))); + + String pom = CanId.artifactId("edit", "pom.xml"); + String props = CanId.artifactId("edit", "src/main/resources/application.properties"); + assertEquals(1, count("MATCH (:Artifact {id: $a})-[r:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) " + + "RETURN count(r)", "a", pom, "pkg", CanId.purlMaven("org.example", "dropped")), + "precondition: the dependency is declared"); + + writeArtifacts(app, "org.example:kept"); + writeConfig(app, "kept.key"); + assertEquals(0, pushUnrooted(app, "edit", tmp.resolve("o2"))); + + assertEquals(0, count("MATCH (:Artifact {id: $a})-[r:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) " + + "RETURN count(r)", "a", pom, "pkg", CanId.purlMaven("org.example", "dropped")), + "the removed dependency's edge is gone"); + assertEquals(1, count("MATCH (:Artifact {id: $a})-[r:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) " + + "RETURN count(r)", "a", pom, "pkg", CanId.purlMaven("org.example", "kept")), + "the kept dependency's edge stays"); + assertEquals(0, count("MATCH (k:ConfigKey {id: $id}) RETURN count(k)", + "id", CanId.configKeyId(props, "dropped.key")), "the removed config key is gone"); + assertEquals(1, count("MATCH (k:ConfigKey {id: $id}) RETURN count(k)", + "id", CanId.configKeyId(props, "kept.key")), "the kept config key stays"); + } + + @Test + void aSiblingAnalyzersArtifactsAndEdgesSurviveAJavaPush(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + assertEquals(0, pushUnrooted(app, "poly", tmp.resolve("o1"))); + + // Seed what codeanalyzer-python would have written: a claim on the shared pom.xml (with a + // pypi dependency edge), plus an artifact and config key Java never emits. + String pom = CanId.artifactId("poly", "pom.xml"); + String pyOnly = CanId.artifactId("poly", "pyproject.toml"); + String pyKey = CanId.configKeyId(pyOnly, "tool.name"); + run("MATCH (a:Artifact {id: $pom}) SET a:PyCanNode " + + "MERGE (app:PyApplication {id: 'can://poly'}) MERGE (app)-[:HAS_ARTIFACT]->(a) " + + "MERGE (p:Package {id: 'pkg:pypi/requests'}) MERGE (a)-[:DECLARES_DEPENDENCY {_k: 'main'}]->(p) " + + "MERGE (o:Artifact:PyCanNode {id: $py}) MERGE (app)-[:HAS_ARTIFACT]->(o) " + + "MERGE (k:ConfigKey:PyCanNode {id: $key}) MERGE (o)-[:DEFINES_CONFIG]->(k) " + + "MERGE (o)-[:DECLARES_DEPENDENCY {_k: 'main'}]->(p)", + "pom", pom, "py", pyOnly, "key", pyKey); + + // Java now stops emitting pom.xml: it releases its claim but must not delete a node Python holds. + Files.delete(app.resolve("pom.xml")); + assertEquals(0, pushUnrooted(app, "poly", tmp.resolve("o2"))); + + assertEquals(1, count("MATCH (a:Artifact:PyCanNode {id: $id}) WHERE NOT a:JCanNode RETURN count(a)", + "id", pom), "the shared artifact survives with Python's claim and without Java's"); + assertEquals(0, count("MATCH (:JApplication)-[r:HAS_ARTIFACT]->(:Artifact {id: $id}) RETURN count(r)", + "id", pom), "Java's HAS_ARTIFACT to the released artifact is gone"); + assertEquals(0, count("MATCH (:Artifact {id: $id})-[r:DECLARES_DEPENDENCY]->(p:Package) " + + "WHERE p.id STARTS WITH 'pkg:maven/' RETURN count(r)", "id", pom), + "Java's maven dependency edge from the released artifact is gone"); + assertEquals(1, count("MATCH (:Artifact {id: $id})-[r:DECLARES_DEPENDENCY]->(:Package {id: 'pkg:pypi/requests'}) " + + "RETURN count(r)", "id", pom), "Python's dependency edge on the shared artifact stays"); + assertEquals(1, count("MATCH (:Artifact {id: $id})-[:DEFINES_CONFIG]->(:ConfigKey {id: $key}) " + + "RETURN count(*)", "id", pyOnly, "key", pyKey), + "an artifact and config key only Python wrote are untouched"); + } + + @Test + void anAppWhoseNameExtendsAnothersIsNotReconciledByIt(@TempDir Path tmp) throws IOException { + Path a = tmp.resolve("a"); + Path b = tmp.resolve("b"); + for (Path p : new Path[] {a, b}) { + writeClass(p, "Main"); + writeArtifacts(p, "org.example:lib"); + } + assertEquals(0, pushUnrooted(b, "apps", tmp.resolve("o1"))); + long apps = count("MATCH (n:Artifact) WHERE n.id STARTS WITH $p RETURN count(n)", "p", artifactPrefix("apps")); + assertTrue(apps > 0, "precondition"); + + assertEquals(0, pushUnrooted(a, "app", tmp.resolve("o2"))); + + assertEquals(apps, count("MATCH (n:Artifact:JCanNode) WHERE n.id STARTS WITH $p RETURN count(n)", + "p", artifactPrefix("apps")), "can://app's reconciliation never reaches can://apps"); + assertEquals(1, count("MATCH (:Artifact {id: $id})-[r:DECLARES_DEPENDENCY]->() RETURN count(r)", + "id", CanId.artifactId("apps", "pom.xml")), "can://apps keeps its dependency edge"); + } + + @Test + void aConfigKeyAnotherAnalyzerClaimsSurvivesJavaDroppingIt(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeConfig(app, "shared.key", "other"); + assertEquals(0, pushUnrooted(app, "ckshare", tmp.resolve("o1"))); + + String props = CanId.artifactId("ckshare", "src/main/resources/application.properties"); + String key = CanId.configKeyId(props, "shared.key"); + run("MATCH (k:ConfigKey {id: $key}) SET k:PyCanNode " + + "MERGE (b:PyBodyNode {id: 'can://ckshare/python/x.py/f@1:1'}) MERGE (b)-[:PY_USES_CONFIG]->(k)", + "key", key); + + writeConfig(app, "other"); + assertEquals(0, pushUnrooted(app, "ckshare", tmp.resolve("o2"))); + + assertEquals(1, count("MATCH (k:ConfigKey:PyCanNode {id: $id}) WHERE NOT k:JCanNode RETURN count(k)", + "id", key), "the key survives with Python's claim and without Java's"); + assertEquals(1, count("MATCH (:Artifact {id: $a})-[:DEFINES_CONFIG]->(:ConfigKey {id: $k}) RETURN count(*)", + "a", props, "k", key), "its DEFINES_CONFIG stays"); + assertEquals(1, count("MATCH ()-[r:PY_USES_CONFIG]->(:ConfigKey {id: $k}) RETURN count(r)", "k", key), + "Python's read of it stays"); + } + + @Test + void anArtifactAnUnmarkedSiblingInventoriesSurvivesJavaDroppingIt(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + assertEquals(0, pushUnrooted(app, "oldsib", tmp.resolve("o1"))); + + // A pre-1.5.0 codeanalyzer-python: inventories the file via HAS_ARTIFACT, puts no marker on it. + String pom = CanId.artifactId("oldsib", "pom.xml"); + run("MATCH (a:Artifact {id: $pom}) MERGE (p:PyApplication {id: 'can://oldsib'}) " + + "MERGE (p)-[:HAS_ARTIFACT]->(a)", "pom", pom); + + Files.delete(app.resolve("pom.xml")); + assertEquals(0, pushUnrooted(app, "oldsib", tmp.resolve("o2"))); + + assertEquals(1, count("MATCH (:PyApplication)-[:HAS_ARTIFACT]->(a:Artifact {id: $id}) " + + "WHERE NOT a:JCanNode RETURN count(a)", "id", pom), + "the artifact and the sibling's HAS_ARTIFACT survive; only Java's claim is released"); + } + + @Test + void codeEdgesIntoConfigFollowAConfigOnlyChange(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + Path pkg = Files.createDirectories(app.resolve("src/main/java/com/example")); + Files.writeString(pkg.resolve("Main.java"), "package com.example;\npublic class Main {\n" + + " public String port() { return System.getenv(\"PORT\"); }\n}\n", StandardCharsets.UTF_8); + Files.writeString(app.resolve(".env"), "OTHER=1\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "cfguse", tmp.resolve("o1"))); + String env = CanId.artifactId("cfguse", ".env"); + assertEquals(0, count("MATCH ()-[r:J_USES_CONFIG]->(k) WHERE k.id STARTS WITH $p RETURN count(r)", + "p", env), "precondition: PORT is undeclared"); + assertEquals(1, count("MATCH (:JApplication {id: 'can://cfguse'})-[r:J_READS_CONFIG_UNRESOLVED {key: 'PORT'}]->() " + + "RETURN count(r)"), "precondition: the read is unresolved"); + + // Only the config changes; Main.java's content_hash is identical. + Files.writeString(app.resolve(".env"), "PORT=8080\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "cfguse", tmp.resolve("o2"))); + + assertEquals(1, count("MATCH ()-[r:J_USES_CONFIG]->(:ConfigKey {id: $k}) RETURN count(r)", + "k", CanId.configKeyId(env, "PORT")), "the newly declared key gains its read"); + assertEquals(0, count("MATCH (:JApplication {id: 'can://cfguse'})-[r:J_READS_CONFIG_UNRESOLVED {key: 'PORT'}]->() " + + "RETURN count(r)"), "the stale unresolved read is gone"); + + Files.writeString(app.resolve(".env"), "OTHER=1\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "cfguse", tmp.resolve("o3"))); + assertEquals(0, count("MATCH ()-[r:J_USES_CONFIG]->(k) WHERE k.id STARTS WITH $p RETURN count(r)", + "p", env), "removing the key drops its read"); + } + + @Test + void aLockEdgeFollowsAManifestChangeWithAnUnchangedLockfile(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + Files.writeString(app.resolve("build.gradle"), + "dependencies {\n implementation 'org.example:a:1.0'\n implementation 'org.example:b:1.0'\n}\n", + StandardCharsets.UTF_8); + Files.writeString(app.resolve("gradle.lockfile"), + "org.example:a:1.0=runtimeClasspath\nempty=\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "locks", tmp.resolve("o1"))); + String lock = CanId.artifactId("locks", "gradle.lockfile"); + assertEquals(1, count("MATCH (:Artifact {id: $l})-[r:LOCKS]->(:Package {id: $p}) RETURN count(r)", + "l", lock, "p", CanId.purlMaven("org.example", "a")), "precondition: a is locked"); + + // Drop `a` from the manifest only. The lockfile is byte-identical but `a` is now lock-only. + Files.writeString(app.resolve("build.gradle"), + "dependencies {\n implementation 'org.example:b:1.0'\n}\n", StandardCharsets.UTF_8); + assertEquals(0, pushUnrooted(app, "locks", tmp.resolve("o2"))); + + assertEquals(0, count("MATCH (:Artifact {id: $m})-[r:DECLARES_DEPENDENCY]->(:Package {id: $p}) RETURN count(r)", + "m", CanId.artifactId("locks", "build.gradle"), "p", CanId.purlMaven("org.example", "a")), + "the manifest no longer declares a"); + assertEquals(1, count("MATCH (:Artifact {id: $l})-[r:LOCKS]->(:Package {id: $p}) RETURN count(r)", + "l", lock, "p", CanId.purlMaven("org.example", "a")), "the lock still pins a, once"); + } + + @Test + void anUnchangedRepushIsIdempotent(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + writeConfig(app, "k"); + assertEquals(0, pushUnrooted(app, "idem", tmp.resolve("o1"))); + String snapshot = "MATCH (n) WHERE n.id STARTS WITH 'can://idem' OPTIONAL MATCH (n)-[r]-() " + + "RETURN count(DISTINCT n) * 1000000 + count(DISTINCT r)"; + long before = count(snapshot); + + assertEquals(0, pushUnrooted(app, "idem", tmp.resolve("o2"))); + + assertEquals(before, count(snapshot), "a re-push with no change leaves node and edge counts unchanged"); + assertEquals(1, count("MATCH (:Artifact {id: $a})-[r:DECLARES_DEPENDENCY]->() RETURN count(r)", + "a", CanId.artifactId("idem", "pom.xml")), "no keyed edge is doubled"); + } + + @Test + void preRekeyArtifactsAreReleasedToo(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + // What a 3.0.x-3.1.0 push left behind: the artifact keyed `can://artifact//...`. + String legacy = "can://artifact/legacy/pom.xml"; + run("MERGE (a:Artifact:JCanNode {id: $id}) MERGE (k:ConfigKey:JCanNode {id: $key}) " + + "MERGE (a)-[:DEFINES_CONFIG]->(k) " + + "MERGE (p:Package {id: 'pkg:maven/org.example/lib'}) MERGE (a)-[:DECLARES_DEPENDENCY {_k: 'main'}]->(p)", + "id", legacy, "key", legacy + "@key/x"); + + assertEquals(0, pushUnrooted(app, "legacy", tmp.resolve("o1"))); + + assertEquals(0, count("MATCH (n) WHERE n.id STARTS WITH 'can://artifact/legacy/' RETURN count(n)"), + "the pre-re-key artifact and its config key are gone"); + assertEquals(1, count("MATCH (n:Artifact {id: $id}) RETURN count(n)", + "id", CanId.artifactId("legacy", "pom.xml")), "the current artifact is written"); + } + + @Test + void aV1PushLeavesV2ArtifactsAlone(@TempDir Path tmp) throws IOException { + Path app = tmp.resolve("app"); + writeClass(app, "Main"); + writeArtifacts(app, "org.example:lib"); + assertEquals(0, pushUnrooted(app, "gen", tmp.resolve("o1"))); + long artifacts = count("MATCH (n:Artifact) WHERE n.id STARTS WITH $p RETURN count(n)", "p", artifactPrefix("gen")); + assertTrue(artifacts > 0, "precondition"); + + // A v1 full run's root is keyed on its bare name and it emits no artifacts. Driven through + // the writer directly: the v1 CLI path needs compiled classes, which --no-build cannot give. + GraphRows.NodeRow v1Root = new GraphRows.NodeRow(List.of("JApplication"), "name", "gen", + new java.util.HashMap<>(Map.of("name", "gen"))); + new BoltWriter().write(new GraphRows(List.of(v1Root), List.of()), + new BoltConfig(CONTAINER.getBoltUrl(), "neo4j", PASSWORD, null), true); + + assertEquals(artifacts, count("MATCH (n:Artifact:JCanNode) WHERE n.id STARTS WITH $p RETURN count(n)", + "p", artifactPrefix("gen")), "v1 emits no artifacts, so it must not release v2's"); + } + + @Test + void reconcilingOneAppLeavesAnotherAppsArtifactsAlone(@TempDir Path tmp) throws IOException { + Path a = tmp.resolve("a"); + Path b = tmp.resolve("b"); + for (Path p : new Path[] {a, b}) { + writeClass(p, "Main"); + writeArtifacts(p, "org.example:lib"); + writeConfig(p, "k"); + } + assertEquals(0, pushUnrooted(a, "appa", tmp.resolve("o1"))); + assertEquals(0, pushUnrooted(b, "appb", tmp.resolve("o2"))); + long bArtifacts = count("MATCH (n) WHERE (n:Artifact OR n:ConfigKey) AND n.id STARTS WITH $p " + + "RETURN count(n)", "p", artifactPrefix("appb")); + assertTrue(bArtifacts > 0, "precondition: the other app has artifacts"); + + Files.delete(a.resolve("pom.xml")); + Files.delete(a.resolve("src/main/resources/application.properties")); + assertEquals(0, pushUnrooted(a, "appa", tmp.resolve("o3"))); + + assertEquals(0, count("MATCH (n:Artifact {id: $id}) RETURN count(n)", + "id", CanId.artifactId("appa", "pom.xml")), "this app's vanished artifact is deleted"); + assertEquals(bArtifacts, count("MATCH (n) WHERE (n:Artifact OR n:ConfigKey) AND n.id STARTS WITH $p " + + "RETURN count(n)", "p", artifactPrefix("appb")), "the other app's artifacts are unchanged"); + } + @Test void aVanishedRepoRelativeModuleIsPrunedAndOtherAppsAreUntouched(@TempDir Path tmp) throws IOException { Path repo = tmp.resolve("repo"); From dcef0a6dcc32dbcb97295eb9de06e7e86b191eda Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Fri, 30 Oct 2026 13:41:08 -0400 Subject: [PATCH 07/11] test: add namespace identity contract fixtures --- src/test/resources/identity/namespaces.json | 139 ++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 src/test/resources/identity/namespaces.json diff --git a/src/test/resources/identity/namespaces.json b/src/test/resources/identity/namespaces.json new file mode 100644 index 00000000..8a892193 --- /dev/null +++ b/src/test/resources/identity/namespaces.json @@ -0,0 +1,139 @@ +{ + "identity_version": 2, + "roots": [ + { + "application_name": "inventory", + "namespace": null, + "expected_root": "can://inventory/inventory" + }, + { + "application_name": "currencyservice", + "namespace": "online-boutique", + "expected_root": "can://online-boutique/currencyservice" + }, + { + "application_name": "a%b", + "namespace": "team space", + "expected_root": "can://team%20space/a%25b" + }, + { + "application_name": "caf\u00e9", + "namespace": "\u56e2\u961f", + "expected_root": "can://%E5%9B%A2%E9%98%9F/caf%C3%A9" + }, + { + "application_name": "Inventory", + "namespace": "Team", + "expected_root": "can://Team/Inventory" + }, + { + "application_name": "inventory", + "namespace": "team", + "expected_root": "can://team/inventory" + }, + { + "application_name": " cafe\u0301 ", + "namespace": "team", + "expected_root": "can://team/%20cafe%CC%81%20" + } + ], + "invalid": [ + { + "value": "", + "reason": "empty" + }, + { + "value": " ", + "reason": "whitespace-only" + }, + { + "value": "a/b", + "reason": "slash" + }, + { + "value": "a\\b", + "reason": "backslash" + }, + { + "value": ".", + "reason": "dot" + }, + { + "value": "..", + "reason": "dot-dot" + }, + { + "value": "a\u0001b", + "reason": "control" + }, + { + "value": "\ud800", + "reason": "unpaired-high-surrogate" + }, + { + "value": "\udc00", + "reason": "unpaired-low-surrogate" + } + ], + "modules": [ + { + "application_name": "currencyservice", + "namespace": "online-boutique", + "language": "javascript", + "file_key": "server.js", + "expected_uri": "can://online-boutique/currencyservice/javascript/server.js" + }, + { + "application_name": "paymentservice", + "namespace": "online-boutique", + "language": "javascript", + "file_key": "server.js", + "expected_uri": "can://online-boutique/paymentservice/javascript/server.js" + }, + { + "application_name": "inventory", + "namespace": null, + "language": "python", + "file_key": "src/main.py", + "expected_uri": "can://inventory/inventory/python/src/main.py" + }, + { + "application_name": "inventory", + "namespace": null, + "language": "typescript", + "file_key": "src/main.ts", + "expected_uri": "can://inventory/inventory/typescript/src/main.ts" + }, + { + "application_name": "inventory", + "namespace": null, + "language": "java", + "file_key": "src/Main.java", + "expected_uri": "can://inventory/inventory/java/src/Main.java" + }, + { + "application_name": "inventory", + "namespace": null, + "language": "go", + "file_key": "main.go", + "expected_uri": "can://inventory/inventory/go/main.go" + } + ], + "duplicate_module_rows": { + "rows": [ + { + "module_id": "can://online-boutique/currencyservice/javascript/server.js", + "file_key": "server.js" + }, + { + "module_id": "can://online-boutique/paymentservice/javascript/server.js", + "file_key": "server.js" + }, + { + "module_id": "can://online-boutique/currencyservice/javascript/server.js", + "file_key": "server.js" + } + ], + "expected_unique_id_count": 2 + } +} From e35d7dcab8c62119a8bdeb4288e9d7cda523fb2e Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Fri, 30 Oct 2026 14:52:07 -0400 Subject: [PATCH 08/11] feat: qualify Java analysis identities by namespace --- README.md | 52 ++--- .../design/specs/2026-10-24-workspace-root.md | 205 +++--------------- schema.neo4j.json | 3 + src/main/java/com/ibm/cldk/CodeAnalyzer.java | 50 ++--- .../ibm/cldk/artifacts/ArtifactDiscovery.java | 15 +- .../com/ibm/cldk/artifacts/ConfigUses.java | 9 +- .../ibm/cldk/artifacts/ViewDispatches.java | 7 +- .../java/com/ibm/cldk/neo4j/BoltWriter.java | 87 ++++---- .../java/com/ibm/cldk/neo4j/CypherWriter.java | 87 ++------ .../java/com/ibm/cldk/neo4j/RowBuilder.java | 25 ++- .../com/ibm/cldk/neo4j/V2GraphProjector.java | 14 +- .../com/ibm/cldk/neo4j/V2SchemaCatalog.java | 3 +- .../java/com/ibm/cldk/schema/Analysis.java | 1 + src/main/java/com/ibm/cldk/schema/CanId.java | 63 +++++- .../com/ibm/cldk/schema/JApplication.java | 2 + .../java/com/ibm/cldk/schema/V2Emitter.java | 13 +- .../ibm/cldk/syntactic_analysis/L1Cache.java | 4 +- .../cldk/syntactic_analysis/L1Extractor.java | 15 +- .../cldk/syntactic_analysis/L2CallGraph.java | 8 +- .../com/ibm/cldk/CodeAnalyzerV2CliTest.java | 114 ++++++++-- .../java/com/ibm/cldk/L3WalaOverlaysTest.java | 2 +- .../ibm/cldk/WorkspaceRootNeo4jPushTest.java | 73 +++---- .../ibm/cldk/neo4j/BoltWriterPurgeTest.java | 46 ++-- .../cldk/neo4j/CanonicalIdentityRowsTest.java | 49 +++++ .../cldk/neo4j/CypherWriterStreamingTest.java | 12 +- .../neo4j/V2Neo4jSchemaConformanceTest.java | 8 +- .../java/com/ibm/cldk/schema/CanIdTest.java | 41 +++- .../ibm/cldk/schema/L2SchemaOracleTest.java | 2 + .../ibm/cldk/schema/L3SchemaOracleTest.java | 2 + .../ibm/cldk/schema/L4SchemaOracleTest.java | 2 + .../java/com/ibm/cldk/schema/V2JsonTest.java | 13 ++ .../cldk/syntactic_analysis/L1CacheTest.java | 10 + .../resources/schema/analysis.v2.schema.json | 15 +- 33 files changed, 556 insertions(+), 496 deletions(-) create mode 100644 src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java diff --git a/README.md b/README.md index ad332c9e..82d210af 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ Usage: codeanalyzer [-hvV] [--no-build] [--no-clean-dependencies] [-a=] [-b=] [-f=] [-i=] [-o=] [-s=] [--emit=] [--app-name=] - [--workspace-root=

    ] + [--namespace=] [--neo4j-uri=] [--neo4j-user=] [--neo4j-password=] [--neo4j-database=] [-t=]... @@ -146,9 +146,7 @@ Analyze java application. schema (the Neo4j schema.neo4j.json contract). --app-name= Logical application name for the graph :JApplication anchor (default: input dir name). - --workspace-root= Directory that file keys, module ids and artifact - ids are relative to (default: --input). Must - contain --input. Not supported with --schema v1. + --namespace= Identity namespace (defaults to application name). --neo4j-uri= Push the graph to a live Neo4j over Bolt (incremental); omit to write graph.cypher. Falls back to the NEO4J_URI environment variable. @@ -161,18 +159,23 @@ Analyze java application. -V, --version Print version information and exit. ``` -**Repository-relative paths.** By default, `symbol_table` keys and the ids built from them are relative to -`--input`. When the project sits inside a larger repository, pass `--workspace-root` to make them -repository-relative instead: +**Application identity.** `--app-name` defaults to the basename of `--input` and `--namespace` +defaults to the resolved application name. The canonical root always contains both segments: +`can:///`, including `can://inventory/inventory` for a default monolith. +Names are case-sensitive raw strings encoded as UTF-8 URI segments without trimming or Unicode +normalization. Explicit empty/whitespace-only names, controls, unpaired surrogates, slashes, +backslashes, `.` and `..` are rejected. `--namespace` is unavailable with `--schema v1`. ```bash -codeanalyzer -i monorepo/services/orders --workspace-root monorepo -# symbol_table key: services/orders/src/main/java/com/acme/Order.java +codeanalyzer -i monorepo/services/orders --app-name orders --namespace commerce +# root: can://commerce/orders +# symbol_table key: src/main/java/com/acme/Order.java ``` -Only the path part of ids changes. The application id (`can://`) stays the same. Keys -change when you toggle the flag, so a `--cache-dir` written without it is not reused by a run with -it, and the other way round. +File and artifact keys remain relative to `--input`. The draft `--workspace-root` option has +been removed. Analysis envelopes and graph application anchors carry `identity_version: 2`; +anchors also retain raw `application_name` and `identity_namespace`. Namespace changes invalidate +the L1 cache. Legacy outputs must be regenerated from source; retire legacy graph data explicitly. ## 3. Installing `codeanalyzer` as a native binary (once built, no JVM will be required for running `codeanalyzer`) @@ -258,15 +261,14 @@ build manifest or a configuration key is not a Java concept — a sibling-langua the same repository lands on the same nodes instead of a per-language duplicate. `SCHEMA_VERSION` is stamped onto the `:JApplication` node of every emitted graph. -Every node id starts with `can:///` — `can:///java/…` for code, `can:///artifact/…` -for build manifests and config keys, `can:///@external/…` for library symbols — so -`can://` is a prefix of every node the application emits, which is what the destructive +Every node id starts with `can:////` — `can:////java/…` for code, `can:////artifact/…` +for build manifests and config keys, `can:////@external/…` for library symbols — so +`can:///` is a prefix of every node the application emits, which is what the destructive statements scope on. (`:Package` is the exception: it is keyed on a `pkg:` purl, sits under no application, and no wipe reaches it.) `:JApplication` is keyed on that id, not on the free-text `--app-name`, so the root is addressable by the same id its descendants are prefixed with. The id -is derived from `--app-name` (`can://`), so it does -**not** disambiguate two services analyzed under the same name — those still merge onto one root. -Give each service its own `--app-name` if they share a database. +is derived from the pair (`--namespace`, `--app-name`). Services with the same raw application name +remain distinct in different namespaces. Services selecting the same pair merge onto the same root. **Polyglot ids.** The `/java/` segment is what makes a code node this analyzer's. Three id families deliberately omit it and are therefore **language-neutral, shared merge targets**: `:Artifact`, @@ -274,7 +276,7 @@ deliberately omit it and are therefore **language-neutral, shared merge targets* ids for them, so a `pom.xml`, a config key or `java.util.Map#get` is *one* node in a merged graph rather than a per-language duplicate. That sharing is the point, and its cost is accepted: two analyzers' notions of a library symbol are not necessarily the same thing, and merging them says -they are. The consequence for writes: because these shared nodes sit inside `can:///`, a Cypher +they are. The consequence for writes: because these shared nodes sit inside `can:////`, a Cypher snapshot's prefix wipe rebuilds them, so a cross-language edge into a shared `:Artifact` is dropped by one analyzer's snapshot and restored on the other's next push. @@ -314,9 +316,9 @@ codeanalyzer -i /path/to/project -a 2 --emit neo4j -o ./out cypher-shell -u neo4j -p < ./out/graph.cypher ``` -The snapshot is **not** incremental: it constraints, scopes-wipes this application's prior subgraph, -then `UNWIND … MERGE`-loads the full truth. The wipe is everything under `can:///`, plus a -containment traversal that also replaces a prior v1 (pre-`3.0.0`) graph of the same application. +Offline snapshots validate incoming identity metadata; they do not inspect an existing database. Execute them only as part of explicit regeneration. The snapshot is **not** incremental: it constraints, scopes-wipes this application's prior subgraph, +then `UNWIND … MERGE`-loads the full truth. The wipe is everything under `can:////`, plus a +containment traversal scoped to the selected canonical application. Legacy roots are not retired automatically. That scope **includes** this application's `:Artifact` and `:ConfigKey` nodes — the snapshot rebuilds them, rather than leaving stale ones to accumulate with nothing ever cleaning them up. The one consequence: a sibling-language analyzer's edge into a shared `:Artifact` is dropped by a Java @@ -329,13 +331,13 @@ codeanalyzer -i /path/to/project -a 2 --emit neo4j \ --neo4j-uri bolt://localhost:7687 --neo4j-user neo4j --neo4j-password ``` -The Bolt writer reads the database's current state and updates **only what changed**: it diffs each +The Bolt writer checks the selected existing anchor's identity marker and raw names before DDL or writes, then updates **only what changed**: it diffs each compilation unit's `content_hash`, replaces just the changed units' subgraphs (idempotent `MERGE` upserts), and — on a full run — prunes units whose source file vanished. Combine with `--target-files` for a targeted, partial re-push (orphan pruning is then skipped). On a full run it also reconciles the application's artifacts, so a deleted or renamed file, a -removed dependency, or re-keying under `--workspace-root` leaves nothing stale: +or removed dependency leaves nothing stale within the selected namespace/application: - Java-owned artifact edges are rebuilt from the run: `HAS_ARTIFACT`, `DECLARES_DEPENDENCY`/`LOCKS` to `pkg:maven/…`, and code reads of them (`J_USES_CONFIG`, `J_DISPATCHES_TO`, @@ -344,7 +346,7 @@ removed dependency, or re-keying under `--workspace-root` leaves nothing stale: then deleted only if no other analyzer's `HAS_ARTIFACT` still points at it; a config key only if no other analyzer's marker (`:PyCanNode`, `:TSCanNode`) still claims it. A sibling's edges are never removed. -- Artifacts written before 3.1.1 (`can://artifact//…`) are reconciled the same way. +- Legacy artifacts require explicit retirement; reconciliation never falls back to a legacy root. ### 4.3. Schema contract diff --git a/docs/design/specs/2026-10-24-workspace-root.md b/docs/design/specs/2026-10-24-workspace-root.md index 8c9f792e..6dbf1244 100644 --- a/docs/design/specs/2026-10-24-workspace-root.md +++ b/docs/design/specs/2026-10-24-workspace-root.md @@ -1,177 +1,28 @@ -# `--workspace-root`: repo-relative file keys - -**Status:** accepted · 2026-10-24 · tracks [#274](https://github.com/codellm-devkit/codeanalyzer-java/issues/274) - -## Problem - -Every v2 file key is relative to `--input`. When the analyzed project sits inside a larger -repository (`services/orders/` in a monorepo), keys read `src/main/java/.../App.java` rather than -the repository path `services/orders/src/main/java/.../App.java`. Tooling that speaks repository -paths — git, IDEs, PR diffs, scanners — cannot join on them without knowing and re-adding the -prefix. - -## Contract-impact triage - -| Question | Answer | -| --- | --- | -| Changes `analysis.json` v2 shape? | No. No node, edge or field is added, removed or renamed. | -| Changes v2 id/key meaning? | Yes, opt-in: with `--workspace-root` the `` segment is relative to the workspace root instead of `--input`. Default output is byte-identical. | -| Changes CLI? | Yes — one new optional flag. Additive → **minor, 3.4.0**. | -| Repos touched | `codeanalyzer-java` (this spec's PR). | - -## Decisions (user-approved 2026-10-24) - -1. **New optional flag `--workspace-root `.** When set, every path-derived v2 key is computed - relative to it rather than to `--input`: - - `application.symbol_table` keys (`L1Extractor.fileKey`); - - `module.id` and every durable id below it (`can:///java//…`), since they are built - from the file key; - - artifact keys and ids (`can:///artifact/`, `ArtifactDiscovery`). - - Unset, behavior is unchanged: the root is `--input`. -2. **The app segment does not move.** `can://` stays `--app-name`, else the `--input` - directory name. Only the `` segment changes, so the L1 cache key, the Neo4j `can:///` - prefix wipe/prune and `@external` ids keep their current scoping. -3. **`--input` must be inside the workspace root.** Both are resolved to absolute, normalized - paths; if `--input` is not the root or a descendant of it, exit non-zero at argument validation. - This keeps the schema invariant intact: keys are relative, never absolute, never `..`. -4. **v2 only.** v1 keys are absolute paths, so there is nothing to relativize. `--workspace-root` - with `--schema v1` is a hard argument error. -5. **No new schema field.** Consumers read source text from `module.source` and never resolve a - key against the filesystem, so the output need not record which root keys are relative to. -6. **Language-neutral semantics.** Artifact and `@external` ids are deliberately shared across - sibling analyzers over the same ``. Sibling analyzers that adopt a workspace root are - expected to use these same semantics (decisions 1–4), so a repository-relative artifact id is - the same node whichever analyzer emitted it. - -## Implementation notes - -- Thread one `keyRoot` path (workspace root if set, else `--input`) to the two relativization - sites: `L1Extractor.fileKey` (via `extractAll`) and `ArtifactDiscovery.discover`. Source - discovery, builds and dependency download still run against `--input`. -- `CanId.moduleId` strips leading `[./]+` from the key. Decision 3 makes `..` keys unreachable; - pin that with a test rather than relying on the strip. -- `L3WalaOverlays.deriveApplicationId` recovers the app id by stripping the file key off a module - id; it is key-shape agnostic, but cover it with a nested-root fixture. -- Incremental cache: keys change when the flag is toggled, so a cache written without the flag - simply misses for a run with it. No migration; document it. - -## Verification - -- Unit: nested fixture (`/repo/services/app/…`) run with `--workspace-root /repo` — - every `symbol_table` key, module id and artifact id starts with `services/app/`; no key is - absolute or contains `..`. -- Unit: without the flag, output is byte-identical to today's. -- CLI: `--input` outside the root → non-zero exit with a clear message; `--workspace-root` with - `--schema v1` → non-zero exit. -- L2/L3 on the nested fixture: call-graph `src`/`dst` and L3 overlay ids carry the same - repo-relative module ids. -- Neo4j: an incremental push with the flag wipes/prunes only that app's nodes. - -## Release plan - -Ships in **codeanalyzer-java 3.4.0** as an opt-in flag; default output is unchanged, so no -consumer needs to move in lockstep. README documents the flag, the repo-relative keys it produces, -and the cache miss on toggling it. - -## Addendum: Bolt artifact reconciliation (user-approved 2026-10-28) - -### Problem - -The Bolt writer never deletes an `:Artifact` or `:ConfigKey`. They carry no `_module`, so they -are treated as shared nodes: MERGE-upserted, never purged. Toggling `--workspace-root` on an app -that already has a graph changes every artifact id (`can:///artifact/pom.xml` becomes -`can:///artifact/services/app/pom.xml`), so the old-keyed artifacts survive alongside the -new ones. Each still hangs off `:JApplication` via `HAS_ARTIFACT` and still declares its -`:Package`s. The same leak exists without the flag, for any deleted or renamed artifact, and -for the outgoing edges of an artifact whose content changed (a dependency removed from -`pom.xml` keeps its `DECLARES_DEPENDENCY` edge; a removed yaml key keeps its `:ConfigKey`). -The Cypher snapshot does not have this problem, because its `can:///` prefix wipe -rebuilds artifacts from scratch. - -### Triage - -| Question | Answer | -| --- | --- | -| Changes `analysis.json` / `schema.neo4j.json` shape? | No. Write semantics only. | -| Changes cross-analyzer graph semantics? | Yes, narrowly: a Java Bolt full run now removes Java's own claim on artifacts and config keys it no longer emits, and deletes a node only when no other analyzer claims it. Sibling-owned nodes and edges are never touched. | -| Repos touched | `codeanalyzer-java` now. `codeanalyzer-python` (`neo4j/bolt.py`) and `codeanalyzer-typescript` (`build/neo4j/bolt.ts`) have the same gap; tracked, filed when picked up. | - -### Ownership: what is Java's - -The artifact layer's vocabulary is shared (`:Artifact`, `:ConfigKey`, `:Package`, -`HAS_ARTIFACT`, `DEFINES_CONFIG`, `DECLARES_DEPENDENCY`, `LOCKS` are unprefixed and identical in -all three analyzers), but the facts each analyzer writes under it are not: Java parses Maven and -Gradle manifests and its own set of config formats; Python writes `pkg:pypi` dependencies and -toml/ini/json config keys; TypeScript writes `pkg:npm`. Ownership is recoverable without a new -property: - -| Fact | Java's when | -| --- | --- | -| an `:Artifact` / `:ConfigKey` node | it carries `:JCanNode` (every `can://` node Java writes gets it, `RowBuilder.node`) | -| `HAS_ARTIFACT` | its source is this app's `:JApplication` | -| `DECLARES_DEPENDENCY` / `LOCKS` | its target `:Package` id starts with `pkg:maven/` | -| `DEFINES_CONFIG` | shared fact; kept while any analyzer still claims the key | - -A node claimed by another analyzer carries that analyzer's marker (`:PyCanNode`, `:TSCanNode`), -so "claimed by someone else" is `any(l IN labels(x) WHERE l ENDS WITH 'CanNode' AND -l <> 'JCanNode')`. - -### Decisions - -1. **Release, don't delete.** On a full Bolt run, Java first drops its own edges, then releases - `:JCanNode` from each `:JCanNode` artifact/config key under `can:///artifact/` (and the - pre-3.1.1 `can://artifact//`) that the run no longer emits. Deletion then depends on the - node type: - - **Artifact:** deleted when no `HAS_ARTIFACT` edge is left. Every analyzer emits `HAS_ARTIFACT` - from its own app root for every artifact it inventories, and has since it first wrote - artifacts, so a remaining edge means another analyzer still holds it. This works whether or - not that analyzer marks the node. - - **Config key:** deleted with its artifact, unless another `*CanNode` label still claims the - key. A vanished key on a surviving artifact is also deleted when no other `*CanNode` label - claims it. A sibling version that does not mark keys loses such a key until its next push. - - A node that never carried `:JCanNode` was not written by Java and is not matched. -2. **Java-owned artifact edges are rebuilt on every full run.** `sha256` is one property every - writer `SET`s, so a sibling's push can make a changed manifest look unchanged to Java; a sha - diff is therefore unsound on shared nodes. Instead, each full run deletes this app's - Java-owned artifact edges (`HAS_ARTIFACT` from its `:JApplication`; - `DECLARES_DEPENDENCY`/`LOCKS` from its artifacts to `pkg:maven/` packages) and re-upserts - them from the run. That also covers `LOCKS` going stale when a manifest changes and the lock - file does not. Artifact edges number tens to hundreds, so the cost is negligible. Java's - code-to-artifact edges (`J_USES_CONFIG`, `J_DISPATCHES_TO`, `J_READS_CONFIG_UNRESOLVED`) are - rebuilt the same way, because a config or template change alters them while every module's - `content_hash` stays the same. -3. **Batched.** The release runs as `CALL { … } IN TRANSACTIONS`: a toggle on a large repository - releases one node per non-source file, which a single transaction cannot hold. Partial failure - is safe, since the next full push finishes the reconciliation. -4. **Full runs only.** Releasing and rebuilding run only when `fullRun`. v2 is always a full run - (`--target-files` is rejected), so this covers every v2 push. -5. **Ordering.** Edge purge, then release/delete, then nodes and edges are upserted as today. - The ids to keep come from the run's rows, not the DB. -6. **Non-goals.** `:Package` nodes left with no edges are not deleted (they sit outside the app - prefix and are shared across apps). The Cypher snapshot keeps its broader wipe, which - `DETACH DELETE`s every `:JCanNode` under the app, sibling-shared artifacts included. Aligning - it with decision 1 is a recorded follow-up. - -### Verification - -- Bolt (Testcontainers), failing before the fix: push without `--workspace-root`, then push the - same project with it; assert every `:Artifact`/`:ConfigKey` under `can:///artifact/` - carries the repo-relative id, no old-keyed artifact remains, and `:JApplication` has exactly - as many `HAS_ARTIFACT` edges as the run emitted. -- Bolt: change a manifest (drop a dependency, drop a config key); re-push; assert the dropped - `DECLARES_DEPENDENCY` edge and `:ConfigKey` are gone. -- Bolt, polyglot: seed a sibling-shaped artifact (`:Artifact:PyCanNode`, a `pkg:pypi` - `DECLARES_DEPENDENCY`, a `:ConfigKey:PyCanNode`) on an id Java also emits and on one it does - not; after Java's re-push both survive with their sibling edges, and the shared one keeps - `:PyCanNode`. -- Bolt: a second application in the same database keeps all its artifacts. - -### Docs - -README §4.2: the Bolt writer reconciles this app's artifacts on a full run. It releases Java's -claim on artifacts and config keys it no longer emits, deletes them only when no other analyzer -claims them, and rebuilds Java-owned artifact edges. §4.1: state that the snapshot's wipe is -broader. Update the `BoltWriter` / `CypherWriter.DESCENDANTS` comments that say Bolt never -deletes artifacts. Releases with 3.4.0; no lockstep with siblings. +# Application identity namespaces (supersedes draft workspace-root design) + +The unmerged `--workspace-root` proposal is withdrawn. File keys, module suffixes and artifact +paths remain relative to the selected `--input` directory. No path prefix rekeying or URI +translation is performed. + +`application_name` is the supplied `--app-name`, otherwise the input basename. `identity_namespace` +is the supplied `--namespace`, otherwise the resolved application name. Both segments are always +present: `can:///`, including `can://inventory/inventory`. + +Names retain case and Unicode spelling. UTF-8 bytes outside ASCII letters/digits/`-._~` are +percent-encoded with uppercase escapes. Empty or whitespace-only names, controls, unpaired +surrogates, slashes/backslashes and whole-name `.`/`..` are invalid. Explicit invalid flags never +fall back to defaults. Legacy schema v1 rejects `--namespace`. + +The existing schema family remains `2.0.0`. The analysis envelope and graph application anchors +carry `identity_version: 2`; anchors retain raw `application_name` and `identity_namespace`. +Missing persisted metadata is unsupported, not defaulted. Regenerate from source and retire legacy +outputs explicitly. Cache identity includes the canonical namespace/application root and marker. + +Java keeps its existing `/java///` suffixes and body ordinals. External +symbols and artifacts share the canonical application root; `pkg:` identities remain unchanged. +Canonical-parent builders forward their root unchanged. No mutable global namespace is used. + +Bolt checks the selected existing application anchor before DDL or data writes. Incremental hashes, +module purges, orphan pruning and artifact reconciliation stay within that selected application. +Shared-artifact retention rules remain intact. Offline snapshots validate incoming metadata but +cannot inspect the target database; execution is an explicit regeneration operation. diff --git a/schema.neo4j.json b/schema.neo4j.json index 54a5897d..c0130713 100644 --- a/schema.neo4j.json +++ b/schema.neo4j.json @@ -13,6 +13,9 @@ "properties": { "id": "string", "name": "string", + "application_name": "string", + "identity_namespace": "string", + "identity_version": "integer", "schema_version": "string", "analyzer_name": "string", "analyzer_version": "string", diff --git a/src/main/java/com/ibm/cldk/CodeAnalyzer.java b/src/main/java/com/ibm/cldk/CodeAnalyzer.java index 412f5a90..698160fd 100644 --- a/src/main/java/com/ibm/cldk/CodeAnalyzer.java +++ b/src/main/java/com/ibm/cldk/CodeAnalyzer.java @@ -31,6 +31,7 @@ import com.ibm.cldk.neo4j.BoltConfig; import com.ibm.cldk.neo4j.Neo4jEmitter; import com.ibm.cldk.schema.Analysis; +import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JArtifact; import com.ibm.cldk.schema.JDependency; import com.ibm.cldk.schema.JEntrypointReport; @@ -146,10 +147,8 @@ public class CodeAnalyzer implements Runnable { // values between CommandLine instances in the same JVM. New flags do not add to that. private String schema = "v2"; - @Option(names = { - "--workspace-root" }, description = "Directory that file keys, module ids and artifact ids are " - + "relative to (default: --input). Must contain --input. Not supported with --schema v1.") - private String workspaceRoot; + @Option(names = "--namespace", description = "Identity namespace (defaults to application name)") + private String namespace; @Option(names = {"-c", "--cache-dir" }, description = "Directory holding the incremental analysis cache. When set, " @@ -266,10 +265,10 @@ private void analyze() throws Exception { analyzeV2(); return; } - if (workspaceRoot != null) { + if (namespace != null) { // v1 keys are absolute paths, so there is nothing for a workspace root to relativize. throw new ParameterException(spec.commandLine(), - "error: --workspace-root is not supported with --schema v1"); + "error: --namespace is not supported with --schema v1"); } JsonObject combinedJsonObject = new JsonObject(); @@ -467,20 +466,17 @@ private void analyzeV2() throws Exception { if (input == null) { throw new ParameterException(spec.commandLine(), "error: --input is required"); } - // What every emitted file key, module id and artifact id is relative to. Discovery, builds and - // dependency download still run against --input; only the key changes. Path inputRoot = Paths.get(input).toAbsolutePath().normalize(); - Path keyRoot = workspaceRoot == null ? inputRoot : Paths.get(workspaceRoot).toAbsolutePath().normalize(); - if (!inputRoot.startsWith(keyRoot)) { - // Relativizing would yield `..` keys, which the schema forbids (relative, never absolute or `..`). - throw new ParameterException(spec.commandLine(), - "error: --input " + inputRoot + " is not inside --workspace-root " + keyRoot); + Path keyRoot = inputRoot; + String application = spec.commandLine().getParseResult().hasMatchedOption("--app-name") + ? appName : inputRoot.getFileName().toString(); + String applicationId; + try { + applicationId = CanId.applicationId(application, namespace); + } catch (IllegalArgumentException e) { + throw new ParameterException(spec.commandLine(), "error: invalid --app-name or --namespace: " + e.getMessage()); } - String application = appName != null && !appName.isBlank() - ? appName - : Paths.get(input).toAbsolutePath().normalize().getFileName().toString(); - // Always attempt library type resolution: without the dependency jars on the solver's path, // third-party types degrade to bare spellings (`Model` rather than `org.springframework.ui.Model`) // and consumers lose the qualified names they join on. A failure here only thins resolution, so @@ -506,7 +502,7 @@ private void analyzeV2() throws Exception { // L1 artifact that must not carry L3 overlays. So bypass the cache entirely at level >= 3. Map cached = (eager || analysisLevel >= 3) ? new java.util.LinkedHashMap<>() - : L1Cache.load(cache, application, version); + : L1Cache.load(cache, applicationId, version); Map modules; List rtaEndpoints = null; @@ -517,8 +513,8 @@ private void analyzeV2() throws Exception { // is what turns them into a non-zero exit for a caller that cannot read stderr. List degraded = new ArrayList<>(); try { - modules = L1Extractor.extractAll( - Paths.get(input), keyRoot, application, dependencyDir, cached, + modules = L1Extractor.extractAllForApplication( + Paths.get(input), applicationId, dependencyDir, cached, analysisLevel, graphFieldDepth, l3Engine, entrypointReport); // The WALA L3 engine needs the RTA build; --no-rta suppresses it, so a level-3 wala run with // --no-rta would silently carry no overlays. Warn rather than degrade without a signal. @@ -580,7 +576,7 @@ private void analyzeV2() throws Exception { // (an L2 refinement) must not be persisted into it. At level >= 3 the tree carries L3 overlays, // which are not an L1 artifact, so the cache is not written. if (analysisLevel < 3) { - L1Cache.save(cache, application, version, modules); + L1Cache.save(cache, applicationId, version, modules); } // The repository-artifact layer (build manifests, config files, declared dependencies) sits @@ -588,7 +584,7 @@ private void analyzeV2() throws Exception { // -- but unlike them it is L1 data and runs at EVERY analysis level, so it is computed here, // ahead of the level gate, rather than inside either branch of it. Map artifacts = - ArtifactDiscovery.discover(inputRoot, keyRoot, application, artifactText, artifactTextMaxBytes); + ArtifactDiscovery.discover(inputRoot, application, artifactText, artifactTextMaxBytes, namespace); List dependencies = DependencyView.build(keyRoot, artifacts); for (JArtifact a : artifacts.values()) { if (ConfigKeys.isEligible(a)) { @@ -610,7 +606,7 @@ private void analyzeV2() throws Exception { // engine-free, so they run whenever analysisLevel >= 4 regardless of the WALA build's fate. Analysis analysis; if (analysisLevel >= 2) { - L2CallGraph.Result l2 = L2CallGraph.build(application, modules, rtaEndpoints, externalCalls); + L2CallGraph.Result l2 = L2CallGraph.buildForApplication(applicationId, modules, rtaEndpoints, externalCalls); // SdgVertices needs the callee ids L2CallGraph.build just backfilled onto call body nodes, // and no dependency jars, so it runs here rather than inside the try block above. SdgVertices.Result sdg = null; @@ -625,10 +621,10 @@ private void analyzeV2() throws Exception { analysis = V2Emitter.emit(application, analysisLevel, modules, version, l2.callGraph(), l2.externalSymbols(), sdg == null ? null : sdg.paramIn, sdg == null ? null : sdg.paramOut, - artifacts, dependencies); + artifacts, dependencies, namespace); } else { analysis = V2Emitter.emit(application, analysisLevel, modules, version, - null, null, null, null, artifacts, dependencies); + null, null, null, null, artifacts, dependencies, namespace); } // Enforced here, after every overlay pass has had its chance and before anything is written: @@ -650,7 +646,7 @@ private void analyzeV2() throws Exception { // Config reads join the L1 tree to the artifact layer's declared keys, so it runs after both // exist and at every analysis level -- the literal tier needs no call graph. Absence means // "no fact", so an empty result leaves both keys off the envelope. - ConfigUses.Result configReads = ConfigUses.detect(application, modules, artifacts, + ConfigUses.Result configReads = ConfigUses.detectForApplication(applicationId, modules, artifacts, analysisLevel, analysis.getApplication().getCallGraph()); if (!configReads.uses.isEmpty()) { analysis.getApplication().setConfigUses(configReads.uses); @@ -661,7 +657,7 @@ private void analyzeV2() throws Exception { // View dispatches (#259) join dispatch sites to the artifact layer's view templates -- the // same shape as config reads, at every level, with the dataflow tiers widening from -a 3. - ViewDispatches.Result views = ViewDispatches.detect(application, modules, artifacts, + ViewDispatches.Result views = ViewDispatches.detectForApplication(applicationId, modules, artifacts, analysisLevel, analysis.getApplication().getCallGraph()); if (!views.dispatches.isEmpty()) { analysis.getApplication().setViewDispatches(views.dispatches); diff --git a/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java b/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java index 909ae48b..565cc579 100644 --- a/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java +++ b/src/main/java/com/ibm/cldk/artifacts/ArtifactDiscovery.java @@ -136,20 +136,15 @@ private static final class Rule { */ public static Map discover( Path projectDir, String appName, boolean captureText, int textMaxBytes) throws IOException { - return discover(projectDir, projectDir, appName, captureText, textMaxBytes); + return discover(projectDir, appName, captureText, textMaxBytes, null); } - /** - * As above, with each artifact's key, {@code path} and id relative to {@code keyRoot} rather than - * {@code projectDir} — {@code --workspace-root}. Ignore rules and classification still see the - * {@code projectDir}-relative path, so a rule like {@code gradle/libs.versions.toml} matches the - * same files whatever the key root is. {@code projectDir} must be {@code keyRoot} or inside it. - */ + /** Discover application-relative artifacts in the selected identity namespace. */ public static Map discover( - Path projectDir, Path keyRoot, String appName, boolean captureText, int textMaxBytes) + Path projectDir, String appName, boolean captureText, int textMaxBytes, String namespace) throws IOException { Path root = projectDir.toAbsolutePath().normalize(); - Path keyBase = keyRoot.toAbsolutePath().normalize(); + Path keyBase = projectDir.toAbsolutePath().normalize(); List files; try (Stream walk = Files.walk(root)) { files = walk.filter(Files::isRegularFile).sorted().collect(Collectors.toList()); @@ -187,7 +182,7 @@ public static Map discover( String keyPosix = keyBase.relativize(file).toString().replace('\\', '/'); JArtifact artifact = new JArtifact(); - artifact.setId(CanId.artifactId(appName, keyPosix)); + artifact.setId(CanId.artifactId(appName, keyPosix, namespace)); artifact.setPath(keyPosix); artifact.setFormat(format); artifact.setRoles(roles); diff --git a/src/main/java/com/ibm/cldk/artifacts/ConfigUses.java b/src/main/java/com/ibm/cldk/artifacts/ConfigUses.java index cca7b1ed..0f976b2a 100644 --- a/src/main/java/com/ibm/cldk/artifacts/ConfigUses.java +++ b/src/main/java/com/ibm/cldk/artifacts/ConfigUses.java @@ -188,6 +188,11 @@ public static Result detect(String appName, Map modules, */ public static Result detect(String appName, Map modules, Map artifacts, int analysisLevel, List callGraph) { + return detectForApplication(CanId.applicationId(appName), modules, artifacts, analysisLevel, callGraph); + } + + public static Result detectForApplication(String appName, Map modules, + Map artifacts, int analysisLevel, List callGraph) { Map> keysByNamespace = keysByNamespace(artifacts); List reads = new ArrayList<>(); @@ -291,7 +296,7 @@ private static void collectCallSite(String appName, String callableId, String lo ? node.getCalleeSignature() : node.getMethodName(); reads.add(new Read(CanId.ordinalId(callableId, localId), - CanId.externalId(appName, rule.declaringType, signature), + CanId.externalIdForApplication(appName, rule.declaringType, signature), literal, rule.namespaces, false, callableId, localId, keyName)); return; } @@ -486,6 +491,6 @@ private static String annotationMember(JDecorator decorator, String... names) { } private static String ghost(String appName, String annotationType) { - return CanId.externalId(appName, annotationType, "value()"); + return CanId.externalIdForApplication(appName, annotationType, "value()"); } } diff --git a/src/main/java/com/ibm/cldk/artifacts/ViewDispatches.java b/src/main/java/com/ibm/cldk/artifacts/ViewDispatches.java index b32711e2..ddde7a96 100644 --- a/src/main/java/com/ibm/cldk/artifacts/ViewDispatches.java +++ b/src/main/java/com/ibm/cldk/artifacts/ViewDispatches.java @@ -110,6 +110,11 @@ public static Result detect(String appName, Map modules, public static Result detect(String appName, Map modules, Map artifacts, int analysisLevel, List callGraph) { + return detectForApplication(CanId.applicationId(appName), modules, artifacts, analysisLevel, callGraph); + } + + public static Result detectForApplication(String appName, Map modules, + Map artifacts, int analysisLevel, List callGraph) { List sites = new ArrayList<>(); if (modules != null) { for (JModule module : modules.values()) { @@ -195,7 +200,7 @@ private static Site siteOf(String appName, JCallable callable, String localId, J } String signature = node.getCalleeSignature() != null ? node.getCalleeSignature() : method; return new Site(CanId.ordinalId(callableId, localId), - CanId.externalId(appName, receiver, signature), via, target, callableId, localId); + CanId.externalIdForApplication(appName, receiver, signature), via, target, callableId, localId); } /** diff --git a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java index 63685f58..28961b9b 100644 --- a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java @@ -76,24 +76,13 @@ public final class BoltWriter implements BoltSink { "MATCH (x:" + RowBuilder.CAN_NODE + ") WHERE (x.id = $mid OR x.id STARTS WITH $pre) " + "AND NOT x.id IN $keys DETACH DELETE x"; - /** - * The orphan prune: matches the application root by its {@code can://} {@code id} when it is a - * v2 root (exact, unique-constrained), OR by {@code name} when it is a legacy v1 root -- - * identifiable because v1 never writes {@code id} (see {@code GraphProjector.java}). An - * id-only match would orphan a prior v1 graph instead of pruning it; an ungated - * {@code a.name =} disjunct would reach every v2 root carrying that display name -- including - * roots this analyzer never minted -- since {@code name} stopped being unique-constrained once - * the root was re-keyed. Hence the {@code a.id IS NULL} guard, which confines the name branch - * to legacy roots. - * - *

    This does not separate two distinct applications sharing an {@code --app-name}: - * {@link com.ibm.cldk.schema.CanId#applicationId} is {@code "can://" + appName}, so they share - * an id byte-for-byte and are one node. Distinct services need distinct {@code --app-name} - * values; no predicate here can recover a distinction the id never encoded. - */ + /** Hashes and orphan pruning are confined to the selected canonical application anchor. */ + static final String MODULE_HASHES_V2 = "MATCH (a:JApplication {id: $appId})-[:J_HAS_MODULE]->(c:JModule) " + + "RETURN c.file_key AS k, c.content_hash AS h"; + static final String PRUNE_VANISHED_UNITS_V2 = "MATCH (a:JApplication)-[:J_HAS_UNIT|J_HAS_MODULE]->(c) " - + "WHERE (a.id = $appId OR (a.id IS NULL AND a.name = $appName)) " + + "WHERE a.id = $appId " + "AND NOT c.file_key IN $present " + "OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x) " + "DETACH DELETE x, c RETURN count(c) AS pruned"; @@ -135,13 +124,9 @@ public final class BoltWriter implements BoltSink { + "DETACH DELETE n } IN TRANSACTIONS OF 1000 ROWS " + "RETURN count(*) AS released"; - /** - * Where this app's artifacts live: {@code can:///artifact/}, and {@code - * can://artifact//} for those written before 3.1.1 moved the app outermost. Both are - * separator-terminated, so {@code app} never reaches {@code apps}. - */ + /** Only the selected application's artifacts; legacy output requires explicit retirement. */ static List artifactPrefixes(String appId, String appName) { - return List.of(appId + "/artifact/", CanId.SCHEME + "artifact/" + appName + "/"); + return List.of(appId + "/artifact/"); } /** @@ -172,6 +157,21 @@ private static String canModuleIdOf(List nodes) { private static final int BATCH = 1000; + static void requireCompatibleApplicationIdentity(String id, Map incoming, Map stored) { + Object name = incoming.get("application_name"); + Object namespace = incoming.get("identity_namespace"); + Object version = incoming.get("identity_version"); + Object storedVersion = stored.get("identity_version"); + if (!(name instanceof String) || !(namespace instanceof String) + || (!(version instanceof Integer) && !(version instanceof Long)) || ((Number) version).longValue() != 2 + || !id.equals(CanId.applicationId((String) name, (String) namespace)) + || (!(storedVersion instanceof Integer) && !(storedVersion instanceof Long)) || ((Number) storedVersion).longValue() != 2 + || !name.equals(stored.get("application_name")) + || !namespace.equals(stored.get("identity_namespace"))) { + throw new IllegalArgumentException("Incompatible existing application identity; regenerate graph explicitly"); + } + } + /** Public no-arg constructor: {@link Neo4jEmitter} instantiates this reflectively via {@link BoltSink}. */ public BoltWriter() {} @@ -198,6 +198,19 @@ private Session session() { } void run(GraphRows rows, boolean fullRun) { + // Validate the selected canonical anchor before any schema or data mutation. + for (NodeRow node : rows.nodes) { + if (!node.labels.contains("JApplication") || !RowBuilder.isCanId(node.value)) continue; + requireCompatibleApplicationIdentity(node.value, node.props, node.props); + try (Session s = session()) { + var records = s.run("MATCH (a:JApplication {id: $id}) RETURN a.identity_version AS identity_version, " + + "a.application_name AS application_name, a.identity_namespace AS identity_namespace", + Values.parameters("id", node.value)).list(); + for (var record : records) { + requireCompatibleApplicationIdentity(node.value, node.props, record.asMap()); + } + } + } // 1. schema (DDL runs in its own autocommit transactions). Migrations run FIRST: a // constraint an older release created on a property this generation no longer keys on // is still live and enforcing, and would fail the very first push (see Schema.MIGRATIONS). @@ -230,8 +243,9 @@ void run(GraphRows rows, boolean fullRun) { // :JCompilationUnit and v2 :JModule. Map dbHash = new HashMap<>(); try (Session s = session()) { - s.run("MATCH (c) WHERE c:JCompilationUnit OR c:JModule " - + "RETURN c.file_key AS k, c.content_hash AS h").list() + boolean v2 = rows.nodes.stream().anyMatch(n -> n.labels.contains("JApplication") && RowBuilder.isCanId(n.value)); + String query = v2 ? MODULE_HASHES_V2 : "MATCH (c:JCompilationUnit) RETURN c.file_key AS k, c.content_hash AS h"; + s.run(query, Values.parameters("appId", applicationIdOf(rows))).list() .forEach(rec -> dbHash.put(rec.get("k").asString(null), rec.get("h").asString(null))); } Set changed = new HashSet<>(); @@ -305,19 +319,9 @@ void run(GraphRows rows, boolean fullRun) { if (fullRun) { List present = new ArrayList<>(byUnit.keySet()); String appName = appNameOf(rows); - String appId = CanId.applicationId(appName); - // Checked against the same hazard as the _module purges above (#213) and found - // sound, so deliberately left alone: this traversal never leaves java's own graph. - // It enters at :JApplication (java-owned, and scoped to this app by its unique - // can:// id when the root is v2, or by name when it is a legacy v1 root that never - // got an id -- see PRUNE_VANISHED_UNITS_V2's javadoc), hops a java-owned - // relationship type, and expands only through DESCENDANTS -- every member of which - // is J_-prefixed containment. `c` and `x` are unlabelled but unreachable except - // along those edges, and DESCENDANTS deliberately excludes HAS_ARTIFACT, so no - // cross-language :Artifact/:ConfigKey and no shared :JPackage/:JAnnotation can be - // reached. Do not add a non-containment type to DESCENDANTS without re-checking that. - // - // Artifacts belong to no unit; reconcileArtifacts (step 3) handles them. + String appId = applicationIdOf(rows); + // Only Java containment under this exact anchor is traversed. Shared artifacts + // are reconciled separately and sibling namespaces are never fallback matches. try (Session s = session()) { long pruned = s.run(PRUNE_VANISHED_UNITS_V2, Values.parameters("present", present, "appId", appId, "appName", appName)) @@ -348,7 +352,7 @@ private void reconcileArtifacts(GraphRows rows) { args.put("appId", appId); args.put("pres", artifactPrefixes(appId, appNameOf(rows))); args.put("keep", keep); - // Auto-commit, batched: a --workspace-root toggle releases one node per non-source file. + // Auto-commit and batched to bound full-run artifact reconciliation. // A partial failure is safe; the next full push finishes it. try (Session s = session()) { s.run(PURGE_ARTIFACT_EDGES, args).consume(); @@ -437,6 +441,13 @@ private static String hashOf(List nodes, String fileKey) { * identically by both generations, so this always yields the true app name to re-derive * the id from. */ + private static String applicationIdOf(GraphRows rows) { + for (NodeRow n : rows.nodes) { + if (n.labels.contains("JApplication") && RowBuilder.isCanId(n.value)) return n.value; + } + return CanId.applicationId(appNameOf(rows)); + } + private static String appNameOf(GraphRows rows) { for (NodeRow n : rows.nodes) { if (n.labels.get(0).equals("JApplication")) { diff --git a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java index 274afbcc..88cb359f 100644 --- a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java @@ -34,30 +34,7 @@ public final class CypherWriter { private static final int BATCH = 500; - /** - * Every containment relationship either graph generation emits — v1 (unit-rooted) and v2 - * (module-rooted) together, so a v2 push wipes/prunes a prior v1 graph of the same app and vice - * versa (spec: one app name = one graph, latest push wins). - * - *

    This traversal is the only thing that reaches a v1 graph, and that is why - * {@link #wipe}'s {@code can://} prefix sweep cannot replace it: v1 ids are fully-qualified - * names carrying no application segment, and {@code GraphProjector.java} keys - * {@code :JApplication} on {@code name}, so no prefix predicate sees a single v1 node. Folding - * the two into one prefix sweep was tried on this branch and reverted; it orphans every v1 - * graph. - * - *

    {@code HAS_ARTIFACT} (and the {@code :Package} edges {@code DECLARES_DEPENDENCY} / - * {@code LOCKS}) stay out of this list — but no longer because the artifact subtree is - * untouchable. {@link #wipe}'s prefix sweep now does reach this application's - * {@code :Artifact}/{@code :ConfigKey} nodes, deliberately; see its javadoc. They stay out of - * {@code DESCENDANTS} because this constant is shared with - * {@link BoltWriter#PRUNE_VANISHED_UNITS_V2}, whose unit of deletion is one vanished - * compilation unit, not the application: an app-level {@code :Artifact} is not a descendant of - * any one unit and must not die with one (Bolt reconciles artifacts separately, see - * {@link BoltWriter#RELEASE_VANISHED_ARTIFACTS}). {@code :Package} ({@code pkg:} purls) is outside both - * — it is not under the app prefix either — as are the shared {@code :JPackage}/ - * {@code :JAnnotation} merge targets. - */ + /** Java containment relationships; traversed only below the selected application anchor. */ static final String DESCENDANTS = "[:J_DECLARES_TYPE|J_HAS_NESTED_TYPE|J_HAS_CALLABLE|J_HAS_FIELD|J_HAS_PARAMETER" + "|J_HAS_CALLSITE|J_DECLARES_VAR|J_HAS_ENUM_CONSTANT|J_HAS_RECORD_COMPONENT|J_HAS_INIT_BLOCK" + "|J_HAS_CRUD_OPERATION|J_HAS_CRUD_QUERY|J_HAS_COMMENT" @@ -115,7 +92,7 @@ public static void writeCypher(Appendable out, GraphRows rows, String appName) t s.add(""); s.add("// ── wipe this project's prior subgraph (packages/annotations are shared and survive; " + "artifacts/config keys are inside the app prefix and get rebuilt) ──"); - s.add(wipe(appName)); + s.add(wipe(applicationIdOf(rows, appName))); s.add(""); s.add("// ── nodes ──"); @@ -166,58 +143,24 @@ static String applicationPrefix(String appName) { return CanId.applicationId(appName) + "/"; } - /** - * The application root by equality plus everything under {@code can:///}, then the same - * application's prior v1 graph by containment, then the orphan sweep. - * - *

    Statement 1, the prefix sweep. Scoped by id, so it is one application by - * construction: a second java app sharing a source path, and a sibling analyzer's own - * {@code can://} graph under a different app, are outside it. {@code :Package} nodes - * ({@code pkg:} purls) stay outside too — they are not under the app prefix. Two things moved - * with the app-outermost grammar: the root is matched by its {@code can://} id rather than - * by the free-text {@code --app-name}, so the id the graph is actually keyed on is the id the - * wipe uses; and this application's {@code :Artifact}/{@code :ConfigKey} nodes are now - * inside the prefix, so the snapshot rebuilds them instead of leaving them to accumulate - * forever with nothing ever cleaning them up. That is deliberate, and it is the one behavioural - * widening here: a cross-language edge into a shared {@code :Artifact} is dropped by a java - * snapshot and restored on that analyzer's next push. - * - *

    Statement 2, the containment traversal. Not redundant with the sweep above and not - * foldable into it — see {@link #DESCENDANTS}. v1 ids are fully-qualified names, so the prefix - * predicate cannot reach a v1 node; this hop is what makes "one app name = one graph, latest - * push wins" hold across generations. The unit hop is unlabeled and lists both generations' rel - * types (v1 {@code J_HAS_UNIT} → {@code :JCompilationUnit}, v2 {@code J_HAS_MODULE} → - * {@code :JModule}) so either generation's push replaces whichever the DB currently holds. The - * root is matched by id when it is v2 (exact, unique-constrained) OR by name when it is a legacy - * v1 root — identifiable because v1 never writes {@code id}. An id-only match would orphan a - * prior v1 graph instead of replacing it, breaking the very cross-generation guarantee this - * exists for. The {@code a.id IS NULL} guard confines the name branch to those legacy roots; an - * ungated {@code a.name =} disjunct would additionally reach every v2 root carrying this display - * name, including roots this analyzer never minted, so do not widen it. - * - *

    Statement 3, the orphan sweep. Fully-isolated {@code :JSymbol} nodes the containment - * traversal cannot reach — v1's import-materialized bodyless {@code :JType} stubs hang off units - * via {@code J_IMPORTS} only, so the deletes above orphan them. Degree-0 symbols are - * unreferencable junk in any generation, and a symbol another application still uses keeps its - * edges and survives. - * - *

    What none of this does — and must not be documented as doing — is separate two distinct - * applications sharing an {@code --app-name}. {@code CanId.applicationId} is - * {@code "can://" + appName}, so they share an id byte-for-byte and are ONE node; there is - * nothing here to tell apart. Distinct services need distinct {@code --app-name} values, not a - * wider predicate. - */ - private static String wipe(String appName) { - String appId = cypherValue(CanId.applicationId(appName)); + /** Canonical roots come from projected rows, preserving the selected namespace. */ + private static String applicationIdOf(GraphRows rows, String appName) { + for (NodeRow n : rows.nodes) { + if (n.labels.contains("JApplication") && RowBuilder.isCanId(n.value)) return n.value; + } + return CanId.applicationId(appName); + } + + private static String wipe(String applicationId) { + String appId = cypherValue(applicationId); return "MATCH (x:" + RowBuilder.CAN_NODE + ") WHERE x.id = " + appId - + " OR x.id STARTS WITH " + cypherValue(applicationPrefix(appName)) + "\n" + + " OR x.id STARTS WITH " + cypherValue(applicationId + "/") + "\n" + "CALL { WITH x DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS;\n" + "MATCH (a:JApplication) WHERE a.id = " + appId - + " OR (a.id IS NULL AND a.name = " + cypherValue(appName) + ")\n" + + "\n" + "OPTIONAL MATCH (a)-[:J_HAS_UNIT|J_HAS_MODULE]->(c)\n" + "OPTIONAL MATCH (c)-" + DESCENDANTS + "->(x)\n" - + "DETACH DELETE x, c, a;\n" - + "MATCH (s:JSymbol) WHERE NOT (s)--() DELETE s;"; + + "DETACH DELETE x, c, a;\n"; } // ---------------------------------------------------------------------------------------------- diff --git a/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java b/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java index 9f39bfb9..e11e9242 100644 --- a/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java +++ b/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java @@ -67,13 +67,24 @@ public static Map prune(Map in) { } /** - * Upsert a node. Re-seeing the same {@code (labels[0], value)} merges props (last write wins) - * and unions labels. + * Upsert a node. Canonical ids deduplicate across labels, unioning labels and compatible + * properties. Conflicting canonical properties are rejected. Legacy keys retain label scope. */ public NodeRef node(List labels, String keyProp, String value, Map props) { - String id = labels.get(0) + " " + value; + String id = isCanId(value) ? value : labels.get(0) + " " + value; NodeRow existing = nodes.get(id); if (existing != null) { + if (isCanId(value)) { + if (!existing.keyProp.equals(keyProp)) { + throw new IllegalArgumentException("Conflicting canonical entity key: " + value); + } + for (var entry : props.entrySet()) { + if (existing.props.containsKey(entry.getKey()) + && !java.util.Objects.equals(existing.props.get(entry.getKey()), entry.getValue())) { + throw new IllegalArgumentException("Conflicting canonical entity property " + entry.getKey() + ": " + value); + } + } + } existing.props.putAll(props); for (String l : labels) { if (!existing.labels.contains(l)) { @@ -165,6 +176,10 @@ public void edgeIfBothResolved(String type, NodeRef from, NodeRef to, Map(b) SET r += p`). Map uniqueEdges = new LinkedHashMap<>(); for (EdgeRow e : edges) { - uniqueEdges.put(e.type + "|" + e.from.label + ":" + e.from.value - + "|" + e.to.label + ":" + e.to.value + uniqueEdges.put(e.type + "|" + endpointKey(e.from) + + "|" + endpointKey(e.to) + (e.key == null ? "" : "|" + e.key), e); } diff --git a/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java b/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java index ecfa9bfd..22094a72 100644 --- a/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java +++ b/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java @@ -73,6 +73,15 @@ private V2GraphProjector() {} private static final List SYMBOL_EXTERNAL = Arrays.asList("JSymbol", "JExternal"); public static GraphRows project(Analysis analysis, String appName) { + if (!Integer.valueOf(2).equals(analysis.getIdentityVersion())) { + throw new IllegalArgumentException("Unsupported identity_version; regenerate analysis from source"); + } + String rawName = analysis.getApplication().getApplicationName(); + String namespace = analysis.getApplication().getIdentityNamespace(); + if (rawName == null || namespace == null + || !CanId.applicationId(rawName, namespace).equals(analysis.getApplication().getId())) { + throw new IllegalArgumentException("Missing or incompatible application identity; regenerate from source"); + } RowBuilder b = new RowBuilder(); Map symbolTable = analysis.getApplication().getSymbolTable(); @@ -83,7 +92,10 @@ public static GraphRows project(Analysis analysis, String appName) { "application.id is null -- the can:// root id must be stamped by V2Emitter before projection"); Map appProps = RowBuilder.props(); appProps.put("id", appId); - appProps.put("name", appName); + appProps.put("name", rawName); + appProps.put("application_name", rawName); + appProps.put("identity_namespace", namespace); + appProps.put("identity_version", 2); appProps.put("schema_version", V2SchemaCatalog.SCHEMA_VERSION); if (analysis.getAnalyzer() != null) { appProps.put("analyzer_name", analysis.getAnalyzer().getName()); diff --git a/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java b/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java index f863d83a..75f59065 100644 --- a/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java +++ b/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java @@ -129,7 +129,8 @@ private static List buildNodeLabels() { // one reading the same is the failure this record exists to prevent. Mirrors what // codeanalyzer-python projects onto :PyApplication. n.add(node("JApplication", "JApplication", "id", - new P().put("id", "string").put("name", "string").put("schema_version", "string") + new P().put("id", "string").put("name", "string").put("application_name", "string").put("identity_namespace", "string") + .put("identity_version", "integer").put("schema_version", "string") .put("analyzer_name", "string").put("analyzer_version", "string") .put("entrypoint_frameworks", "string[]") .put("entrypoint_report_json", "string").done())); diff --git a/src/main/java/com/ibm/cldk/schema/Analysis.java b/src/main/java/com/ibm/cldk/schema/Analysis.java index dd3aaa40..d7716d5f 100644 --- a/src/main/java/com/ibm/cldk/schema/Analysis.java +++ b/src/main/java/com/ibm/cldk/schema/Analysis.java @@ -9,6 +9,7 @@ */ @Data public class Analysis { + private Integer identityVersion; private String schemaVersion; private String language; private int maxLevel; diff --git a/src/main/java/com/ibm/cldk/schema/CanId.java b/src/main/java/com/ibm/cldk/schema/CanId.java index e29acfdd..33f570ab 100644 --- a/src/main/java/com/ibm/cldk/schema/CanId.java +++ b/src/main/java/com/ibm/cldk/schema/CanId.java @@ -4,7 +4,7 @@ * Canonical {@code can://} id construction for schema v2. * *

    Durable ids (≥ callable) are containment paths - * {@code can:///java///}; ordinal ids (< callable) are + * {@code can:////java///}; ordinal ids (< callable) are * {@code @} where {@code } is a source position {@code line:col} (real * nodes) or a synthetic tag (e.g. {@code entry}). Pure functions; ids are opaque handles (the * {@code } segment itself may contain {@code /}). @@ -19,9 +19,44 @@ private CanId() {} /** This analyzer's language segment, which now sits INSIDE the app rather than above it. */ public static final String LANG = "java"; - /** {@code can://} — the application root, and the prefix every id below it shares. */ + /** {@code can:///} — the application root, and the prefix every id below it shares. */ public static String applicationId(String appName) { - return SCHEME + appName; + return applicationId(appName, null); + } + + public static String applicationId(String appName, String namespace) { + return SCHEME + encodeName(namespace == null ? appName : namespace) + "/" + encodeName(appName); + } + + private static String encodeName(String value) { + if (value == null || value.isEmpty() || value.equals(".") || value.equals("..")) { + throw new IllegalArgumentException("Invalid identity name"); + } + boolean whitespace = true; + for (int i = 0; i < value.length();) { + int cp = value.codePointAt(i); + if (Character.isISOControl(cp) || cp == '/' || cp == '\\' + || cp >= 0xD800 && cp <= 0xDFFF) { + throw new IllegalArgumentException("Invalid identity name"); + } + whitespace &= Character.isWhitespace(cp) || Character.isSpaceChar(cp); + i += Character.charCount(cp); + } + if (whitespace) { + throw new IllegalArgumentException("Identity name must not be whitespace-only"); + } + StringBuilder out = new StringBuilder(); + for (byte b : value.getBytes(java.nio.charset.StandardCharsets.UTF_8)) { + int c = b & 255; + if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' + || c == '-' || c == '.' || c == '_' || c == '~') { + out.append((char) c); + } else { + out.append('%').append("0123456789ABCDEF".charAt(c >>> 4)) + .append("0123456789ABCDEF".charAt(c & 15)); + } + } + return out.toString(); } /** {@code /java/} (separators normalized to {@code /}). */ @@ -41,7 +76,7 @@ public static String ordinalId(String callableId, String tag) { } /** - * {@code can:///@external//} — a callable outside the project. The + * {@code can:////@external//} — a callable outside the project. The * type is a binary name ({@code java.util.Map$Entry}) so the id is unambiguous and joins * WALA natively. * @@ -53,23 +88,35 @@ public static String ordinalId(String callableId, String tag) { * codeanalyzer-python follows it, and so does this. * *

    Consequently an external id does not match the {@code canId} shape - * ({@code ^can://[^/]+/java/}): the analysis schema's {@code externalCanId} is a separate + * ({@code ^can://[^/]+/[^/]+/java/}): the analysis schema's {@code externalCanId} is a separate * definition, and the two sites that hold either — a call edge's {@code dst} and a body node's * {@code callee} — {@code $ref} the union, not a widened {@code canId}. */ public static String externalId(String appName, String binaryType, String signature) { - return applicationId(appName) + "/@external/" + binaryType + "/" + signature; + return externalId(appName, binaryType, signature, null); + } + + public static String externalId(String appName, String binaryType, String signature, String namespace) { + return externalIdForApplication(applicationId(appName, namespace), binaryType, signature); + } + + public static String externalIdForApplication(String applicationId, String binaryType, String signature) { + return applicationId + "/@external/" + binaryType + "/" + signature; } /** - * {@code can:///artifact/} — a language-neutral artifact id, nested under the app + * {@code can:////artifact/} — a language-neutral artifact id, nested under the app * rather than under a language. The {@code artifact} segment is deliberately chosen over * {@code java} so a sibling-language analyzer scanning the same repository lands on the same * node rather than a duplicate. {@code CypherWriter.DESCENDANTS} cites that sharing by name as * the reason the artifact/config-key subtree stays outside every wipe. */ public static String artifactId(String appName, String relPath) { - return applicationId(appName) + "/artifact/" + relPath; + return artifactId(appName, relPath, null); + } + + public static String artifactId(String appName, String relPath, String namespace) { + return applicationId(appName, namespace) + "/artifact/" + relPath; } /** diff --git a/src/main/java/com/ibm/cldk/schema/JApplication.java b/src/main/java/com/ibm/cldk/schema/JApplication.java index ceb1d491..85b61aa6 100644 --- a/src/main/java/com/ibm/cldk/schema/JApplication.java +++ b/src/main/java/com/ibm/cldk/schema/JApplication.java @@ -9,6 +9,8 @@ @Data public class JApplication { private String id; + private String applicationName; + private String identityNamespace; private String kind = "application"; private Map symbolTable = new LinkedHashMap<>(); diff --git a/src/main/java/com/ibm/cldk/schema/V2Emitter.java b/src/main/java/com/ibm/cldk/schema/V2Emitter.java index 34401be1..df6af7ae 100644 --- a/src/main/java/com/ibm/cldk/schema/V2Emitter.java +++ b/src/main/java/com/ibm/cldk/schema/V2Emitter.java @@ -72,8 +72,18 @@ public static Analysis emit( List paramOut, Map artifacts, List dependencies) { + return emit(appName, maxLevel, modules, analyzerVersion, callGraph, externalSymbols, + paramIn, paramOut, artifacts, dependencies, null); + } + + public static Analysis emit(String appName, int maxLevel, Map modules, + String analyzerVersion, List callGraph, Map externalSymbols, + List paramIn, List paramOut, Map artifacts, + List dependencies, String namespace) { JApplication application = new JApplication(); - application.setId(CanId.applicationId(appName)); + application.setId(CanId.applicationId(appName, namespace)); + application.setApplicationName(appName); + application.setIdentityNamespace(namespace == null ? appName : namespace); // Sort file keys so output is deterministic (the -j gate). Map sorted = new LinkedHashMap<>(); @@ -103,6 +113,7 @@ public static Analysis emit( } Analysis analysis = new Analysis(); + analysis.setIdentityVersion(2); analysis.setSchemaVersion("2.0.0"); analysis.setLanguage("java"); analysis.setMaxLevel(maxLevel); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java index 8e7eb498..68c51ace 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java @@ -34,6 +34,7 @@ private L1Cache() {} /** What is persisted: the modules plus the settings they were built under. */ @Data static class Envelope { + private Integer identityVersion; private String schemaVersion; private String analyzerVersion; private String appName; @@ -62,7 +63,7 @@ public static Map load(Path cacheDir, String appName, String an if (envelope == null || envelope.getModules() == null) { return new LinkedHashMap<>(); } - boolean sameSettings = "2.0.0".equals(envelope.getSchemaVersion()) + boolean sameSettings = Integer.valueOf(2).equals(envelope.getIdentityVersion()) && "2.0.0".equals(envelope.getSchemaVersion()) && java.util.Objects.equals(appName, envelope.getAppName()) && java.util.Objects.equals(analyzerVersion, envelope.getAnalyzerVersion()); if (!sameSettings) { @@ -83,6 +84,7 @@ public static void save( return; } Envelope envelope = new Envelope(); + envelope.setIdentityVersion(2); envelope.setSchemaVersion("2.0.0"); envelope.setAnalyzerVersion(analyzerVersion); envelope.setAppName(appName); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java index cb431c9e..975d901c 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java @@ -121,17 +121,13 @@ public static Map extractAll( Path projectRoot, String appName, Path dependencyDir, Map cached, int analysisLevel, int graphFieldDepth, String l3Engine, JEntrypointReport entrypointReport) throws IOException { - return extractAll(projectRoot, projectRoot, appName, dependencyDir, cached, analysisLevel, + return extractAllForApplication(projectRoot, CanId.applicationId(appName), dependencyDir, cached, analysisLevel, graphFieldDepth, l3Engine, entrypointReport); } - /** - * As above, with file keys (and so every module id below them) relative to {@code keyRoot} rather - * than {@code projectRoot} — {@code --workspace-root}. Source discovery still runs over - * {@code projectRoot}, which the caller guarantees is {@code keyRoot} or inside it. - */ - public static Map extractAll( - Path projectRoot, Path keyRoot, String appName, Path dependencyDir, Map cached, + /** Extract application-relative modules under an already validated canonical application root. */ + public static Map extractAllForApplication( + Path projectRoot, String applicationId, Path dependencyDir, Map cached, int analysisLevel, int graphFieldDepth, String l3Engine, JEntrypointReport entrypointReport) throws IOException { // JavaParser's collection strategy silently finds zero source roots for a path with a `.` @@ -153,12 +149,11 @@ public static Map extractAll( // Collect into a sorted map first: directory listings are not ordered, and output must not // depend on traversal order. Map modules = new TreeMap<>(); - String applicationId = CanId.applicationId(appName); JavaParser parser = new JavaParser(config); int reused = 0; for (SourceRoot sourceRoot : sourceRoots) { for (Path path : javaFilesUnder(sourceRoot.getRoot())) { - String fileKey = fileKey(keyRoot, path); + String fileKey = fileKey(projectRoot, path); // Read the file's own text rather than printing the AST: `span.bytes` must index the // real file, byte for byte. String source = Files.readString(path, StandardCharsets.UTF_8); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L2CallGraph.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L2CallGraph.java index c1f2644d..135535c0 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L2CallGraph.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L2CallGraph.java @@ -133,6 +133,10 @@ public static Result build(String appName, Map modules, List modules, List rtaEdges, boolean includeExternal) { + return buildForApplication(CanId.applicationId(appName), modules, rtaEdges, includeExternal); + } + + public static Result buildForApplication(String appName, Map modules, List rtaEdges, boolean includeExternal) { // The type index (named types only) and the set of every callable id in the tree. The latter // turns no-dangling, the signature-miss case, and the RTA drop rules into O(1) membership checks. Map typeIdByBinaryName = new HashMap<>(); @@ -279,7 +283,7 @@ private static String resolveCallee( if (!includeExternal) { return null; // case 4 gated off: match v1's application-only graph } - String externalId = CanId.externalId(appName, hint, signature); // case 4 + String externalId = CanId.externalIdForApplication(appName, hint, signature); // case 4 externalSymbols.computeIfAbsent(externalId, k -> externalSymbol(signature, hint, node.isConstructorCall())); return externalId; } @@ -314,7 +318,7 @@ private static void joinRtaEdge( if (!includeExternal) { return; // external target gated off: match v1's application-only graph } - dst = CanId.externalId(appName, edge.dstType, edge.dstSignature); + dst = CanId.externalIdForApplication(appName, edge.dstType, edge.dstSignature); externalSymbols.computeIfAbsent( dst, k -> externalSymbol(edge.dstSignature, edge.dstType, edge.dstSignature.startsWith(""))); } diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java index 361b866c..eddbe4dc 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java @@ -804,24 +804,24 @@ private static void assertRepoRelative(Set keys, String prefix) { } @Test - void workspaceRootMakesSymbolTableKeysAndIdsRepoRelative(@TempDir Path tmp) throws IOException { + void namespaceKeepsSymbolTableKeysApplicationRelative(@TempDir Path tmp) throws IOException { Path repo = tmp.resolve("repo"); Path in = project(repo.resolve("services/app")); Path out = tmp.resolve("out"); assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "-a", "2", - "--workspace-root", repo.toString(), "--app-name", "widgets")); + "--namespace", "team", "--app-name", "widgets")); JsonObject app = application(out); - assertEquals(CanId.applicationId("widgets"), app.get("id").getAsString(), + assertEquals(CanId.applicationId("widgets", "team"), app.get("id").getAsString(), "the app segment does not move with the workspace root"); JsonObject symbolTable = app.getAsJsonObject("symbol_table"); - assertRepoRelative(symbolTable.keySet(), "services/app/"); - String key = "services/app/src/main/java/com/example/Widget.java"; + assertRepoRelative(symbolTable.keySet(), "src/"); + String key = "src/main/java/com/example/Widget.java"; assertTrue(symbolTable.has(key), "got: " + symbolTable.keySet()); - assertEquals(CanId.moduleId(CanId.applicationId("widgets"), key), + assertEquals(CanId.moduleId(CanId.applicationId("widgets", "team"), key), symbolTable.getAsJsonObject(key).get("id").getAsString()); - String moduleId = CanId.moduleId(CanId.applicationId("widgets"), key); + String moduleId = CanId.moduleId(CanId.applicationId("widgets", "team"), key); JsonArray callGraph = app.getAsJsonArray("call_graph"); assertFalse(callGraph.isEmpty(), "precondition: size() -> count() is an in-project edge"); for (JsonElement e : callGraph) { @@ -832,18 +832,18 @@ void workspaceRootMakesSymbolTableKeysAndIdsRepoRelative(@TempDir Path tmp) thro } @Test - void workspaceRootCarriesIntoLevelThreeOverlayIds(@TempDir Path tmp) throws IOException { + void namespaceCarriesIntoLevelThreeOverlayIds(@TempDir Path tmp) throws IOException { Path repo = tmp.resolve("repo"); Path in = project(repo.resolve("services/app")); Path out = tmp.resolve("out"); assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "-a", "3", - "--workspace-root", repo.toString(), "--app-name", "widgets")); + "--namespace", "team", "--app-name", "widgets")); - String moduleId = CanId.moduleId(CanId.applicationId("widgets"), - "services/app/src/main/java/com/example/Widget.java"); + String moduleId = CanId.moduleId(CanId.applicationId("widgets", "team"), + "src/main/java/com/example/Widget.java"); String json = Files.readString(out.resolve("analysis.json")); assertTrue(json.contains("\"cfg\""), "precondition: level 3 lays the cfg overlay"); - java.util.regex.Matcher ids = java.util.regex.Pattern.compile("\"(can://widgets/java/[^\"]*)\"").matcher(json); + java.util.regex.Matcher ids = java.util.regex.Pattern.compile("\"(can://team/widgets/java/[^\"]*)\"").matcher(json); int seen = 0; while (ids.find()) { assertTrue(ids.group(1).startsWith(moduleId), @@ -855,39 +855,39 @@ void workspaceRootCarriesIntoLevelThreeOverlayIds(@TempDir Path tmp) throws IOEx } @Test - void workspaceRootPrefixesArtifactKeysWithoutChangingClassification(@TempDir Path tmp) throws IOException { + void namespaceKeepsArtifactKeysAndClassification(@TempDir Path tmp) throws IOException { Path repo = tmp.resolve("repo"); Path in = artifactProject(repo.resolve("services/app")); Path out = tmp.resolve("out"); assertEquals(0, run("-i", in.toString(), "-o", out.toString(), - "--workspace-root", repo.toString(), "--app-name", "widgets")); + "--namespace", "team", "--app-name", "widgets")); JsonObject app = application(out); JsonObject artifacts = app.getAsJsonObject("artifacts"); - assertRepoRelative(artifacts.keySet(), "services/app/"); + assertRepoRelative(artifacts.keySet(), ""); - JsonObject pom = artifacts.getAsJsonObject("services/app/pom.xml"); + JsonObject pom = artifacts.getAsJsonObject("pom.xml"); assertNotNull(pom, "got: " + artifacts.keySet()); - assertEquals("services/app/pom.xml", pom.get("path").getAsString()); - assertEquals(CanId.artifactId("widgets", "services/app/pom.xml"), pom.get("id").getAsString()); + assertEquals("pom.xml", pom.get("path").getAsString()); + assertEquals(CanId.artifactId("widgets", "pom.xml", "team"), pom.get("id").getAsString()); assertEquals("full", pom.get("extraction").getAsString(), "the manifest is still read from disk under the repo-relative key"); assertFalse(app.getAsJsonArray("dependencies").isEmpty(), "and its dependency still extracted"); - JsonArray propsKeys = artifacts.getAsJsonObject("services/app/application.properties") + JsonArray propsKeys = artifacts.getAsJsonObject("application.properties") .getAsJsonArray("config_keys"); assertEquals(1, propsKeys.size(), "config keys are still extracted"); assertConformsToV2Schema(out); } @Test - void workspaceRootEqualToInputChangesNothing(@TempDir Path tmp) throws IOException { + void explicitDefaultNamespaceChangesNothing(@TempDir Path tmp) throws IOException { Path in = artifactProject(tmp.resolve("app")); Path without = tmp.resolve("without"); Path with = tmp.resolve("with"); assertEquals(0, run("-i", in.toString(), "-o", without.toString(), "--app-name", "widgets")); assertEquals(0, run("-i", in.toString(), "-o", with.toString(), "--app-name", "widgets", - "--workspace-root", in.toString())); + "--namespace", "widgets")); assertEquals(application(without).get("symbol_table").toString(), application(with).get("symbol_table").toString()); assertEquals(application(without).get("artifacts").toString(), @@ -895,7 +895,7 @@ void workspaceRootEqualToInputChangesNothing(@TempDir Path tmp) throws IOExcepti } @Test - void workspaceRootMustContainTheInput(@TempDir Path tmp) throws IOException { + void removedWorkspaceRootIsRejected(@TempDir Path tmp) throws IOException { Path in = project(tmp.resolve("app")); Path elsewhere = Files.createDirectories(tmp.resolve("elsewhere")); assertNotEquals(0, run("-i", in.toString(), "-o", tmp.resolve("out").toString(), @@ -905,10 +905,76 @@ void workspaceRootMustContainTheInput(@TempDir Path tmp) throws IOException { } @Test - void workspaceRootIsRejectedWithSchemaV1(@TempDir Path tmp) throws IOException { + void namespaceIsRejectedWithSchemaV1(@TempDir Path tmp) throws IOException { Path in = project(tmp.resolve("app")); assertNotEquals(0, run("-i", in.toString(), "-o", tmp.resolve("out").toString(), - "--schema", "v1", "--workspace-root", tmp.toString()), + "--schema", "v1", "--namespace", "team"), "v1 keys are absolute: the flag has nothing to relativize"); } + @Test + void omittedIdentityFlagsDoNotReusePreviousInvocation(@TempDir Path tmp) throws Exception { + Path in = project(tmp.resolve("inventory")); + assertEquals(0, run("-i", in.toString(), "-o", tmp.resolve("first").toString(), "--app-name", "other", "--namespace", "team")); + assertEquals(0, run("-i", in.toString(), "-o", tmp.resolve("second").toString())); + assertEquals("can://inventory/inventory", application(tmp.resolve("second")).get("id").getAsString()); + } + + @Test + void namespaceScopesReferencesAndCache(@TempDir Path tmp) throws Exception { + Path in = project(tmp.resolve("repo/services/app")); + Files.writeString(in.resolve("application.properties"), "port=8080\n"); + for (String ns : new String[] {"team one", "team two"}) { + Path out = tmp.resolve(ns); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "-a", "2", "--no-build", "--no-rta", + "--external-calls", "--namespace", ns, "--app-name", "widgets", "--cache-dir", tmp.resolve("cache").toString())); + JsonObject app = application(out); + assertEquals(CanId.applicationId("widgets", ns), app.get("id").getAsString()); + assertEquals("widgets", app.get("application_name").getAsString()); + assertEquals(ns, app.get("identity_namespace").getAsString()); + assertTrue(app.getAsJsonObject("symbol_table").has("src/main/java/com/example/Widget.java")); + assertTrue(app.getAsJsonObject("artifacts").has("application.properties")); + assertTrue(app.has("external_symbols")); + assertOwnedReferences(app, CanId.applicationId("widgets", ns)); + var analysis = com.ibm.cldk.schema.V2Json.compact().fromJson( + Files.readString(out.resolve("analysis.json")), com.ibm.cldk.schema.Analysis.class); + var rows = com.ibm.cldk.neo4j.V2GraphProjector.project(analysis, "widgets"); + String root = CanId.applicationId("widgets", ns); + for (var node : rows.nodes) { + if (node.value.startsWith("can://")) assertTrue(node.value.equals(root) || node.value.startsWith(root + "/")); + } + for (var edge : rows.edges) { + for (String id : new String[] {edge.from.value, edge.to.value}) { + if (id.startsWith("can://")) assertTrue(id.equals(root) || id.startsWith(root + "/")); + } + } + + assertConformsToV2Schema(out); + } + } + + private static void assertOwnedReferences(JsonElement element, String root) { + if (element.isJsonObject()) { + for (var e : element.getAsJsonObject().entrySet()) { + if (e.getKey().startsWith("can://")) assertTrue(e.getKey().startsWith(root + "/")); + assertOwnedReferences(e.getValue(), root); + } + } else if (element.isJsonArray()) { + for (var e : element.getAsJsonArray()) assertOwnedReferences(e, root); + } else if (element.isJsonPrimitive() && element.getAsJsonPrimitive().isString()) { + String value = element.getAsString(); + if (value.startsWith("can://")) assertTrue(value.equals(root) || value.startsWith(root + "/"), value); + } + } + + @Test + void invalidExplicitIdentityNeverDefaults(@TempDir Path tmp) throws Exception { + Path in = project(tmp.resolve("app")); + for (String flag : new String[] {"--app-name", "--namespace"}) { + for (String value : new String[] {"", " ", "a/b", ".."}) { + assertNotEquals(0, run("-i", in.toString(), "-o", tmp.resolve("out").toString(), flag, value)); + assertFalse(Files.exists(tmp.resolve("out/analysis.json"))); + } + } + } + } diff --git a/src/test/java/com/ibm/cldk/L3WalaOverlaysTest.java b/src/test/java/com/ibm/cldk/L3WalaOverlaysTest.java index f3f78f53..586646ee 100644 --- a/src/test/java/com/ibm/cldk/L3WalaOverlaysTest.java +++ b/src/test/java/com/ibm/cldk/L3WalaOverlaysTest.java @@ -28,6 +28,6 @@ public void deriveApplicationIdFallsBackToCanUnknownWhenTheModuleHasNoId() throw String applicationId = invokeDeriveApplicationId(modules); - assertEquals("can://unknown", applicationId); + assertEquals("can://unknown/unknown", applicationId); } } diff --git a/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java b/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java index 0c487799..fdc8d8c3 100644 --- a/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java +++ b/src/test/java/com/ibm/cldk/WorkspaceRootNeo4jPushTest.java @@ -29,7 +29,7 @@ import org.testcontainers.containers.Neo4jContainer; /** - * {@code --workspace-root} against a live Neo4j: the incremental push's wipe and orphan prune are + * Namespace identity against a live Neo4j: the incremental push's wipe and orphan prune are * scoped by the app's {@code can:///} prefix and each module's own id, so repo-relative module * ids must still be pruned when their file vanishes, and must not reach another application's graph. * @@ -92,11 +92,11 @@ private static void writeClass(Path srcRoot, String name) throws IOException { private static int push(Path in, Path repo, String app, Path out) { return new picocli.CommandLine(new CodeAnalyzer()).execute( "-i", in.toString(), "-o", out.toString(), "--emit", "neo4j", "--no-build", - "--workspace-root", repo.toString(), "--app-name", app, + "--namespace", repo.getFileName().toString(), "--app-name", app, "--neo4j-uri", CONTAINER.getBoltUrl(), "--neo4j-user", "neo4j", "--neo4j-password", PASSWORD); } - /** As {@link #push}, without {@code --workspace-root}: keys are relative to {@code in}. */ + /** As {@link #push}, with the default namespace: keys remain relative to {@code in}. */ private static int pushUnrooted(Path in, String app, Path out) { return new picocli.CommandLine(new CodeAnalyzer()).execute( "-i", in.toString(), "-o", out.toString(), "--emit", "neo4j", "--no-build", @@ -144,38 +144,27 @@ private static void writeConfig(Path project, String... keys) throws IOException } @Test - void togglingWorkspaceRootLeavesNoOldKeyedArtifacts(@TempDir Path tmp) throws IOException { + void changingNamespacePreservesPreviousApplicationAndUsesInputRelativeKeys(@TempDir Path tmp) throws IOException { Path repo = tmp.resolve("repo"); Path app = repo.resolve("services/app"); writeClass(app, "Main"); writeArtifacts(app, "org.example:lib"); writeConfig(app, "server.port"); - assertEquals(0, pushUnrooted(app, "toggle", tmp.resolve("o1"))); String oldPom = CanId.artifactId("toggle", "pom.xml"); - assertEquals(1, count("MATCH (a:Artifact {id: $id}) RETURN count(a)", "id", oldPom), - "precondition: the unrooted push keys the artifact on its input-relative path"); - assertTrue(count("MATCH (k:ConfigKey) WHERE k.id STARTS WITH $p RETURN count(k)", - "p", oldPom.replace("pom.xml", "src/main/resources/application.properties")) > 0, - "precondition: the config file minted a key"); - + assertEquals(1, count("MATCH (a:Artifact {id: $id}) RETURN count(a)", "id", oldPom)); assertEquals(0, push(app, repo, "toggle", tmp.resolve("o2"))); - - String rooted = artifactPrefix("toggle") + "services/app/"; - assertEquals(0, count("MATCH (a:Artifact) WHERE a.id STARTS WITH $p AND NOT a.id STARTS WITH $r " - + "RETURN count(a)", "p", artifactPrefix("toggle"), "r", rooted), - "no artifact keyed on the old input-relative path survives"); - assertEquals(0, count("MATCH (k:ConfigKey) WHERE k.id STARTS WITH $p AND NOT k.id STARTS WITH $r " - + "RETURN count(k)", "p", artifactPrefix("toggle"), "r", rooted), - "no config key keyed on the old input-relative path survives"); - long artifacts = count("MATCH (a:Artifact) WHERE a.id STARTS WITH $r RETURN count(a)", "r", rooted); - assertTrue(artifacts > 0, "the rooted push wrote its artifacts"); - assertEquals(artifacts, count("MATCH (:JApplication {id: $app})-[r:HAS_ARTIFACT]->() RETURN count(r)", - "app", CanId.applicationId("toggle")), "one HAS_ARTIFACT per artifact the run emitted"); - assertEquals(1, count("MATCH (a:Artifact)-[r:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) " - + "WHERE a.id STARTS WITH $p RETURN count(r)", - "p", artifactPrefix("toggle"), "pkg", CanId.purlMaven("org.example", "lib")), - "the dependency is declared once, by the rooted manifest"); + assertEquals(1, count("MATCH (a:Artifact {id: $id}) RETURN count(a)", "id", oldPom), + "the prior namespace is outside reconciliation scope"); + String root = CanId.applicationId("toggle", "repo"); + String pom = CanId.artifactId("toggle", "pom.xml", "repo"); + assertEquals(1, count("MATCH (:JApplication {id: $app})-[:HAS_ARTIFACT]->(:Artifact {id: $pom}) RETURN count(*)", + "app", root, "pom", pom)); + String module = CanId.moduleId(root, "src/main/java/com/example/Main.java"); + assertEquals(1, count("MATCH (:JModule {id: $id}) RETURN count(*)", "id", module), + "same-file content hashes in another namespace cannot suppress module insertion"); + assertEquals(1, count("MATCH (:Artifact {id: $id})-[:DECLARES_DEPENDENCY]->(:Package {id: $pkg}) RETURN count(*)", + "id", pom, "pkg", CanId.purlMaven("org.example", "lib"))); } @Test @@ -221,7 +210,7 @@ void aSiblingAnalyzersArtifactsAndEdgesSurviveAJavaPush(@TempDir Path tmp) throw String pyOnly = CanId.artifactId("poly", "pyproject.toml"); String pyKey = CanId.configKeyId(pyOnly, "tool.name"); run("MATCH (a:Artifact {id: $pom}) SET a:PyCanNode " - + "MERGE (app:PyApplication {id: 'can://poly'}) MERGE (app)-[:HAS_ARTIFACT]->(a) " + + "MERGE (app:PyApplication {id: 'can://poly/poly'}) MERGE (app)-[:HAS_ARTIFACT]->(a) " + "MERGE (p:Package {id: 'pkg:pypi/requests'}) MERGE (a)-[:DECLARES_DEPENDENCY {_k: 'main'}]->(p) " + "MERGE (o:Artifact:PyCanNode {id: $py}) MERGE (app)-[:HAS_ARTIFACT]->(o) " + "MERGE (k:ConfigKey:PyCanNode {id: $key}) MERGE (o)-[:DEFINES_CONFIG]->(k) " @@ -276,7 +265,7 @@ void aConfigKeyAnotherAnalyzerClaimsSurvivesJavaDroppingIt(@TempDir Path tmp) th String props = CanId.artifactId("ckshare", "src/main/resources/application.properties"); String key = CanId.configKeyId(props, "shared.key"); run("MATCH (k:ConfigKey {id: $key}) SET k:PyCanNode " - + "MERGE (b:PyBodyNode {id: 'can://ckshare/python/x.py/f@1:1'}) MERGE (b)-[:PY_USES_CONFIG]->(k)", + + "MERGE (b:PyBodyNode {id: 'can://ckshare/ckshare/python/x.py/f@1:1'}) MERGE (b)-[:PY_USES_CONFIG]->(k)", "key", key); writeConfig(app, "other"); @@ -299,7 +288,7 @@ void anArtifactAnUnmarkedSiblingInventoriesSurvivesJavaDroppingIt(@TempDir Path // A pre-1.5.0 codeanalyzer-python: inventories the file via HAS_ARTIFACT, puts no marker on it. String pom = CanId.artifactId("oldsib", "pom.xml"); - run("MATCH (a:Artifact {id: $pom}) MERGE (p:PyApplication {id: 'can://oldsib'}) " + run("MATCH (a:Artifact {id: $pom}) MERGE (p:PyApplication {id: 'can://oldsib/oldsib'}) " + "MERGE (p)-[:HAS_ARTIFACT]->(a)", "pom", pom); Files.delete(app.resolve("pom.xml")); @@ -321,7 +310,7 @@ void codeEdgesIntoConfigFollowAConfigOnlyChange(@TempDir Path tmp) throws IOExce String env = CanId.artifactId("cfguse", ".env"); assertEquals(0, count("MATCH ()-[r:J_USES_CONFIG]->(k) WHERE k.id STARTS WITH $p RETURN count(r)", "p", env), "precondition: PORT is undeclared"); - assertEquals(1, count("MATCH (:JApplication {id: 'can://cfguse'})-[r:J_READS_CONFIG_UNRESOLVED {key: 'PORT'}]->() " + assertEquals(1, count("MATCH (:JApplication {id: 'can://cfguse/cfguse'})-[r:J_READS_CONFIG_UNRESOLVED {key: 'PORT'}]->() " + "RETURN count(r)"), "precondition: the read is unresolved"); // Only the config changes; Main.java's content_hash is identical. @@ -330,7 +319,7 @@ void codeEdgesIntoConfigFollowAConfigOnlyChange(@TempDir Path tmp) throws IOExce assertEquals(1, count("MATCH ()-[r:J_USES_CONFIG]->(:ConfigKey {id: $k}) RETURN count(r)", "k", CanId.configKeyId(env, "PORT")), "the newly declared key gains its read"); - assertEquals(0, count("MATCH (:JApplication {id: 'can://cfguse'})-[r:J_READS_CONFIG_UNRESOLVED {key: 'PORT'}]->() " + assertEquals(0, count("MATCH (:JApplication {id: 'can://cfguse/cfguse'})-[r:J_READS_CONFIG_UNRESOLVED {key: 'PORT'}]->() " + "RETURN count(r)"), "the stale unresolved read is gone"); Files.writeString(app.resolve(".env"), "OTHER=1\n", StandardCharsets.UTF_8); @@ -384,7 +373,7 @@ void anUnchangedRepushIsIdempotent(@TempDir Path tmp) throws IOException { } @Test - void preRekeyArtifactsAreReleasedToo(@TempDir Path tmp) throws IOException { + void legacyArtifactsRequireExplicitRetirement(@TempDir Path tmp) throws IOException { Path app = tmp.resolve("app"); writeClass(app, "Main"); writeArtifacts(app, "org.example:lib"); @@ -397,8 +386,8 @@ void preRekeyArtifactsAreReleasedToo(@TempDir Path tmp) throws IOException { assertEquals(0, pushUnrooted(app, "legacy", tmp.resolve("o1"))); - assertEquals(0, count("MATCH (n) WHERE n.id STARTS WITH 'can://artifact/legacy/' RETURN count(n)"), - "the pre-re-key artifact and its config key are gone"); + assertEquals(2, count("MATCH (n) WHERE n.id STARTS WITH 'can://artifact/legacy/' RETURN count(n)"), + "legacy data is outside the selected canonical application"); assertEquals(1, count("MATCH (n:Artifact {id: $id}) RETURN count(n)", "id", CanId.artifactId("legacy", "pom.xml")), "the current artifact is written"); } @@ -460,15 +449,15 @@ void aVanishedRepoRelativeModuleIsPrunedAndOtherAppsAreUntouched(@TempDir Path t assertEquals(0, push(orders, repo, "orders", tmp.resolve("o1"))); assertEquals(0, push(billing, repo, "billing", tmp.resolve("o2"))); - String key = "services/orders/src/main/java/com/example/"; - String gone = CanId.moduleId(CanId.applicationId("orders"), key + "Gone.java"); - String keep = CanId.moduleId(CanId.applicationId("orders"), key + "Keep.java"); - String billingGone = CanId.moduleId(CanId.applicationId("billing"), - "services/billing/src/main/java/com/example/Gone.java"); + String key = "src/main/java/com/example/"; + String gone = CanId.moduleId(CanId.applicationId("orders", "repo"), key + "Gone.java"); + String keep = CanId.moduleId(CanId.applicationId("orders", "repo"), key + "Keep.java"); + String billingGone = CanId.moduleId(CanId.applicationId("billing", "repo"), + "src/main/java/com/example/Gone.java"); assertEquals(1, count("MATCH (m:JModule {id: $id}) RETURN count(m)", "id", gone), "precondition: the module is pushed under its repo-relative id"); long billingNodes = count("MATCH (n) WHERE n.id STARTS WITH $p RETURN count(n)", - "p", CanId.applicationId("billing") + "/"); + "p", CanId.applicationId("billing", "repo") + "/"); assertTrue(billingNodes > 0, "precondition: the other app is in the same database"); Files.delete(orders.resolve("src/main/java/com/example/Gone.java")); @@ -481,6 +470,6 @@ void aVanishedRepoRelativeModuleIsPrunedAndOtherAppsAreUntouched(@TempDir Path t assertEquals(1, count("MATCH (m:JModule {id: $id}) RETURN count(m)", "id", billingGone), "a same-named file in another app is a different node and is not touched"); assertEquals(billingNodes, count("MATCH (n) WHERE n.id STARTS WITH $p RETURN count(n)", - "p", CanId.applicationId("billing") + "/"), "the other app's graph is unchanged"); + "p", CanId.applicationId("billing", "repo") + "/"), "the other app's graph is unchanged"); } } diff --git a/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java b/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java index 097010e3..107ba941 100644 --- a/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java @@ -25,6 +25,28 @@ */ class BoltWriterPurgeTest { + @Test + void existingAnchorMustHaveExactMarkerAndRawIdentity() { + var identity = java.util.Map.of("identity_version", 2, + "application_name", "app", "identity_namespace", "team"); + BoltWriter.requireCompatibleApplicationIdentity("can://team/app", identity, identity); + for (var legacy : java.util.List.of(java.util.Map.of(), + java.util.Map.of("identity_version", 2.5, "application_name", "app", "identity_namespace", "team"), + java.util.Map.of("identity_version", 1, "application_name", "app", "identity_namespace", "team"), + java.util.Map.of("identity_version", 2, "application_name", "app", "identity_namespace", "other"))) { + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> BoltWriter.requireCompatibleApplicationIdentity("can://team/app", identity, legacy)); + } + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> BoltWriter.requireCompatibleApplicationIdentity("can://team/app", java.util.Map.of(), identity)); + } + + @Test + void incrementalHashesAreScopedToSelectedApplication() { + assertTrue(BoltWriter.MODULE_HASHES_V2.contains("{id: $appId}")); + assertTrue(BoltWriter.MODULE_HASHES_V2.contains("J_HAS_MODULE")); + } + @Test void thePurgeIsScopedByCanIdPrefixAndNotByTheRetiredProperty() { for (String stmt : new String[] {BoltWriter.PURGE_MODULE_EDGES_V2, BoltWriter.PURGE_VANISHED_NODES_V2}) { @@ -63,26 +85,12 @@ void theDescendantPrefixEndsWithTheSeparator() { } @Test - void theOrphanPruneMatchesAV2RootByIdAndALegacyV1RootByNameButNotAnotherApplication() { - // v2 roots are keyed on `id` -- unique-constrained and exact -- but legacy v1 roots - // (GraphProjector.java) never write `id`, only `name`. An id-only match orphans a prior v1 - // graph instead of pruning it; an ungated `a.name =` disjunct reaches every v2 root - // carrying that display name, including roots this analyzer never minted, since `name` - // stopped being unique once the id re-key landed. Note this is scoping hygiene, not - // disambiguation: CanId.applicationId is "can://" + appName, so two services sharing an - // --app-name share an id and are one node either way. + void orphanPruneNeverFallsBackToLegacyName() { String stmt = BoltWriter.PRUNE_VANISHED_UNITS_V2; - assertTrue(stmt.contains("a.id = $appId"), - "must reach a v2 root of THIS app by its unique id: " + stmt); - assertTrue(stmt.contains("a.id IS NULL AND a.name = $appName"), - "must reach a legacy v1 root (no id) by name, guarded by id IS NULL: " + stmt); - // Every `a.name =` must sit inside the `a.id IS NULL` guard: an unguarded one reaches v2 - // roots this analyzer never minted that merely carry the same display name. - assertFalse(stmt.replace("a.id IS NULL AND a.name = $appName", "").contains("a.name ="), - "a.name must never be matched outside the a.id IS NULL guard -- that would reach " - + "any v2 root carrying this display name, not just this app's: " + stmt); - assertFalse(stmt.contains("{name:"), - "the prune must not property-match :JApplication by its no-longer-unique name: " + stmt); + assertTrue(stmt.contains("a.id = $appId")); + assertFalse(stmt.contains("a.name")); + org.junit.jupiter.api.Assertions.assertEquals(java.util.List.of("can://team/app/artifact/"), + BoltWriter.artifactPrefixes("can://team/app", "app")); } @Test diff --git a/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java b/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java new file mode 100644 index 00000000..2ee7b680 --- /dev/null +++ b/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java @@ -0,0 +1,49 @@ +package com.ibm.cldk.neo4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.gson.JsonParser; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class CanonicalIdentityRowsTest { + @Test + void conflictingCanonicalEntityPropertiesAreRejected() { + RowBuilder builder = new RowBuilder(); + String id = "can://team/app/java/Foo.java/Foo"; + builder.node(List.of("JSymbol"), "id", id, Map.of("kind", "class")); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> builder.node(List.of("JType"), "id", id, Map.of("kind", "method"))); + } + + @Test + void canonicalIdsDeduplicateAcrossLabelsWithoutCollapsingEdgeKindsOrSites() throws Exception { + var fixture = JsonParser.parseReader(new InputStreamReader( + getClass().getResourceAsStream("/identity/namespaces.json"), StandardCharsets.UTF_8)) + .getAsJsonObject().getAsJsonObject("duplicate_module_rows"); + RowBuilder builder = new RowBuilder(); + GraphRows.NodeRef first = null; + GraphRows.NodeRef second = null; + for (var row : fixture.getAsJsonArray("rows")) { + String id = row.getAsJsonObject().get("module_id").getAsString(); + var ref = builder.node(List.of("JModule"), "id", id, Map.of("id", id)); + if (first == null) first = ref; + else if (!first.value.equals(id)) second = ref; + } + assertEquals(fixture.get("expected_unique_id_count").getAsInt(), builder.finish().nodes.size()); + var alternate = builder.node(List.of("JSymbol"), "id", first.value, Map.of("id", first.value)); + builder.keyedEdge("J_CALLS", first, second, Map.of(), "site1"); + builder.keyedEdge("J_CALLS", alternate, second, Map.of(), "site1"); + builder.keyedEdge("J_CALLS", first, second, Map.of(), "site2"); + builder.keyedEdge("J_RESOLVES_TO", first, second, Map.of(), "site1"); + var rows = builder.finish(); + assertEquals(2, rows.nodes.size()); + assertEquals(3, rows.edges.size()); + org.junit.jupiter.api.Assertions.assertTrue(rows.nodes.stream() + .filter(n -> n.value.equals(alternate.value)).findFirst().orElseThrow() + .labels.containsAll(List.of("JModule", "JSymbol", "JCanNode"))); + } +} diff --git a/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java b/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java index 84964c45..326aa544 100644 --- a/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java @@ -148,15 +148,9 @@ void theWipePreambleMatchesAV2RootByIdAndALegacyV1RootByNameButNotAnotherApplica // "can://" + appName, so those share an id and are one node regardless. String out = CypherWriter.renderCypher(new RowBuilder().finish(), "app"); - assertTrue(out.contains("a.id = 'can://app'"), + assertTrue(out.contains("a.id = 'can://app/app'"), "must reach a v2 root of THIS app by its unique id, got: " + out); - assertTrue(out.contains("a.id IS NULL AND a.name = 'app'"), - "must reach a legacy v1 root (no id) by name, guarded by id IS NULL, got: " + out); - // Every `a.name =` in the predicate must be inside the `a.id IS NULL AND ...` guard: an - // unguarded one reaches v2 roots this analyzer never minted that carry the same name. - assertFalse(out.replace("a.id IS NULL AND a.name = 'app'", "").contains("a.name ="), - "a.name must never be matched outside the a.id IS NULL guard -- that would reach " - + "any v2 root carrying this display name, not just this app's, got: " + out); + assertFalse(out.contains("a.name ="), "legacy roots require explicit retirement"); assertFalse(out.contains("JApplication {name:"), "the wipe must not property-match :JApplication by its no-longer-unique name: " + out); } @@ -173,7 +167,7 @@ void anEmptyApplicationNameIsRefusedRatherThanEmittingAnUnscopedWipe() { "an unscoped destructive statement must never be rendered, app name: " + name); } // ... and the guard lives on the prefix itself, so every future caller inherits it. - assertEquals("can://app/", CypherWriter.applicationPrefix("app"), + assertEquals("can://app/app/", CypherWriter.applicationPrefix("app"), "the swept prefix must end in the separator, or `can://appX` is in scope too"); } } diff --git a/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java b/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java index ec93318d..5d674a13 100644 --- a/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java @@ -258,7 +258,7 @@ public void wipeCoversBothGenerationsSoV2ReplacesAPriorV1Graph() { String cypher = CypherWriter.renderCypher(rows, APP_NAME); assertTrue(cypher.contains("J_HAS_UNIT|J_HAS_MODULE"), "the wipe must traverse both generations' unit relationship"); - assertTrue(cypher.contains("MATCH (s:JSymbol) WHERE NOT (s)--() DELETE s"), + assertFalse(cypher.contains("MATCH (s:JSymbol) WHERE NOT (s)--() DELETE s"), "the wipe must sweep orphaned symbols (v1 import-materialized type stubs)"); // The v1 path is NOT reachable by the can:// prefix sweep -- v1 ids are FQNs and v1 roots // carry no `id` at all -- so replacing the containment traversal with the sweep orphans @@ -267,7 +267,7 @@ public void wipeCoversBothGenerationsSoV2ReplacesAPriorV1Graph() { && cypher.contains("OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x)") && cypher.contains("DETACH DELETE x, c, a;"), "the v1-reaching containment traversal must survive alongside the prefix sweep: " + cypher); - assertTrue(cypher.contains("a.id IS NULL AND a.name = '" + APP_NAME + "'"), + assertFalse(cypher.contains("a.id IS NULL AND a.name = '" + APP_NAME + "'"), "a legacy v1 root has no id and is reachable only by the guarded name branch: " + cypher); for (String rel : new String[] {"J_HAS_CALLSITE", "J_HAS_COMMENT", "J_HAS_PARAMETER", "J_DECLARES", "J_HAS_METHOD", "J_HAS_BODY_NODE"}) { @@ -470,9 +470,9 @@ void theWipeReachesThisAppsArtifactsAndConfigKeysButNeverAPackage() { // analyzer's next push. :Package (a `pkg:` purl) sits under no application, so it stays out // by construction -- assert that, because it is the only thing left holding the line. String cypher = CypherWriter.renderCypher(rows, APP_NAME); - String prefix = "can://" + APP_NAME + "/"; + String prefix = CanId.applicationId(APP_NAME) + "/"; - assertTrue(cypher.contains("MATCH (x:" + RowBuilder.CAN_NODE + ") WHERE x.id = 'can://" + APP_NAME + assertTrue(cypher.contains("MATCH (x:" + RowBuilder.CAN_NODE + ") WHERE x.id = '" + CanId.applicationId(APP_NAME) + "' OR x.id STARTS WITH '" + prefix + "'"), "the wipe must sweep the whole app prefix, rooted on the label-anchored index: " + cypher); assertTrue(cypher.contains("CALL { WITH x DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS;"), diff --git a/src/test/java/com/ibm/cldk/schema/CanIdTest.java b/src/test/java/com/ibm/cldk/schema/CanIdTest.java index 38f0cb47..02d8f9a1 100644 --- a/src/test/java/com/ibm/cldk/schema/CanIdTest.java +++ b/src/test/java/com/ibm/cldk/schema/CanIdTest.java @@ -13,22 +13,43 @@ */ class CanIdTest { + @Test + void namespaceFixtureAndInvalidNames() throws Exception { + var fixture = com.google.gson.JsonParser.parseReader(new java.io.InputStreamReader( + getClass().getResourceAsStream("/identity/namespaces.json"), java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + for (var row : fixture.getAsJsonArray("roots")) { + var r = row.getAsJsonObject(); + String ns = r.get("namespace").isJsonNull() ? null : r.get("namespace").getAsString(); + assertEquals(r.get("expected_root").getAsString(), CanId.applicationId(r.get("application_name").getAsString(), ns)); + } + for (var row : fixture.getAsJsonArray("invalid")) { + String value = row.getAsJsonObject().get("value").getAsString(); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, () -> CanId.applicationId(value)); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, () -> CanId.applicationId("app", value)); + } + assertEquals("can://team/%21%27%28%29%2A~", CanId.applicationId("!'()*~", "team")); + for (String value : List.of("\u00a0", "\u2007", "\u202f", "\u0085")) { + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, () -> CanId.applicationId(value)); + } + assertEquals("can://team/%F0%9F%98%80", CanId.applicationId("😀", "team")); + } + @Test void applicationIdPutsTheAppOutermost() { - assertEquals("can://daytrader8", CanId.applicationId("daytrader8")); + assertEquals("can://daytrader8/daytrader8", CanId.applicationId("daytrader8")); } @Test void moduleIdNestsTheLanguageUnderTheApp() { String app = CanId.applicationId("daytrader8"); - assertEquals("can://daytrader8/java/src/Foo.java", CanId.moduleId(app, "src/Foo.java")); + assertEquals("can://daytrader8/daytrader8/java/src/Foo.java", CanId.moduleId(app, "src/Foo.java")); } @Test void moduleId_normalizesBackslashesAndLeadingDotSlash() { String app = CanId.applicationId("myapp"); assertEquals( - "can://myapp/java/a/b/C.java", CanId.moduleId(app, "./a\\b\\C.java")); + "can://myapp/myapp/java/a/b/C.java", CanId.moduleId(app, "./a\\b\\C.java")); } @Test @@ -36,22 +57,22 @@ void moduleId_keepsARepoRelativeKeyWhole() { // --workspace-root keys carry the path from the repository root; the leading-[./] strip must // only drop a `./`, never eat into the key's own first segment. String app = CanId.applicationId("myapp"); - assertEquals("can://myapp/java/services/app/src/Foo.java", + assertEquals("can://myapp/myapp/java/services/app/src/Foo.java", CanId.moduleId(app, "services/app/src/Foo.java")); - assertEquals("can://myapp/java/services/app/src/Foo.java", + assertEquals("can://myapp/myapp/java/services/app/src/Foo.java", CanId.moduleId(app, "./services/app/src/Foo.java")); } @Test void childId_appendsSegmentWithSlash() { assertEquals( - "can://myapp/java/src/Foo.java/com.example.Foo", - CanId.childId("can://myapp/java/src/Foo.java", "com.example.Foo")); + "can://myapp/myapp/java/src/Foo.java/com.example.Foo", + CanId.childId("can://myapp/myapp/java/src/Foo.java", "com.example.Foo")); } @Test void ordinalId_appendsTagAfterAt() { - String callableId = "can://myapp/java/src/Foo.java/com.example.Foo/bar(int)"; + String callableId = "can://myapp/myapp/java/src/Foo.java/com.example.Foo/bar(int)"; assertEquals(callableId + "@15:2", CanId.ordinalId(callableId, "15:2")); assertEquals(callableId + "@entry", CanId.ordinalId(callableId, "entry")); } @@ -61,14 +82,14 @@ void externalIdIsLanguageNeutralUnderTheApp() { // No `/java/` segment: @external sits where the language sits for code nodes, so a sibling // analyzer over the same app mints the same id and merges onto the same node (#244). assertEquals( - "can://daytrader8/@external/java.util.Map/get(java.lang.Object)", + "can://daytrader8/daytrader8/@external/java.util.Map/get(java.lang.Object)", CanId.externalId("daytrader8", "java.util.Map", "get(java.lang.Object)")); } @Test void artifactIdNestsUnderTheAppInsteadOfAParallelScheme() { // Was can://artifact// — a third outermost shape. Now one rule. - assertEquals("can://daytrader8/artifact/pom.xml", CanId.artifactId("daytrader8", "pom.xml")); + assertEquals("can://daytrader8/daytrader8/artifact/pom.xml", CanId.artifactId("daytrader8", "pom.xml")); } @Test diff --git a/src/test/java/com/ibm/cldk/schema/L2SchemaOracleTest.java b/src/test/java/com/ibm/cldk/schema/L2SchemaOracleTest.java index c0eb77b1..0b61e752 100644 --- a/src/test/java/com/ibm/cldk/schema/L2SchemaOracleTest.java +++ b/src/test/java/com/ibm/cldk/schema/L2SchemaOracleTest.java @@ -44,11 +44,13 @@ class L2SchemaOracleTest { /** Wrap application-level members in an otherwise-minimal conformant envelope. */ private static String payload(String applicationMembers) { return "{" + + "\"identity_version\":2," + "\"schema_version\":\"2.0.0\"," + "\"language\":\"java\"," + "\"max_level\":2," + "\"application\":{" + "\"id\":\"" + APP_ID + "\"," + + "\"application_name\":\"myapp\",\"identity_namespace\":\"myapp\"," + "\"kind\":\"application\"," + "\"symbol_table\":{}" + (applicationMembers.isEmpty() ? "" : "," + applicationMembers) diff --git a/src/test/java/com/ibm/cldk/schema/L3SchemaOracleTest.java b/src/test/java/com/ibm/cldk/schema/L3SchemaOracleTest.java index 83be68ee..38c31b36 100644 --- a/src/test/java/com/ibm/cldk/schema/L3SchemaOracleTest.java +++ b/src/test/java/com/ibm/cldk/schema/L3SchemaOracleTest.java @@ -52,11 +52,13 @@ private static String payload(String overlays) { + "\"source\":\"\"," + "\"types\":{\"Foo\":" + type + "}}"; return "{" + + "\"identity_version\":2," + "\"schema_version\":\"2.0.0\"," + "\"language\":\"java\"," + "\"max_level\":3," + "\"application\":{" + "\"id\":\"" + APP_ID + "\"," + + "\"application_name\":\"myapp\",\"identity_namespace\":\"myapp\"," + "\"kind\":\"application\"," + "\"symbol_table\":{\"Foo.java\":" + module + "}}}"; } diff --git a/src/test/java/com/ibm/cldk/schema/L4SchemaOracleTest.java b/src/test/java/com/ibm/cldk/schema/L4SchemaOracleTest.java index ef26b345..3a199025 100644 --- a/src/test/java/com/ibm/cldk/schema/L4SchemaOracleTest.java +++ b/src/test/java/com/ibm/cldk/schema/L4SchemaOracleTest.java @@ -53,11 +53,13 @@ private static String payload(String localId, String kind) { + "\"source\":\"\"," + "\"types\":{\"Foo\":" + type + "}}"; return "{" + + "\"identity_version\":2," + "\"schema_version\":\"2.0.0\"," + "\"language\":\"java\"," + "\"max_level\":4," + "\"application\":{" + "\"id\":\"" + APP_ID + "\"," + + "\"application_name\":\"myapp\",\"identity_namespace\":\"myapp\"," + "\"kind\":\"application\"," + "\"symbol_table\":{\"Foo.java\":" + module + "}}}"; } diff --git a/src/test/java/com/ibm/cldk/schema/V2JsonTest.java b/src/test/java/com/ibm/cldk/schema/V2JsonTest.java index 4c644786..5f1a5db3 100644 --- a/src/test/java/com/ibm/cldk/schema/V2JsonTest.java +++ b/src/test/java/com/ibm/cldk/schema/V2JsonTest.java @@ -46,6 +46,19 @@ private static JsonObject payload() { return V2Json.compact().toJsonTree(V2Emitter.emit("myapp", 1, modules)).getAsJsonObject(); } + @Test + void identityMetadataIsRequiredAndNotDefaultedWhenReadingOldPayloads() { + JsonObject json = payload(); + assertEquals(2, json.get("identity_version").getAsInt()); + assertEquals("myapp", json.getAsJsonObject("application").get("application_name").getAsString()); + assertEquals("myapp", json.getAsJsonObject("application").get("identity_namespace").getAsString()); + json.remove("identity_version"); + Analysis legacy = V2Json.compact().fromJson(json, Analysis.class); + assertNull(legacy.getIdentityVersion()); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> com.ibm.cldk.neo4j.V2GraphProjector.project(legacy, "myapp")); + } + private static JsonObject module() { return payload().getAsJsonObject("application").getAsJsonObject("symbol_table").getAsJsonObject(FILE_KEY); } diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/L1CacheTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/L1CacheTest.java index 2631f2f4..46155a2f 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/L1CacheTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/L1CacheTest.java @@ -54,6 +54,16 @@ private static String hintOf(Map modules) { .getBody().get("4:5").getDeclaringTypeHint(); } + @Test + void missingIdentityMarkerCannotReuseOldCache(@TempDir Path cacheDir) throws Exception { + L1Cache.save(cacheDir, APP, VERSION, modulesWithHint("hint")); + Path path = L1Cache.fileIn(cacheDir); + String json = java.nio.file.Files.readString(path).replace("\"identity_version\":2,", ""); + java.nio.file.Files.writeString(path, json); + org.junit.jupiter.api.Assertions.assertTrue(L1Cache.load(cacheDir, APP, VERSION).isEmpty()); + assertEquals(json, java.nio.file.Files.readString(path), "incompatible persisted input remains read-only"); + } + @Test void save_thenLoad_preservesTheDeclaringTypeHint(@TempDir Path cacheDir) { L1Cache.save(cacheDir, APP, VERSION, modulesWithHint("java.util.Map$Entry")); diff --git a/src/test/resources/schema/analysis.v2.schema.json b/src/test/resources/schema/analysis.v2.schema.json index 18107ffa..efcd4e41 100644 --- a/src/test/resources/schema/analysis.v2.schema.json +++ b/src/test/resources/schema/analysis.v2.schema.json @@ -5,8 +5,9 @@ "description": "Conformance oracle for codeanalyzer-java's v2 output. Encodes the canonical CPG shape: an envelope carrying one application node, whose containment tree is named maps down to callable depth, with body nodes keyed by local id. Strict (additionalProperties: false) so an accidentally renamed or stray key fails the gate rather than silently reaching consumers. Replace with the SDK's v2 models once they exist.", "type": "object", "additionalProperties": false, - "required": ["schema_version", "language", "max_level", "application"], + "required": ["identity_version", "schema_version", "language", "max_level", "application"], "properties": { + "identity_version": { "const": 2 }, "schema_version": { "const": "2.0.0" }, "language": { "const": "java" }, "max_level": { "type": "integer", "minimum": 1, "maximum": 4 }, @@ -27,12 +28,12 @@ "canId": { "type": "string", - "pattern": "^can://[^/]+/java/", + "pattern": "^can://[^/]+/[^/]+/java/", "description": "Durable id for nodes at or above callable depth: can:///java/..." }, "applicationCanId": { "type": "string", - "pattern": "^can://[^/]+$", + "pattern": "^can://[^/]+/[^/]+$", "description": "Id of the application root node: can://, no trailing segment." }, @@ -105,9 +106,11 @@ "application": { "type": "object", "additionalProperties": false, - "required": ["id", "kind", "symbol_table"], + "required": ["id", "kind", "symbol_table", "application_name", "identity_namespace"], "properties": { "id": { "$ref": "#/$defs/applicationCanId" }, + "application_name": { "type": "string", "minLength": 1 }, + "identity_namespace": { "type": "string", "minLength": 1 }, "kind": { "const": "application" }, "symbol_table": { "type": "object", @@ -280,7 +283,7 @@ "externalCanId": { "type": "string", - "pattern": "^can://[^/]+/@external/", + "pattern": "^can://[^/]+/[^/]+/@external/", "description": "Id of a symbol outside the project, so an in-project id landing in the external map fails the gate instead of validating happily. Language-NEUTRAL like artifactCanId: @external sits where the language segment sits for code nodes, so sibling analyzers over the same name a library symbol identically. It therefore does NOT match canId." }, @@ -306,7 +309,7 @@ "artifactCanId": { "type": "string", - "pattern": "^can://[^/]+/artifact/", + "pattern": "^can://[^/]+/[^/]+/artifact/", "description": "Id of a repository-artifact node: nested under the app, language-neutral." }, From cbfbd01dd9ed3f5bfca51237487d0273f4eff4ec Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Fri, 30 Oct 2026 14:57:24 -0400 Subject: [PATCH 09/11] fix: preserve explicit legacy Java graph cleanup --- .../java/com/ibm/cldk/neo4j/BoltWriter.java | 19 ++++++++++++++---- .../java/com/ibm/cldk/neo4j/CypherWriter.java | 20 ++++++++++++++++++- .../ibm/cldk/neo4j/BoltWriterPurgeTest.java | 16 +++++++++++++++ .../cldk/neo4j/CypherWriterStreamingTest.java | 18 +++++++++++++++++ 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java index 28961b9b..631fc659 100644 --- a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java @@ -87,6 +87,18 @@ public final class BoltWriter implements BoltSink { + "OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x) " + "DETACH DELETE x, c RETURN count(c) AS pruned"; + static final String PRUNE_VANISHED_UNITS_V1 = + "MATCH (a:JApplication)-[:J_HAS_UNIT]->(c) " + + "WHERE a.id IS NULL AND a.name = $appName " + + "AND NOT c.file_key IN $present " + + "OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x) " + + "DETACH DELETE x, c RETURN count(c) AS pruned"; + + static String orphanPruneQuery(GraphRows rows) { + return CypherWriter.legacyApplicationName(rows) != null + ? PRUNE_VANISHED_UNITS_V1 : PRUNE_VANISHED_UNITS_V2; + } + /** * Drop every Java-owned edge on this app's artifacts and config keys so the run rebuilds them: * {@code HAS_ARTIFACT} from a {@code :JApplication}, {@code DECLARES_DEPENDENCY}/{@code LOCKS} @@ -313,9 +325,8 @@ void run(GraphRows rows, boolean fullRun) { } upsertEdges(edges); - // 6. orphan prune — only safe on a full run. Reaches both generations' per-file nodes - // through the shared application anchor, so a v2 push also prunes a prior v1 graph's - // units (and other applications in the database are never touched). + // 6. Full-run orphan pruning uses the actual incoming schema family. A v1 request + // selects only an id-less legacy anchor; v2 selects only its exact canonical anchor. if (fullRun) { List present = new ArrayList<>(byUnit.keySet()); String appName = appNameOf(rows); @@ -323,7 +334,7 @@ void run(GraphRows rows, boolean fullRun) { // Only Java containment under this exact anchor is traversed. Shared artifacts // are reconciled separately and sibling namespaces are never fallback matches. try (Session s = session()) { - long pruned = s.run(PRUNE_VANISHED_UNITS_V2, + long pruned = s.run(orphanPruneQuery(rows), Values.parameters("present", present, "appId", appId, "appName", appName)) .single().get("pruned").asLong(0); Log.info("neo4j(bolt): pruned " + pruned + " vanished unit(s)"); diff --git a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java index 88cb359f..2e469aa8 100644 --- a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java @@ -92,7 +92,8 @@ public static void writeCypher(Appendable out, GraphRows rows, String appName) t s.add(""); s.add("// ── wipe this project's prior subgraph (packages/annotations are shared and survive; " + "artifacts/config keys are inside the app prefix and get rebuilt) ──"); - s.add(wipe(applicationIdOf(rows, appName))); + String legacyName = legacyApplicationName(rows); + s.add(legacyName != null ? wipeLegacy(legacyName) : wipe(applicationIdOf(rows, appName))); s.add(""); s.add("// ── nodes ──"); @@ -143,6 +144,23 @@ static String applicationPrefix(String appName) { return CanId.applicationId(appName) + "/"; } + /** Actual name-keyed v1 rows select legacy cleanup; v2 never falls back by display name. */ + static String legacyApplicationName(GraphRows rows) { + for (NodeRow node : rows.nodes) { + if (node.labels.contains("JApplication") && "name".equals(node.keyProp)) { + return node.value; + } + } + return null; + } + + private static String wipeLegacy(String appName) { + return "MATCH (a:JApplication) WHERE a.id IS NULL AND a.name = " + cypherValue(appName) + "\n" + + "OPTIONAL MATCH (a)-[:J_HAS_UNIT]->(c)\n" + + "OPTIONAL MATCH (c)-" + DESCENDANTS + "->(x)\n" + + "DETACH DELETE x, c, a;\n"; + } + /** Canonical roots come from projected rows, preserving the selected namespace. */ private static String applicationIdOf(GraphRows rows, String appName) { for (NodeRow n : rows.nodes) { diff --git a/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java b/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java index 107ba941..a331fd6f 100644 --- a/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/BoltWriterPurgeTest.java @@ -25,6 +25,22 @@ */ class BoltWriterPurgeTest { + @Test + void fullRunPruneSelectsActualSchemaFamilyAndProtectsSharedDisplayNames() { + var legacy = GraphProjector.project(java.util.Map.of(), null, "shared"); + String query = BoltWriter.orphanPruneQuery(legacy); + assertTrue(query.contains("a.id IS NULL AND a.name = $appName")); + assertTrue(query.contains("[:J_HAS_UNIT]")); + assertTrue(query.contains("AND NOT c.file_key IN $present")); + assertTrue(query.contains("DETACH DELETE x, c")); + assertFalse(query.contains("a.id = $appId")); + var analysis = com.ibm.cldk.schema.V2Emitter.emit("shared", 1, java.util.Map.of(), null, + null, null, null, null, null, null, "team"); + String canonical = BoltWriter.orphanPruneQuery(V2GraphProjector.project(analysis, "shared")); + assertTrue(canonical.contains("a.id = $appId")); + assertFalse(canonical.contains("a.name")); + } + @Test void existingAnchorMustHaveExactMarkerAndRawIdentity() { var identity = java.util.Map.of("identity_version", 2, diff --git a/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java b/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java index 326aa544..cbb22946 100644 --- a/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/CypherWriterStreamingTest.java @@ -61,6 +61,24 @@ public void flush() {} public void close() {} } + @Test + void actualV1SnapshotsReplaceOnlyTheNameKeyedLegacyApplication() { + GraphRows legacy = GraphProjector.project(Map.of(), null, "shared"); + String script = CypherWriter.renderCypher(legacy, "shared"); + assertTrue(script.contains("a.id IS NULL AND a.name = 'shared'"), script); + assertTrue(script.contains("OPTIONAL MATCH (a)-[:J_HAS_UNIT]->(c)"), script); + assertTrue(script.contains("OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x)"), script); + assertTrue(script.contains("DETACH DELETE x, c, a;"), script); + assertFalse(script.contains("STARTS WITH"), "v1 must not sweep a synthesized canonical root"); + assertTrue(script.indexOf("DETACH DELETE x, c, a;") < script.indexOf("// ── nodes")); + var analysis = com.ibm.cldk.schema.V2Emitter.emit("shared", 1, Map.of(), null, + null, null, null, null, null, null, "team"); + String canonical = CypherWriter.renderCypher(V2GraphProjector.project(analysis, "shared"), "shared"); + assertTrue(canonical.contains("a.id = 'can://team/shared'")); + assertFalse(canonical.contains("a.name ="), "the same display name cannot activate v1 cleanup for v2"); + assertFalse(canonical.contains("can://shared/shared")); + } + @Test void streamedOutputIsIdenticalToTheRenderedString() throws IOException { GraphRows rows = sampleRows(1200); // > 2 batches of 500 From 89b95237ee2623dfdf84ddd6e71f07ab2472699b Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Sat, 31 Oct 2026 00:14:48 -0400 Subject: [PATCH 10/11] Document strict canonical node property conflict contract --- src/main/java/com/ibm/cldk/neo4j/RowBuilder.java | 4 +++- .../com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java b/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java index e11e9242..92d0eeae 100644 --- a/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java +++ b/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java @@ -28,7 +28,9 @@ * a target that never materialized are dropped (the "edge-only-when-resolved" rule). * *

    This is the in-memory analog of {@code MERGE (n:Label {key}) SET n += props}: re-seeing the - * same (mergeLabel, value) merges props (last write wins) and unions labels. + * same canonical ID merges compatible properties and unions labels. A differing value for any + * existing node property, including an ordinary display property, is rejected rather than refined. + * Noncanonical rows retain their label/key identity. */ public final class RowBuilder { diff --git a/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java b/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java index 2ee7b680..4b130388 100644 --- a/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/CanonicalIdentityRowsTest.java @@ -19,6 +19,15 @@ void conflictingCanonicalEntityPropertiesAreRejected() { () -> builder.node(List.of("JType"), "id", id, Map.of("kind", "method"))); } + @Test + void ordinaryCanonicalDisplayPropertyRefinementIsRejected() { + RowBuilder builder = new RowBuilder(); + String id = "can://team/app/java/Foo.java/Foo"; + builder.node(List.of("JSymbol"), "id", id, Map.of("name", "Foo")); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, + () -> builder.node(List.of("JSymbol"), "id", id, Map.of("name", "refined Foo"))); + } + @Test void canonicalIdsDeduplicateAcrossLabelsWithoutCollapsingEdgeKindsOrSites() throws Exception { var fixture = JsonParser.parseReader(new InputStreamReader( From 884d1def3fe3427270e9b701988d91ff440e229f Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Sat, 31 Oct 2026 08:58:45 -0400 Subject: [PATCH 11/11] chore: prepare 3.4.0 release --- CHANGELOG.md | 9 +++++++++ gradle.properties | 2 +- packaging/python/src/codeanalyzer_java/__init__.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..70877e3b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## [3.4.0] - 2026-10-31 + +- Add `--namespace` and namespace-qualified application IDs. +- Scope embedded references, graph reconciliation and caches to the selected application. +- Validate stored identity metadata before connected writes. + +Regenerate existing analyses and graphs from source for the new ID roots. diff --git a/gradle.properties b/gradle.properties index 427a07a3..e6640dcd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=3.3.4 +version=3.4.0 diff --git a/packaging/python/src/codeanalyzer_java/__init__.py b/packaging/python/src/codeanalyzer_java/__init__.py index 65375429..2069b67d 100644 --- a/packaging/python/src/codeanalyzer_java/__init__.py +++ b/packaging/python/src/codeanalyzer_java/__init__.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import List -__version__ = "0.0.0" +__version__ = "3.4.0" __all__ = ["jar_path", "java_path", "command", "__version__"]