From 47c85c59cea0bab045563141ca2d61d7d034b84e Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Mon, 19 Oct 2026 17:45:50 +0200 Subject: [PATCH 1/2] feat(trace): e2e checks for the remaining Claude Code rows Add end-to-end checks for hook installation and deletion, the trace run wrapper, a missed session start, partial commits, pasted image pointers, and media skipped by redaction (spec issue-3635). Generated line counts are reported as not covered, because neither the user nor the evidence can see them yet. The Claude Code cells that the suite verified are now Yes. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: be5af72c-30e7-48f4-b46c-dc7a5a754c41 --- app/cli/internal/trace/README.md | 11 +- app/cli/internal/trace/e2e/checks.go | 26 ++- app/cli/internal/trace/e2e/checks_more.go | 185 ++++++++++++++++++++++ app/cli/internal/trace/e2e/claude.go | 121 +++++++++++++- app/cli/internal/trace/e2e/harness.go | 25 +++ app/cli/internal/trace/e2e/images.go | 75 +++++++++ app/cli/internal/trace/e2e/images_test.go | 47 ++++++ app/cli/internal/trace/e2e/runner.go | 23 ++- app/cli/internal/trace/e2e/runner_test.go | 2 + app/cli/internal/trace/e2e/scenarios.go | 80 ++++++++++ app/cli/internal/trace/e2e/workspace.go | 43 ++++- 11 files changed, 619 insertions(+), 19 deletions(-) create mode 100644 app/cli/internal/trace/e2e/checks_more.go create mode 100644 app/cli/internal/trace/e2e/images.go create mode 100644 app/cli/internal/trace/e2e/images_test.go diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 90fd6aac8..3adb217c5 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -14,13 +14,14 @@ How Trace is added to a repository and to an agent. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | |---------|---------------|-------------|--------|--------------|------------| -| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Unknown | Unknown | Unknown | Unknown | -| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Unknown | Unknown | Unknown | Unknown | +| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Unknown | Unknown | Unknown | +| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Unknown | Unknown | Unknown | | More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | Why not Yes: -- **All rows, all agents:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **Hook installation and deletion, and Trace run wrapper, Cursor and OpenCode:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **More than one agent in one repository, all agents:** not verified yet. The end-to-end suite needs a second agent to test it (spec issue-3635). ## Session collection @@ -32,7 +33,7 @@ What the session evidence holds about the conversation. | Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | | Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | | Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | -| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Unknown | Unknown | Unknown | Unknown | +| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Unknown | Why not Yes: @@ -41,7 +42,7 @@ Why not Yes: - **Model detection, Cursor:** the primary model comes from the session-start hook. The transcript does not tell the models, so the evidence does not list the other models. - **Subagents, Cursor:** the provider does not read subagent conversations. - **Subagents, OpenCode:** a subagent runs in a child session with its own export. It is not part of the evidence of its parent session. -- **Missed session start, all agents:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **Missed session start, Cursor and OpenCode:** not verified yet. The end-to-end suite finds the values (spec issue-3635). ## Code attribution diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go index 874751adf..09e569f89 100644 --- a/app/cli/internal/trace/e2e/checks.go +++ b/app/cli/internal/trace/e2e/checks.go @@ -39,8 +39,11 @@ var errNotCovered = errors.New("not covered") type Session struct { Scenario string Model string + Harness Harness Workspace *Workspace - Output *RunOutput + // Checkout is the checkout where the agent committed and pushed. + Checkout string + Output *RunOutput // Stored is the session that the push stored in local storage mode. Stored *StoredSession // HeadMessage is the message of the commit that the agent made. @@ -59,6 +62,10 @@ type Feature struct { // ControlPlane is true for a feature that needs a control plane. The suite // runs in local storage mode only, so its cells are "not covered" (D-007). ControlPlane bool + // NotCovered is the reason why the suite cannot see the feature at all, + // for example because neither the user nor the evidence can observe it. + // Its cells are "not covered". + NotCovered string // ModelDriven is true for a feature that depends on the model following an // instruction. Its cells run more than once (R-006). ModelDriven bool @@ -69,7 +76,18 @@ type Feature struct { // features are the test definitions of the feature rows. A row of the table // with no entry here is reported as "no check". var features = map[string]Feature{ + // Setup + "Hook installation and deletion": {Checks: []Check{{scenarioSetup, checkHookInstallation}}}, + "Trace run wrapper": {Checks: []Check{ + {scenarioTraceRun, checkHookConfigRestored}, + {scenarioTraceRun, checkGenericSessionStored}, + }}, + // Session collection + "Missed session start": {Checks: []Check{ + {scenarioNoStart, checkNoSessionStartHook}, + {scenarioNoStart, checkCommitTrailer}, + }}, "Transcript": {Checks: []Check{{scenarioEditAndCommit, checkTranscript}}}, "Token usage and cost": {Checks: []Check{{scenarioEditAndCommit, checkTokenUsage}}}, "Model detection": {Checks: []Check{{scenarioEditAndCommit, checkModel}}}, @@ -78,6 +96,10 @@ var features = map[string]Feature{ // Code attribution "Line attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("greeting.txt")}}}, "Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}}, + "Partial commits": {Checks: []Check{{scenarioPartial, checkBothCommitsCredited}}}, + "Generated line counts": { + NotCovered: "neither the user nor the evidence can see the generated line counts yet", + }, // User experience "Welcome message": {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}}, @@ -93,10 +115,12 @@ var features = map[string]Feature{ "Spec source role, title, and description": { ModelDriven: true, Checks: []Check{{scenarioTicket, checkTicketHeader}}, }, + "Pasted image pointers": {Checks: []Check{{scenarioPastedImage, checkPastedImage}}}, // Security "Secret redaction": {Checks: []Check{{scenarioEditAndCommit, checkSecretRedacted}}}, "Allow markers in redaction": {Checks: []Check{{scenarioEditAndCommit, checkAllowedSecretKept}}}, + "Media skipped by redaction": {Checks: []Check{{scenarioMedia, checkMediaKept}}}, // Git integration "Commit trailer": {Checks: []Check{{scenarioEditAndCommit, checkCommitTrailer}}}, diff --git a/app/cli/internal/trace/e2e/checks_more.go b/app/cli/internal/trace/e2e/checks_more.go new file mode 100644 index 000000000..9f3fad2a6 --- /dev/null +++ b/app/cli/internal/trace/e2e/checks_more.go @@ -0,0 +1,185 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "image/png" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +// commandTimeout limits a command that a check runs in the workspace. +const commandTimeout = 2 * time.Minute + +// readHookConfig reads the hook configuration file of the harness. +func readHookConfig(s *Session) ([]byte, error) { + return os.ReadFile(filepath.Join(s.Workspace.Dir, s.Harness.HookConfig().Path)) +} + +// checkHookInstallation checks that `trace init` installs the Chainloop +// hooks next to a hook of the user, that a second init changes nothing, and +// that `trace uninstall` deletes only the Chainloop hooks. +func checkHookInstallation(s *Session) error { + hc := s.Harness.HookConfig() + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + + installed, err := readHookConfig(s) + if err != nil { + return err + } + if !bytes.Contains(installed, []byte(hc.ChainloopMarker)) { + return fmt.Errorf("trace init did not install the Chainloop hooks in %s", hc.Path) + } + if !bytes.Contains(installed, []byte(hc.UserMarker)) { + return fmt.Errorf("trace init deleted the hook of the user from %s", hc.Path) + } + + if _, err := s.Workspace.TraceInit(ctx, s.Harness); err != nil { + return err + } + again, err := readHookConfig(s) + if err != nil { + return err + } + if !bytes.Equal(installed, again) { + return fmt.Errorf("a second trace init changed %s", hc.Path) + } + + if _, err := s.Workspace.TraceUninstall(ctx); err != nil { + return err + } + after, err := readHookConfig(s) + if err != nil { + return err + } + if bytes.Contains(after, []byte(hc.ChainloopMarker)) { + return fmt.Errorf("trace uninstall left Chainloop hooks in %s", hc.Path) + } + if !bytes.Contains(after, []byte(hc.UserMarker)) { + return fmt.Errorf("trace uninstall deleted the hook of the user from %s", hc.Path) + } + return nil +} + +// checkHookConfigRestored checks that the hook configuration is the same as +// before the wrapped session. +func checkHookConfigRestored(s *Session) error { + got, err := readHookConfig(s) + if err != nil { + return err + } + if string(got) != s.Harness.HookConfig().UserConfig { + return fmt.Errorf("the hook configuration changed after trace run: %s", got) + } + return nil +} + +// checkGenericSessionStored checks that `trace run --local` stored the +// wrapped session as a generic session. +func checkGenericSessionStored(s *Session) error { + if mode := evidence(s).Session.Mode; mode != aicodingsession.ModeGeneric { + return fmt.Errorf("the stored session has mode %q, want %q", mode, aicodingsession.ModeGeneric) + } + return nil +} + +// checkNoSessionStartHook makes sure that the session-start hook did not +// fire, so that the scenario tests a missed session start. +func checkNoSessionStartHook(s *Session) error { + if _, err := hookMessage(s, "SessionStart"); err == nil { + return errors.New("the session-start hook fired, so the scenario does not test a missed start") + } + return nil +} + +// checkBothCommitsCredited checks that each of the two commits of a file +// credits the session in its trailer. +func checkBothCommitsCredited(s *Session) error { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + + out, err := s.Workspace.run(ctx, s.Checkout, "git", "log", "-2", "--format=%B%x00") + if err != nil { + return err + } + sessionID := evidence(s).Session.ID + for i, msg := range strings.Split(strings.TrimRight(strings.TrimSpace(out), "\x00"), "\x00") { + if !slices.Contains(state.ParseSessionIDsFromTrailer(msg), sessionID) { + return fmt.Errorf("commit %d of 2 does not credit the session: %q", 2-i, strings.TrimSpace(msg)) + } + } + return nil +} + +// checkPastedImage checks that the pasted image is stored once as an image +// spec entry, and that the transcript holds a pointer in place of the image. +func checkPastedImage(s *Session) error { + for _, e := range evidence(s).Spec { + if e.Kind != aicodingsession.SpecKindImage || e.Title != "Pasted image 1" { + continue + } + content, err := s.Stored.Material(e.Digest) + if err != nil { + return fmt.Errorf("read the image material: %w", err) + } + if _, err := png.Decode(bytes.NewReader(content)); err != nil { + return fmt.Errorf("the image material is not a valid PNG: %w", err) + } + + pasted := base64.StdEncoding.EncodeToString(pastedPNG()) + hasPointer := false + for _, msg := range evidence(s).RawSession["main"] { + if bytes.Contains(msg, []byte(pasted)) { + return errors.New("the transcript still holds the pasted image inline") + } + if bytes.Contains(msg, []byte(pointer.Marker)) && bytes.Contains(msg, []byte(e.Digest)) { + hasPointer = true + } + } + if !hasPointer { + return errors.New("the transcript has no pointer to the pasted image") + } + return nil + } + return errors.New(`the evidence has no image spec entry "Pasted image 1"`) +} + +// checkMediaKept checks that the redaction did not change the image that the +// agent read. Its base64 holds a fake secret, which a redaction that scans +// media would replace. +func checkMediaKept(s *Session) error { + found, err := storedContains(s, "/"+mediaSecret+"/") + if err != nil { + return err + } + if !found { + return errors.New("the stored transcript does not hold the image that the agent read, unchanged") + } + return nil +} diff --git a/app/cli/internal/trace/e2e/claude.go b/app/cli/internal/trace/e2e/claude.go index 42017b65e..b05f0d0c1 100644 --- a/app/cli/internal/trace/e2e/claude.go +++ b/app/cli/internal/trace/e2e/claude.go @@ -19,9 +19,12 @@ import ( "bufio" "bytes" "context" + "encoding/base64" "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "strings" ) @@ -79,9 +82,31 @@ func (c *Claude) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { "--allowedTools", strings.Join(claudeTools, ","), } - cmd := exec.CommandContext(ctx, c.bin(), args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario + // A pasted image goes in a stream-json user message on stdin, the same + // content block that an interactive paste makes. + var stdin []byte + if len(req.Images) > 0 { + msg, err := claudeUserMessage(req.Prompt, req.Images) + if err != nil { + return nil, err + } + stdin = msg + // The prompt is in the message, so -p takes no argument. + args = append([]string{"-p", "--input-format", "stream-json"}, args[2:]...) + } + + name := c.bin() + if len(req.Wrapper) > 0 { + name = req.Wrapper[0] + args = append(append(append([]string{}, req.Wrapper[1:]...), c.bin()), args...) + } + + cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario cmd.Dir = req.Dir cmd.Env = req.Env + if stdin != nil { + cmd.Stdin = bytes.NewReader(stdin) + } var stderr bytes.Buffer cmd.Stderr = &stderr @@ -98,6 +123,100 @@ func (c *Claude) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { return &RunOutput{Raw: raw, Hooks: hooks, HooksReported: true}, nil } +// claudeContent is one content block of a stream-json user message. +type claudeContent struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + Source *claudeImageSource `json:"source,omitempty"` +} + +type claudeImageSource struct { + Type string `json:"type"` + MediaType string `json:"media_type"` + Data string `json:"data"` +} + +// claudeUserMessage builds the stream-json user message of a prompt with +// pasted PNG images. +func claudeUserMessage(prompt string, images [][]byte) ([]byte, error) { + content := make([]claudeContent, 0, len(images)+1) + for _, img := range images { + content = append(content, claudeContent{Type: "image", Source: &claudeImageSource{ + Type: "base64", MediaType: "image/png", Data: base64.StdEncoding.EncodeToString(img), + }}) + } + content = append(content, claudeContent{Type: "text", Text: prompt}) + + msg := struct { + Type string `json:"type"` + Message struct { + Role string `json:"role"` + Content []claudeContent `json:"content"` + } `json:"message"` + }{Type: "user"} + msg.Message.Role = "user" + msg.Message.Content = content + + data, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("encode claude user message: %w", err) + } + return append(data, '\n'), nil +} + +// claudeSettingsPath is the project settings file where `trace init` installs +// the Claude Code hooks. +const claudeSettingsPath = ".claude/settings.json" + +// HookConfig implements Harness. +func (c *Claude) HookConfig() HookConfig { + return HookConfig{ + Path: claudeSettingsPath, + ChainloopMarker: "chainloop trace hook claude", + UserConfig: `{ + "hooks": { + "Stop": [ + {"hooks": [{"type": "command", "command": "echo e2e-user-hook"}]} + ] + } +} +`, + UserMarker: "e2e-user-hook", + } +} + +// DisableSessionStart implements Harness. It deletes the SessionStart entry +// from the hooks of the project settings and keeps the other entries as +// they are. +func (c *Claude) DisableSessionStart(repo string) error { + path := filepath.Join(repo, claudeSettingsPath) + data, err := os.ReadFile(path) + if err != nil { + return err + } + + var settings map[string]json.RawMessage + if err := json.Unmarshal(data, &settings); err != nil { + return fmt.Errorf("decode %s: %w", claudeSettingsPath, err) + } + var hooks map[string]json.RawMessage + if err := json.Unmarshal(settings["hooks"], &hooks); err != nil { + return fmt.Errorf("decode the hooks of %s: %w", claudeSettingsPath, err) + } + if _, ok := hooks["SessionStart"]; !ok { + return fmt.Errorf("%s has no SessionStart hook", claudeSettingsPath) + } + delete(hooks, "SessionStart") + + if settings["hooks"], err = json.Marshal(hooks); err != nil { + return err + } + if data, err = json.MarshalIndent(settings, "", " "); err != nil { + return err + } + return os.WriteFile(path, data, 0o600) +} + // claudeStreamEvent is the part of a stream-json line that the suite reads. A // hook event has type "system" and subtype "hook_response". Its stdout is // the response that the hook printed. diff --git a/app/cli/internal/trace/e2e/harness.go b/app/cli/internal/trace/e2e/harness.go index ddba863ca..5240b7047 100644 --- a/app/cli/internal/trace/e2e/harness.go +++ b/app/cli/internal/trace/e2e/harness.go @@ -30,6 +30,25 @@ type RunRequest struct { Prompt string // Env is the environment of the harness process. Env []string + // Wrapper is a command that runs the harness, for example + // `chainloop trace run --local --claude --`. Empty runs the harness + // directly. + Wrapper []string + // Images are PNG images that the user pastes with the prompt. + Images [][]byte +} + +// HookConfig describes the file where the harness reads its hooks. +type HookConfig struct { + // Path is the file, relative to the repository root. + Path string + // ChainloopMarker is a text that only a Chainloop hook in the file holds. + ChainloopMarker string + // UserConfig is the content of the file with one hook of the user and no + // Chainloop hook. + UserConfig string + // UserMarker is a text that only the hook of the user holds. + UserMarker string } // HookMessage is a message that a Trace hook returned to the harness. @@ -68,4 +87,10 @@ type Harness interface { // Run runs one headless session and returns its output. An error means // that the session could not run, not that a feature failed. Run(ctx context.Context, req RunRequest) (*RunOutput, error) + // HookConfig describes the hook configuration file of the harness. + HookConfig() HookConfig + // DisableSessionStart removes the session-start hook that `trace init` + // installed in the repository, so that a session starts with no + // session-start hook. + DisableSessionStart(repo string) error } diff --git a/app/cli/internal/trace/e2e/images.go b/app/cli/internal/trace/e2e/images.go new file mode 100644 index 000000000..d7c1f57bc --- /dev/null +++ b/app/cli/internal/trace/e2e/images.go @@ -0,0 +1,75 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/png" +) + +// mediaSecret is a fake AWS access key that the secret scanner detects. It is +// not a real key. The media image holds it in its base64 form, so a redaction +// that scans media would rewrite the image. +const mediaSecret = "AKIAQX7RZ2M4KJ6WPL5N" //nolint:gosec // gitleaks:allow a fake key inside the base64 of a test image + +// pngSignature and the IHDR chunk take the first 33 bytes of a PNG. +const pngHeaderLen = 8 + 25 + +// solidPNG returns a small PNG of one color. +func solidPNG(c color.RGBA) []byte { + img := image.NewRGBA(image.Rect(0, 0, 8, 8)) + for x := range 8 { + for y := range 8 { + img.Set(x, y, c) + } + } + var buf bytes.Buffer + _ = png.Encode(&buf, img) // writing to a buffer does not fail + return buf.Bytes() +} + +// pastedPNG is the image that the user pastes in the pasted image scenario. +func pastedPNG() []byte { + return solidPNG(color.RGBA{R: 200, G: 30, B: 30, A: 255}) +} + +// mediaPNG returns a valid PNG whose base64 form holds mediaSecret between +// two '/' characters, so that the secret scanner sees a whole word. A tEXt +// chunk after the IHDR chunk carries the bytes. Its keyword has 3 characters, +// so that the chunk data after the keyword starts on a 3-byte boundary of +// the file, where a base64 group starts. +func mediaPNG() []byte { + secret, _ := base64.StdEncoding.DecodeString(mediaSecret) // a constant of 20 base64 characters + // 0xFF bytes encode to "////" around the secret. + text := append(append([]byte("e2e\x00\xff\xff\xff"), secret...), 0xff, 0xff, 0xff) + + var chunk bytes.Buffer + _ = binary.Write(&chunk, binary.BigEndian, uint32(len(text))) + chunk.WriteString("tEXt") + chunk.Write(text) + _ = binary.Write(&chunk, binary.BigEndian, crc32.ChecksumIEEE(append([]byte("tEXt"), text...))) + + base := solidPNG(color.RGBA{R: 30, G: 30, B: 200, A: 255}) + out := make([]byte, 0, len(base)+chunk.Len()) + out = append(out, base[:pngHeaderLen]...) + out = append(out, chunk.Bytes()...) + return append(out, base[pngHeaderLen:]...) +} diff --git a/app/cli/internal/trace/e2e/images_test.go b/app/cli/internal/trace/e2e/images_test.go new file mode 100644 index 000000000..4050fcd9f --- /dev/null +++ b/app/cli/internal/trace/e2e/images_test.go @@ -0,0 +1,47 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "encoding/base64" + "image/png" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImages(t *testing.T) { + testCases := []struct { + name string + data []byte + }{ + {name: "pasted image", data: pastedPNG()}, + {name: "media image", data: mediaPNG()}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := png.Decode(bytes.NewReader(tc.data)) + require.NoError(t, err, "the image must stay a valid PNG") + }) + } + + t.Run("media image base64 holds the secret as a whole word", func(t *testing.T) { + encoded := base64.StdEncoding.EncodeToString(mediaPNG()) + assert.Contains(t, encoded, "/"+mediaSecret+"/") + }) +} diff --git a/app/cli/internal/trace/e2e/runner.go b/app/cli/internal/trace/e2e/runner.go index ae1d21b01..b3652c603 100644 --- a/app/cli/internal/trace/e2e/runner.go +++ b/app/cli/internal/trace/e2e/runner.go @@ -132,10 +132,12 @@ func (r *Runner) runHarnessModel(ctx context.Context, h Harness, version string, switch { case !support.Tested(): c.Status, c.Reason = StatusSkipped, "the feature table marks it "+string(support) - case !hasDef || (!def.ControlPlane && len(def.Checks) == 0): + case !hasDef || (!def.ControlPlane && def.NotCovered == "" && len(def.Checks) == 0): c.Status, c.Reason = StatusNoCheck, "no check is registered for this feature" case def.ControlPlane: c.Status, c.Reason = StatusNotCovered, "the feature needs a control plane" + case def.NotCovered != "": + c.Status, c.Reason = StatusNotCovered, def.NotCovered case versionErr != nil: c.Status, c.Reason = StatusError, versionErr.Error() default: @@ -211,12 +213,20 @@ func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Sc base := filepath.Join(r.WorkDir, dirName(h.Column()), dirName(model), sc.Name, strconv.Itoa(i+1)) r.Logf("%s / %s / %s #%d: start in %s", h.Column(), model, sc.Name, i+1, base) - ws, err := NewWorkspace(ctx, base, r.ChainloopBin, h, sc.Seed) + ws, err := NewWorkspace(ctx, base, r.ChainloopBin, h, sc) if err != nil { return nil, fmt.Errorf("create workspace: %w", err) } - out, runErr := h.Run(ctx, RunRequest{Dir: ws.Dir, Model: model, Prompt: sc.Prompt, Env: ws.Env}) + if sc.NoAgent { + return &Session{Scenario: sc.Name, Model: model, Harness: h, Workspace: ws, Output: &RunOutput{}, Checkout: ws.Dir}, nil + } + + req := RunRequest{Dir: ws.Dir, Model: model, Prompt: sc.Prompt, Env: ws.Env, Images: sc.Images} + if sc.WrapTraceRun { + req.Wrapper = []string{r.ChainloopBin, "trace", "run", "--local", h.TraceFlag(), "--"} + } + out, runErr := h.Run(ctx, req) if out != nil { // The harness output stays next to the workspace to debug a failure. if err := os.WriteFile(filepath.Join(base, "harness-output.jsonl"), out.Raw, 0o600); err != nil { @@ -227,7 +237,7 @@ func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Sc return nil, runErr } - s := &Session{Scenario: sc.Name, Model: model, Workspace: ws, Output: out} + s := &Session{Scenario: sc.Name, Model: model, Harness: h, Workspace: ws, Output: out, Checkout: ws.Dir} // The agent pushes in the session, as a user does. The pre-push hook // stores the session in local storage mode. The suite does not push after @@ -244,6 +254,7 @@ func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Sc } } + s.Checkout = checkout if s.HeadMessage, err = ws.run(ctx, checkout, "git", "log", "-1", "--format=%B"); err != nil { return nil, err } @@ -284,8 +295,10 @@ func evaluate(p plannedCell, sessions map[string][]sessionResult) Cell { return c } + // A scenario with no agent stores no session. Its checks read the + // workspace. var err error - if sr.session.Stored == nil { + if sr.session.Stored == nil && !scenarios[chk.Scenario].NoAgent { err = errNoStoredSession } else { err = chk.Verify(sr.session) diff --git a/app/cli/internal/trace/e2e/runner_test.go b/app/cli/internal/trace/e2e/runner_test.go index 668f69e27..cfc06b97b 100644 --- a/app/cli/internal/trace/e2e/runner_test.go +++ b/app/cli/internal/trace/e2e/runner_test.go @@ -136,6 +136,8 @@ func (fakeHarness) Version(context.Context) (string, error) { func (fakeHarness) Run(context.Context, RunRequest) (*RunOutput, error) { return nil, errors.New("not installed") } +func (fakeHarness) HookConfig() HookConfig { return HookConfig{} } +func (fakeHarness) DisableSessionStart(string) error { return errors.New("not installed") } func TestRunnerPlansCells(t *testing.T) { exp, err := ParseExpected(strings.NewReader(`| Feature | What it means | Claude Code | Cursor | diff --git a/app/cli/internal/trace/e2e/scenarios.go b/app/cli/internal/trace/e2e/scenarios.go index d7cbe0068..5c404783f 100644 --- a/app/cli/internal/trace/e2e/scenarios.go +++ b/app/cli/internal/trace/e2e/scenarios.go @@ -27,6 +27,21 @@ type Scenario struct { // Checkout is the checkout where the agent commits and pushes, relative // to the workspace base. Empty means the repository of the workspace. Checkout string + // Images are PNG images that the user pastes with the prompt. + Images [][]byte + // NoAgent runs no harness session. The checks run commands in the + // workspace themselves. + NoAgent bool + // NoInit skips `chainloop trace init` in the workspace. + NoInit bool + // WrapTraceRun runs the harness with `chainloop trace run --local`. + WrapTraceRun bool + // SeedUserHooks writes a hook configuration of the harness with one hook + // of the user before `chainloop trace init`. + SeedUserHooks bool + // DisableSessionStart removes the session-start hook after + // `chainloop trace init`. + DisableSessionStart bool } // The fake secrets of the scenarios. They match the GitHub token rule of the @@ -53,6 +68,18 @@ const ( scenarioSkill = "skill" scenarioSubagent = "subagent" scenarioClone = "clone" + scenarioSetup = "setup" + scenarioTraceRun = "trace-run" + scenarioNoStart = "no-session-start" + scenarioPartial = "partial-commit" + scenarioPastedImage = "pasted-image" + scenarioMedia = "media" +) + +const ( + wrappedFile = "wrapped.txt" + partialFile = "partial.txt" + mediaFile = "diagram.png" ) const ( @@ -123,4 +150,57 @@ Then run: git add farewell.txt && git commit -m "add farewell" && git push 2. cd ../` + cloneDir + ` && printf 'from the clone\n' > ` + cloneFile + ` 3. cd ../` + cloneDir + ` && git add ` + cloneFile + ` && git commit -m "add clone file" && git push`, }, + // setup runs no agent. Its check runs `trace init` and `trace uninstall` + // over a hook configuration that already has a hook of the user. + scenarioSetup: { + Name: scenarioSetup, + Seed: map[string]string{readmePath: seedReadme}, + NoAgent: true, + SeedUserHooks: true, + }, + scenarioTraceRun: { + Name: scenarioTraceRun, + Seed: map[string]string{readmePath: seedReadme}, + NoInit: true, + WrapTraceRun: true, + SeedUserHooks: true, + Prompt: "This is an automated test. With your file-writing tool, create the file " + wrappedFile + " with the single line: wrapped. Do nothing else.", + }, + scenarioNoStart: { + Name: scenarioNoStart, + Seed: map[string]string{readmePath: seedReadme}, + DisableSessionStart: true, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. With your file-writing tool, create the file greeting.txt with the single line: hello. +2. Run this shell command: git add greeting.txt && git commit -m "add greeting" && git push`, + }, + // partial-commit commits the first two AI lines of a file, then the + // other two. The first commit stages a blob that holds only the first + // two lines, with no change to the working tree. + scenarioPartial: { + Name: scenarioPartial, + Seed: map[string]string{readmePath: seedReadme}, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. With your file-writing tool, create the file ` + partialFile + ` with exactly these four lines: one, two, three, four. +2. Run this shell command: git update-index --add --cacheinfo "100644,$(head -n 2 ` + partialFile + ` | git hash-object -w --stdin),` + partialFile + `" && git commit -m "first half" +3. Run this shell command: git add ` + partialFile + ` && git commit -m "second half" && git push`, + }, + scenarioPastedImage: { + Name: scenarioPastedImage, + Seed: map[string]string{readmePath: seedReadme}, + Images: [][]byte{pastedPNG()}, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. With your file-writing tool, create the file color.txt with one word: the main color of the pasted image. +2. Run this shell command: git add color.txt && git commit -m "add color" && git push`, + }, + // media has the agent read an image file. The image stays inline in the + // tool result of the transcript, and its base64 holds a fake secret. + scenarioMedia: { + Name: scenarioMedia, + Seed: map[string]string{readmePath: seedReadme, mediaFile: string(mediaPNG())}, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. Read the image ` + mediaFile + ` with your file-reading tool. +2. With your file-writing tool, create the file seen.txt with the single line: seen. +3. Run this shell command: git add seen.txt && git commit -m "add seen" && git push`, + }, } diff --git a/app/cli/internal/trace/e2e/workspace.go b/app/cli/internal/trace/e2e/workspace.go index 783ce67f6..afb6a98ad 100644 --- a/app/cli/internal/trace/e2e/workspace.go +++ b/app/cli/internal/trace/e2e/workspace.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io/fs" + "maps" "os" "os/exec" "path/filepath" @@ -58,12 +59,15 @@ type Workspace struct { // included: the chainloop binary under test first in PATH, and an // isolated git configuration. Env []string + // Chainloop is the chainloop CLI under test. + Chainloop string } -// NewWorkspace creates a workspace under base. It writes the seed files, -// runs `chainloop trace init --local` for the harness, commits the result and -// pushes it, so the session starts from a clean, pushed tree. -func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, seed map[string]string) (*Workspace, error) { +// NewWorkspace creates a workspace under base for a scenario. It writes the +// seed files, runs `chainloop trace init --local` for the harness unless the +// scenario says no, commits the result and pushes it, so the session starts +// from a clean, pushed tree. +func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, sc Scenario) (*Workspace, error) { repo := filepath.Join(base, "repo") remote := filepath.Join(base, "remote.git") gitConfigPath := filepath.Join(base, "gitconfig") @@ -76,7 +80,8 @@ func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, see } w := &Workspace{ - Dir: repo, + Dir: repo, + Chainloop: chainloopBin, Env: append(os.Environ(), "PATH="+filepath.Dir(chainloopBin)+string(os.PathListSeparator)+os.Getenv("PATH"), "GIT_CONFIG_GLOBAL="+gitConfigPath, @@ -93,6 +98,12 @@ func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, see return nil, err } + seed := sc.Seed + if sc.SeedUserHooks { + hc := h.HookConfig() + seed = maps.Clone(seed) + seed[hc.Path] = hc.UserConfig + } for name, content := range seed { path := filepath.Join(repo, filepath.FromSlash(name)) if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { @@ -103,8 +114,15 @@ func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, see } } - if _, err := w.run(ctx, repo, chainloopBin, "trace", "init", "--local", h.TraceFlag()); err != nil { - return nil, err + if !sc.NoInit { + if _, err := w.TraceInit(ctx, h); err != nil { + return nil, err + } + } + if sc.DisableSessionStart { + if err := h.DisableSessionStart(repo); err != nil { + return nil, err + } } for _, args := range [][]string{ @@ -121,6 +139,17 @@ func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, see return w, nil } +// TraceInit runs `chainloop trace init --local` for the harness in the +// repository. +func (w *Workspace) TraceInit(ctx context.Context, h Harness) (string, error) { + return w.run(ctx, w.Dir, w.Chainloop, "trace", "init", "--local", h.TraceFlag()) +} + +// TraceUninstall runs `chainloop trace uninstall` in the repository. +func (w *Workspace) TraceUninstall(ctx context.Context) (string, error) { + return w.run(ctx, w.Dir, w.Chainloop, "trace", "uninstall", "--yes") +} + // Git runs a git command in the repository and returns its output. func (w *Workspace) Git(ctx context.Context, args ...string) (string, error) { return w.run(ctx, w.Dir, "git", args...) From 599b279ca1a86d18dfcaa924fc98a1fa98ad7540 Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Mon, 19 Oct 2026 18:33:31 +0200 Subject: [PATCH 2/2] fix(trace): keep the e2e fake key and harness command out of SAST findings Build the fake AWS key of the media image from two parts, so the source holds no key literal. Mark the harness command with nosemgrep, as the other commands of the CLI do. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: be5af72c-30e7-48f4-b46c-dc7a5a754c41 --- app/cli/internal/trace/e2e/claude.go | 2 +- app/cli/internal/trace/e2e/images.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/cli/internal/trace/e2e/claude.go b/app/cli/internal/trace/e2e/claude.go index b05f0d0c1..dfc5d5ff4 100644 --- a/app/cli/internal/trace/e2e/claude.go +++ b/app/cli/internal/trace/e2e/claude.go @@ -101,7 +101,7 @@ func (c *Claude) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { args = append(append(append([]string{}, req.Wrapper[1:]...), c.bin()), args...) } - cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario + cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario // nosemgrep cmd.Dir = req.Dir cmd.Env = req.Env if stdin != nil { diff --git a/app/cli/internal/trace/e2e/images.go b/app/cli/internal/trace/e2e/images.go index d7c1f57bc..9d7c1e654 100644 --- a/app/cli/internal/trace/e2e/images.go +++ b/app/cli/internal/trace/e2e/images.go @@ -27,8 +27,9 @@ import ( // mediaSecret is a fake AWS access key that the secret scanner detects. It is // not a real key. The media image holds it in its base64 form, so a redaction -// that scans media would rewrite the image. -const mediaSecret = "AKIAQX7RZ2M4KJ6WPL5N" //nolint:gosec // gitleaks:allow a fake key inside the base64 of a test image +// that scans media would rewrite the image. It is built from two parts, so +// that the source code holds no key that a scanner of the repository reports. +const mediaSecret = "AKIA" + "QX7RZ2M4KJ6WPL5N" // pngSignature and the IHDR chunk take the first 33 bytes of a PNG. const pngHeaderLen = 8 + 25