diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 90fd6aac8..3adb217c5 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -14,13 +14,14 @@ How Trace is added to a repository and to an agent. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | |---------|---------------|-------------|--------|--------------|------------| -| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Unknown | Unknown | Unknown | Unknown | -| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Unknown | Unknown | Unknown | Unknown | +| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Unknown | Unknown | Unknown | +| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Unknown | Unknown | Unknown | | More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | Why not Yes: -- **All rows, all agents:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **Hook installation and deletion, and Trace run wrapper, Cursor and OpenCode:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **More than one agent in one repository, all agents:** not verified yet. The end-to-end suite needs a second agent to test it (spec issue-3635). ## Session collection @@ -32,7 +33,7 @@ What the session evidence holds about the conversation. | Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | | Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | | Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | -| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Unknown | Unknown | Unknown | Unknown | +| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Unknown | Why not Yes: @@ -41,7 +42,7 @@ Why not Yes: - **Model detection, Cursor:** the primary model comes from the session-start hook. The transcript does not tell the models, so the evidence does not list the other models. - **Subagents, Cursor:** the provider does not read subagent conversations. - **Subagents, OpenCode:** a subagent runs in a child session with its own export. It is not part of the evidence of its parent session. -- **Missed session start, all agents:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **Missed session start, Cursor and OpenCode:** not verified yet. The end-to-end suite finds the values (spec issue-3635). ## Code attribution diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go index 874751adf..09e569f89 100644 --- a/app/cli/internal/trace/e2e/checks.go +++ b/app/cli/internal/trace/e2e/checks.go @@ -39,8 +39,11 @@ var errNotCovered = errors.New("not covered") type Session struct { Scenario string Model string + Harness Harness Workspace *Workspace - Output *RunOutput + // Checkout is the checkout where the agent committed and pushed. + Checkout string + Output *RunOutput // Stored is the session that the push stored in local storage mode. Stored *StoredSession // HeadMessage is the message of the commit that the agent made. @@ -59,6 +62,10 @@ type Feature struct { // ControlPlane is true for a feature that needs a control plane. The suite // runs in local storage mode only, so its cells are "not covered" (D-007). ControlPlane bool + // NotCovered is the reason why the suite cannot see the feature at all, + // for example because neither the user nor the evidence can observe it. + // Its cells are "not covered". + NotCovered string // ModelDriven is true for a feature that depends on the model following an // instruction. Its cells run more than once (R-006). ModelDriven bool @@ -69,7 +76,18 @@ type Feature struct { // features are the test definitions of the feature rows. A row of the table // with no entry here is reported as "no check". var features = map[string]Feature{ + // Setup + "Hook installation and deletion": {Checks: []Check{{scenarioSetup, checkHookInstallation}}}, + "Trace run wrapper": {Checks: []Check{ + {scenarioTraceRun, checkHookConfigRestored}, + {scenarioTraceRun, checkGenericSessionStored}, + }}, + // Session collection + "Missed session start": {Checks: []Check{ + {scenarioNoStart, checkNoSessionStartHook}, + {scenarioNoStart, checkCommitTrailer}, + }}, "Transcript": {Checks: []Check{{scenarioEditAndCommit, checkTranscript}}}, "Token usage and cost": {Checks: []Check{{scenarioEditAndCommit, checkTokenUsage}}}, "Model detection": {Checks: []Check{{scenarioEditAndCommit, checkModel}}}, @@ -78,6 +96,10 @@ var features = map[string]Feature{ // Code attribution "Line attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("greeting.txt")}}}, "Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}}, + "Partial commits": {Checks: []Check{{scenarioPartial, checkBothCommitsCredited}}}, + "Generated line counts": { + NotCovered: "neither the user nor the evidence can see the generated line counts yet", + }, // User experience "Welcome message": {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}}, @@ -93,10 +115,12 @@ var features = map[string]Feature{ "Spec source role, title, and description": { ModelDriven: true, Checks: []Check{{scenarioTicket, checkTicketHeader}}, }, + "Pasted image pointers": {Checks: []Check{{scenarioPastedImage, checkPastedImage}}}, // Security "Secret redaction": {Checks: []Check{{scenarioEditAndCommit, checkSecretRedacted}}}, "Allow markers in redaction": {Checks: []Check{{scenarioEditAndCommit, checkAllowedSecretKept}}}, + "Media skipped by redaction": {Checks: []Check{{scenarioMedia, checkMediaKept}}}, // Git integration "Commit trailer": {Checks: []Check{{scenarioEditAndCommit, checkCommitTrailer}}}, diff --git a/app/cli/internal/trace/e2e/checks_more.go b/app/cli/internal/trace/e2e/checks_more.go new file mode 100644 index 000000000..9f3fad2a6 --- /dev/null +++ b/app/cli/internal/trace/e2e/checks_more.go @@ -0,0 +1,185 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "image/png" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +// commandTimeout limits a command that a check runs in the workspace. +const commandTimeout = 2 * time.Minute + +// readHookConfig reads the hook configuration file of the harness. +func readHookConfig(s *Session) ([]byte, error) { + return os.ReadFile(filepath.Join(s.Workspace.Dir, s.Harness.HookConfig().Path)) +} + +// checkHookInstallation checks that `trace init` installs the Chainloop +// hooks next to a hook of the user, that a second init changes nothing, and +// that `trace uninstall` deletes only the Chainloop hooks. +func checkHookInstallation(s *Session) error { + hc := s.Harness.HookConfig() + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + + installed, err := readHookConfig(s) + if err != nil { + return err + } + if !bytes.Contains(installed, []byte(hc.ChainloopMarker)) { + return fmt.Errorf("trace init did not install the Chainloop hooks in %s", hc.Path) + } + if !bytes.Contains(installed, []byte(hc.UserMarker)) { + return fmt.Errorf("trace init deleted the hook of the user from %s", hc.Path) + } + + if _, err := s.Workspace.TraceInit(ctx, s.Harness); err != nil { + return err + } + again, err := readHookConfig(s) + if err != nil { + return err + } + if !bytes.Equal(installed, again) { + return fmt.Errorf("a second trace init changed %s", hc.Path) + } + + if _, err := s.Workspace.TraceUninstall(ctx); err != nil { + return err + } + after, err := readHookConfig(s) + if err != nil { + return err + } + if bytes.Contains(after, []byte(hc.ChainloopMarker)) { + return fmt.Errorf("trace uninstall left Chainloop hooks in %s", hc.Path) + } + if !bytes.Contains(after, []byte(hc.UserMarker)) { + return fmt.Errorf("trace uninstall deleted the hook of the user from %s", hc.Path) + } + return nil +} + +// checkHookConfigRestored checks that the hook configuration is the same as +// before the wrapped session. +func checkHookConfigRestored(s *Session) error { + got, err := readHookConfig(s) + if err != nil { + return err + } + if string(got) != s.Harness.HookConfig().UserConfig { + return fmt.Errorf("the hook configuration changed after trace run: %s", got) + } + return nil +} + +// checkGenericSessionStored checks that `trace run --local` stored the +// wrapped session as a generic session. +func checkGenericSessionStored(s *Session) error { + if mode := evidence(s).Session.Mode; mode != aicodingsession.ModeGeneric { + return fmt.Errorf("the stored session has mode %q, want %q", mode, aicodingsession.ModeGeneric) + } + return nil +} + +// checkNoSessionStartHook makes sure that the session-start hook did not +// fire, so that the scenario tests a missed session start. +func checkNoSessionStartHook(s *Session) error { + if _, err := hookMessage(s, "SessionStart"); err == nil { + return errors.New("the session-start hook fired, so the scenario does not test a missed start") + } + return nil +} + +// checkBothCommitsCredited checks that each of the two commits of a file +// credits the session in its trailer. +func checkBothCommitsCredited(s *Session) error { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + + out, err := s.Workspace.run(ctx, s.Checkout, "git", "log", "-2", "--format=%B%x00") + if err != nil { + return err + } + sessionID := evidence(s).Session.ID + for i, msg := range strings.Split(strings.TrimRight(strings.TrimSpace(out), "\x00"), "\x00") { + if !slices.Contains(state.ParseSessionIDsFromTrailer(msg), sessionID) { + return fmt.Errorf("commit %d of 2 does not credit the session: %q", 2-i, strings.TrimSpace(msg)) + } + } + return nil +} + +// checkPastedImage checks that the pasted image is stored once as an image +// spec entry, and that the transcript holds a pointer in place of the image. +func checkPastedImage(s *Session) error { + for _, e := range evidence(s).Spec { + if e.Kind != aicodingsession.SpecKindImage || e.Title != "Pasted image 1" { + continue + } + content, err := s.Stored.Material(e.Digest) + if err != nil { + return fmt.Errorf("read the image material: %w", err) + } + if _, err := png.Decode(bytes.NewReader(content)); err != nil { + return fmt.Errorf("the image material is not a valid PNG: %w", err) + } + + pasted := base64.StdEncoding.EncodeToString(pastedPNG()) + hasPointer := false + for _, msg := range evidence(s).RawSession["main"] { + if bytes.Contains(msg, []byte(pasted)) { + return errors.New("the transcript still holds the pasted image inline") + } + if bytes.Contains(msg, []byte(pointer.Marker)) && bytes.Contains(msg, []byte(e.Digest)) { + hasPointer = true + } + } + if !hasPointer { + return errors.New("the transcript has no pointer to the pasted image") + } + return nil + } + return errors.New(`the evidence has no image spec entry "Pasted image 1"`) +} + +// checkMediaKept checks that the redaction did not change the image that the +// agent read. Its base64 holds a fake secret, which a redaction that scans +// media would replace. +func checkMediaKept(s *Session) error { + found, err := storedContains(s, "/"+mediaSecret+"/") + if err != nil { + return err + } + if !found { + return errors.New("the stored transcript does not hold the image that the agent read, unchanged") + } + return nil +} diff --git a/app/cli/internal/trace/e2e/claude.go b/app/cli/internal/trace/e2e/claude.go index 42017b65e..dfc5d5ff4 100644 --- a/app/cli/internal/trace/e2e/claude.go +++ b/app/cli/internal/trace/e2e/claude.go @@ -19,9 +19,12 @@ import ( "bufio" "bytes" "context" + "encoding/base64" "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "strings" ) @@ -79,9 +82,31 @@ func (c *Claude) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { "--allowedTools", strings.Join(claudeTools, ","), } - cmd := exec.CommandContext(ctx, c.bin(), args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario + // A pasted image goes in a stream-json user message on stdin, the same + // content block that an interactive paste makes. + var stdin []byte + if len(req.Images) > 0 { + msg, err := claudeUserMessage(req.Prompt, req.Images) + if err != nil { + return nil, err + } + stdin = msg + // The prompt is in the message, so -p takes no argument. + args = append([]string{"-p", "--input-format", "stream-json"}, args[2:]...) + } + + name := c.bin() + if len(req.Wrapper) > 0 { + name = req.Wrapper[0] + args = append(append(append([]string{}, req.Wrapper[1:]...), c.bin()), args...) + } + + cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario // nosemgrep cmd.Dir = req.Dir cmd.Env = req.Env + if stdin != nil { + cmd.Stdin = bytes.NewReader(stdin) + } var stderr bytes.Buffer cmd.Stderr = &stderr @@ -98,6 +123,100 @@ func (c *Claude) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { return &RunOutput{Raw: raw, Hooks: hooks, HooksReported: true}, nil } +// claudeContent is one content block of a stream-json user message. +type claudeContent struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + Source *claudeImageSource `json:"source,omitempty"` +} + +type claudeImageSource struct { + Type string `json:"type"` + MediaType string `json:"media_type"` + Data string `json:"data"` +} + +// claudeUserMessage builds the stream-json user message of a prompt with +// pasted PNG images. +func claudeUserMessage(prompt string, images [][]byte) ([]byte, error) { + content := make([]claudeContent, 0, len(images)+1) + for _, img := range images { + content = append(content, claudeContent{Type: "image", Source: &claudeImageSource{ + Type: "base64", MediaType: "image/png", Data: base64.StdEncoding.EncodeToString(img), + }}) + } + content = append(content, claudeContent{Type: "text", Text: prompt}) + + msg := struct { + Type string `json:"type"` + Message struct { + Role string `json:"role"` + Content []claudeContent `json:"content"` + } `json:"message"` + }{Type: "user"} + msg.Message.Role = "user" + msg.Message.Content = content + + data, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("encode claude user message: %w", err) + } + return append(data, '\n'), nil +} + +// claudeSettingsPath is the project settings file where `trace init` installs +// the Claude Code hooks. +const claudeSettingsPath = ".claude/settings.json" + +// HookConfig implements Harness. +func (c *Claude) HookConfig() HookConfig { + return HookConfig{ + Path: claudeSettingsPath, + ChainloopMarker: "chainloop trace hook claude", + UserConfig: `{ + "hooks": { + "Stop": [ + {"hooks": [{"type": "command", "command": "echo e2e-user-hook"}]} + ] + } +} +`, + UserMarker: "e2e-user-hook", + } +} + +// DisableSessionStart implements Harness. It deletes the SessionStart entry +// from the hooks of the project settings and keeps the other entries as +// they are. +func (c *Claude) DisableSessionStart(repo string) error { + path := filepath.Join(repo, claudeSettingsPath) + data, err := os.ReadFile(path) + if err != nil { + return err + } + + var settings map[string]json.RawMessage + if err := json.Unmarshal(data, &settings); err != nil { + return fmt.Errorf("decode %s: %w", claudeSettingsPath, err) + } + var hooks map[string]json.RawMessage + if err := json.Unmarshal(settings["hooks"], &hooks); err != nil { + return fmt.Errorf("decode the hooks of %s: %w", claudeSettingsPath, err) + } + if _, ok := hooks["SessionStart"]; !ok { + return fmt.Errorf("%s has no SessionStart hook", claudeSettingsPath) + } + delete(hooks, "SessionStart") + + if settings["hooks"], err = json.Marshal(hooks); err != nil { + return err + } + if data, err = json.MarshalIndent(settings, "", " "); err != nil { + return err + } + return os.WriteFile(path, data, 0o600) +} + // claudeStreamEvent is the part of a stream-json line that the suite reads. A // hook event has type "system" and subtype "hook_response". Its stdout is // the response that the hook printed. diff --git a/app/cli/internal/trace/e2e/harness.go b/app/cli/internal/trace/e2e/harness.go index ddba863ca..5240b7047 100644 --- a/app/cli/internal/trace/e2e/harness.go +++ b/app/cli/internal/trace/e2e/harness.go @@ -30,6 +30,25 @@ type RunRequest struct { Prompt string // Env is the environment of the harness process. Env []string + // Wrapper is a command that runs the harness, for example + // `chainloop trace run --local --claude --`. Empty runs the harness + // directly. + Wrapper []string + // Images are PNG images that the user pastes with the prompt. + Images [][]byte +} + +// HookConfig describes the file where the harness reads its hooks. +type HookConfig struct { + // Path is the file, relative to the repository root. + Path string + // ChainloopMarker is a text that only a Chainloop hook in the file holds. + ChainloopMarker string + // UserConfig is the content of the file with one hook of the user and no + // Chainloop hook. + UserConfig string + // UserMarker is a text that only the hook of the user holds. + UserMarker string } // HookMessage is a message that a Trace hook returned to the harness. @@ -68,4 +87,10 @@ type Harness interface { // Run runs one headless session and returns its output. An error means // that the session could not run, not that a feature failed. Run(ctx context.Context, req RunRequest) (*RunOutput, error) + // HookConfig describes the hook configuration file of the harness. + HookConfig() HookConfig + // DisableSessionStart removes the session-start hook that `trace init` + // installed in the repository, so that a session starts with no + // session-start hook. + DisableSessionStart(repo string) error } diff --git a/app/cli/internal/trace/e2e/images.go b/app/cli/internal/trace/e2e/images.go new file mode 100644 index 000000000..9d7c1e654 --- /dev/null +++ b/app/cli/internal/trace/e2e/images.go @@ -0,0 +1,76 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/png" +) + +// mediaSecret is a fake AWS access key that the secret scanner detects. It is +// not a real key. The media image holds it in its base64 form, so a redaction +// that scans media would rewrite the image. It is built from two parts, so +// that the source code holds no key that a scanner of the repository reports. +const mediaSecret = "AKIA" + "QX7RZ2M4KJ6WPL5N" + +// pngSignature and the IHDR chunk take the first 33 bytes of a PNG. +const pngHeaderLen = 8 + 25 + +// solidPNG returns a small PNG of one color. +func solidPNG(c color.RGBA) []byte { + img := image.NewRGBA(image.Rect(0, 0, 8, 8)) + for x := range 8 { + for y := range 8 { + img.Set(x, y, c) + } + } + var buf bytes.Buffer + _ = png.Encode(&buf, img) // writing to a buffer does not fail + return buf.Bytes() +} + +// pastedPNG is the image that the user pastes in the pasted image scenario. +func pastedPNG() []byte { + return solidPNG(color.RGBA{R: 200, G: 30, B: 30, A: 255}) +} + +// mediaPNG returns a valid PNG whose base64 form holds mediaSecret between +// two '/' characters, so that the secret scanner sees a whole word. A tEXt +// chunk after the IHDR chunk carries the bytes. Its keyword has 3 characters, +// so that the chunk data after the keyword starts on a 3-byte boundary of +// the file, where a base64 group starts. +func mediaPNG() []byte { + secret, _ := base64.StdEncoding.DecodeString(mediaSecret) // a constant of 20 base64 characters + // 0xFF bytes encode to "////" around the secret. + text := append(append([]byte("e2e\x00\xff\xff\xff"), secret...), 0xff, 0xff, 0xff) + + var chunk bytes.Buffer + _ = binary.Write(&chunk, binary.BigEndian, uint32(len(text))) + chunk.WriteString("tEXt") + chunk.Write(text) + _ = binary.Write(&chunk, binary.BigEndian, crc32.ChecksumIEEE(append([]byte("tEXt"), text...))) + + base := solidPNG(color.RGBA{R: 30, G: 30, B: 200, A: 255}) + out := make([]byte, 0, len(base)+chunk.Len()) + out = append(out, base[:pngHeaderLen]...) + out = append(out, chunk.Bytes()...) + return append(out, base[pngHeaderLen:]...) +} diff --git a/app/cli/internal/trace/e2e/images_test.go b/app/cli/internal/trace/e2e/images_test.go new file mode 100644 index 000000000..4050fcd9f --- /dev/null +++ b/app/cli/internal/trace/e2e/images_test.go @@ -0,0 +1,47 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "encoding/base64" + "image/png" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImages(t *testing.T) { + testCases := []struct { + name string + data []byte + }{ + {name: "pasted image", data: pastedPNG()}, + {name: "media image", data: mediaPNG()}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := png.Decode(bytes.NewReader(tc.data)) + require.NoError(t, err, "the image must stay a valid PNG") + }) + } + + t.Run("media image base64 holds the secret as a whole word", func(t *testing.T) { + encoded := base64.StdEncoding.EncodeToString(mediaPNG()) + assert.Contains(t, encoded, "/"+mediaSecret+"/") + }) +} diff --git a/app/cli/internal/trace/e2e/runner.go b/app/cli/internal/trace/e2e/runner.go index ae1d21b01..b3652c603 100644 --- a/app/cli/internal/trace/e2e/runner.go +++ b/app/cli/internal/trace/e2e/runner.go @@ -132,10 +132,12 @@ func (r *Runner) runHarnessModel(ctx context.Context, h Harness, version string, switch { case !support.Tested(): c.Status, c.Reason = StatusSkipped, "the feature table marks it "+string(support) - case !hasDef || (!def.ControlPlane && len(def.Checks) == 0): + case !hasDef || (!def.ControlPlane && def.NotCovered == "" && len(def.Checks) == 0): c.Status, c.Reason = StatusNoCheck, "no check is registered for this feature" case def.ControlPlane: c.Status, c.Reason = StatusNotCovered, "the feature needs a control plane" + case def.NotCovered != "": + c.Status, c.Reason = StatusNotCovered, def.NotCovered case versionErr != nil: c.Status, c.Reason = StatusError, versionErr.Error() default: @@ -211,12 +213,20 @@ func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Sc base := filepath.Join(r.WorkDir, dirName(h.Column()), dirName(model), sc.Name, strconv.Itoa(i+1)) r.Logf("%s / %s / %s #%d: start in %s", h.Column(), model, sc.Name, i+1, base) - ws, err := NewWorkspace(ctx, base, r.ChainloopBin, h, sc.Seed) + ws, err := NewWorkspace(ctx, base, r.ChainloopBin, h, sc) if err != nil { return nil, fmt.Errorf("create workspace: %w", err) } - out, runErr := h.Run(ctx, RunRequest{Dir: ws.Dir, Model: model, Prompt: sc.Prompt, Env: ws.Env}) + if sc.NoAgent { + return &Session{Scenario: sc.Name, Model: model, Harness: h, Workspace: ws, Output: &RunOutput{}, Checkout: ws.Dir}, nil + } + + req := RunRequest{Dir: ws.Dir, Model: model, Prompt: sc.Prompt, Env: ws.Env, Images: sc.Images} + if sc.WrapTraceRun { + req.Wrapper = []string{r.ChainloopBin, "trace", "run", "--local", h.TraceFlag(), "--"} + } + out, runErr := h.Run(ctx, req) if out != nil { // The harness output stays next to the workspace to debug a failure. if err := os.WriteFile(filepath.Join(base, "harness-output.jsonl"), out.Raw, 0o600); err != nil { @@ -227,7 +237,7 @@ func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Sc return nil, runErr } - s := &Session{Scenario: sc.Name, Model: model, Workspace: ws, Output: out} + s := &Session{Scenario: sc.Name, Model: model, Harness: h, Workspace: ws, Output: out, Checkout: ws.Dir} // The agent pushes in the session, as a user does. The pre-push hook // stores the session in local storage mode. The suite does not push after @@ -244,6 +254,7 @@ func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Sc } } + s.Checkout = checkout if s.HeadMessage, err = ws.run(ctx, checkout, "git", "log", "-1", "--format=%B"); err != nil { return nil, err } @@ -284,8 +295,10 @@ func evaluate(p plannedCell, sessions map[string][]sessionResult) Cell { return c } + // A scenario with no agent stores no session. Its checks read the + // workspace. var err error - if sr.session.Stored == nil { + if sr.session.Stored == nil && !scenarios[chk.Scenario].NoAgent { err = errNoStoredSession } else { err = chk.Verify(sr.session) diff --git a/app/cli/internal/trace/e2e/runner_test.go b/app/cli/internal/trace/e2e/runner_test.go index 668f69e27..cfc06b97b 100644 --- a/app/cli/internal/trace/e2e/runner_test.go +++ b/app/cli/internal/trace/e2e/runner_test.go @@ -136,6 +136,8 @@ func (fakeHarness) Version(context.Context) (string, error) { func (fakeHarness) Run(context.Context, RunRequest) (*RunOutput, error) { return nil, errors.New("not installed") } +func (fakeHarness) HookConfig() HookConfig { return HookConfig{} } +func (fakeHarness) DisableSessionStart(string) error { return errors.New("not installed") } func TestRunnerPlansCells(t *testing.T) { exp, err := ParseExpected(strings.NewReader(`| Feature | What it means | Claude Code | Cursor | diff --git a/app/cli/internal/trace/e2e/scenarios.go b/app/cli/internal/trace/e2e/scenarios.go index d7cbe0068..5c404783f 100644 --- a/app/cli/internal/trace/e2e/scenarios.go +++ b/app/cli/internal/trace/e2e/scenarios.go @@ -27,6 +27,21 @@ type Scenario struct { // Checkout is the checkout where the agent commits and pushes, relative // to the workspace base. Empty means the repository of the workspace. Checkout string + // Images are PNG images that the user pastes with the prompt. + Images [][]byte + // NoAgent runs no harness session. The checks run commands in the + // workspace themselves. + NoAgent bool + // NoInit skips `chainloop trace init` in the workspace. + NoInit bool + // WrapTraceRun runs the harness with `chainloop trace run --local`. + WrapTraceRun bool + // SeedUserHooks writes a hook configuration of the harness with one hook + // of the user before `chainloop trace init`. + SeedUserHooks bool + // DisableSessionStart removes the session-start hook after + // `chainloop trace init`. + DisableSessionStart bool } // The fake secrets of the scenarios. They match the GitHub token rule of the @@ -53,6 +68,18 @@ const ( scenarioSkill = "skill" scenarioSubagent = "subagent" scenarioClone = "clone" + scenarioSetup = "setup" + scenarioTraceRun = "trace-run" + scenarioNoStart = "no-session-start" + scenarioPartial = "partial-commit" + scenarioPastedImage = "pasted-image" + scenarioMedia = "media" +) + +const ( + wrappedFile = "wrapped.txt" + partialFile = "partial.txt" + mediaFile = "diagram.png" ) const ( @@ -123,4 +150,57 @@ Then run: git add farewell.txt && git commit -m "add farewell" && git push 2. cd ../` + cloneDir + ` && printf 'from the clone\n' > ` + cloneFile + ` 3. cd ../` + cloneDir + ` && git add ` + cloneFile + ` && git commit -m "add clone file" && git push`, }, + // setup runs no agent. Its check runs `trace init` and `trace uninstall` + // over a hook configuration that already has a hook of the user. + scenarioSetup: { + Name: scenarioSetup, + Seed: map[string]string{readmePath: seedReadme}, + NoAgent: true, + SeedUserHooks: true, + }, + scenarioTraceRun: { + Name: scenarioTraceRun, + Seed: map[string]string{readmePath: seedReadme}, + NoInit: true, + WrapTraceRun: true, + SeedUserHooks: true, + Prompt: "This is an automated test. With your file-writing tool, create the file " + wrappedFile + " with the single line: wrapped. Do nothing else.", + }, + scenarioNoStart: { + Name: scenarioNoStart, + Seed: map[string]string{readmePath: seedReadme}, + DisableSessionStart: true, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. With your file-writing tool, create the file greeting.txt with the single line: hello. +2. Run this shell command: git add greeting.txt && git commit -m "add greeting" && git push`, + }, + // partial-commit commits the first two AI lines of a file, then the + // other two. The first commit stages a blob that holds only the first + // two lines, with no change to the working tree. + scenarioPartial: { + Name: scenarioPartial, + Seed: map[string]string{readmePath: seedReadme}, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. With your file-writing tool, create the file ` + partialFile + ` with exactly these four lines: one, two, three, four. +2. Run this shell command: git update-index --add --cacheinfo "100644,$(head -n 2 ` + partialFile + ` | git hash-object -w --stdin),` + partialFile + `" && git commit -m "first half" +3. Run this shell command: git add ` + partialFile + ` && git commit -m "second half" && git push`, + }, + scenarioPastedImage: { + Name: scenarioPastedImage, + Seed: map[string]string{readmePath: seedReadme}, + Images: [][]byte{pastedPNG()}, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. With your file-writing tool, create the file color.txt with one word: the main color of the pasted image. +2. Run this shell command: git add color.txt && git commit -m "add color" && git push`, + }, + // media has the agent read an image file. The image stays inline in the + // tool result of the transcript, and its base64 holds a fake secret. + scenarioMedia: { + Name: scenarioMedia, + Seed: map[string]string{readmePath: seedReadme, mediaFile: string(mediaPNG())}, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. Read the image ` + mediaFile + ` with your file-reading tool. +2. With your file-writing tool, create the file seen.txt with the single line: seen. +3. Run this shell command: git add seen.txt && git commit -m "add seen" && git push`, + }, } diff --git a/app/cli/internal/trace/e2e/workspace.go b/app/cli/internal/trace/e2e/workspace.go index 783ce67f6..afb6a98ad 100644 --- a/app/cli/internal/trace/e2e/workspace.go +++ b/app/cli/internal/trace/e2e/workspace.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io/fs" + "maps" "os" "os/exec" "path/filepath" @@ -58,12 +59,15 @@ type Workspace struct { // included: the chainloop binary under test first in PATH, and an // isolated git configuration. Env []string + // Chainloop is the chainloop CLI under test. + Chainloop string } -// NewWorkspace creates a workspace under base. It writes the seed files, -// runs `chainloop trace init --local` for the harness, commits the result and -// pushes it, so the session starts from a clean, pushed tree. -func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, seed map[string]string) (*Workspace, error) { +// NewWorkspace creates a workspace under base for a scenario. It writes the +// seed files, runs `chainloop trace init --local` for the harness unless the +// scenario says no, commits the result and pushes it, so the session starts +// from a clean, pushed tree. +func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, sc Scenario) (*Workspace, error) { repo := filepath.Join(base, "repo") remote := filepath.Join(base, "remote.git") gitConfigPath := filepath.Join(base, "gitconfig") @@ -76,7 +80,8 @@ func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, see } w := &Workspace{ - Dir: repo, + Dir: repo, + Chainloop: chainloopBin, Env: append(os.Environ(), "PATH="+filepath.Dir(chainloopBin)+string(os.PathListSeparator)+os.Getenv("PATH"), "GIT_CONFIG_GLOBAL="+gitConfigPath, @@ -93,6 +98,12 @@ func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, see return nil, err } + seed := sc.Seed + if sc.SeedUserHooks { + hc := h.HookConfig() + seed = maps.Clone(seed) + seed[hc.Path] = hc.UserConfig + } for name, content := range seed { path := filepath.Join(repo, filepath.FromSlash(name)) if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { @@ -103,8 +114,15 @@ func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, see } } - if _, err := w.run(ctx, repo, chainloopBin, "trace", "init", "--local", h.TraceFlag()); err != nil { - return nil, err + if !sc.NoInit { + if _, err := w.TraceInit(ctx, h); err != nil { + return nil, err + } + } + if sc.DisableSessionStart { + if err := h.DisableSessionStart(repo); err != nil { + return nil, err + } } for _, args := range [][]string{ @@ -121,6 +139,17 @@ func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, see return w, nil } +// TraceInit runs `chainloop trace init --local` for the harness in the +// repository. +func (w *Workspace) TraceInit(ctx context.Context, h Harness) (string, error) { + return w.run(ctx, w.Dir, w.Chainloop, "trace", "init", "--local", h.TraceFlag()) +} + +// TraceUninstall runs `chainloop trace uninstall` in the repository. +func (w *Workspace) TraceUninstall(ctx context.Context) (string, error) { + return w.run(ctx, w.Dir, w.Chainloop, "trace", "uninstall", "--yes") +} + // Git runs a git command in the repository and returns its output. func (w *Workspace) Git(ctx context.Context, args ...string) (string, error) { return w.run(ctx, w.Dir, "git", args...)