feat(trace): e2e checks for the remaining Claude Code rows

app/cli/internal/trace/README.md

@@ -14,13 +14,14 @@ How Trace is added to a repository and to an agent.
 
 | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 |
 |---------|---------------|-------------|--------|--------------|------------|
-| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Unknown | Unknown | Unknown | Unknown |
-| Trace run wrapper | `chainloop trace run -- <agent command>` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Unknown | Unknown | Unknown | Unknown |
+| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Unknown | Unknown | Unknown |
+| Trace run wrapper | `chainloop trace run -- <agent command>` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Unknown | Unknown | Unknown |
 | More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown |
 
 Why not Yes:
 
-- **All rows, all agents:** not verified yet. The end-to-end suite finds the values (spec issue-3635).
+- **Hook installation and deletion, and Trace run wrapper, Cursor and OpenCode:** not verified yet. The end-to-end suite finds the values (spec issue-3635).
+- **More than one agent in one repository, all agents:** not verified yet. The end-to-end suite needs a second agent to test it (spec issue-3635).
 
 ## Session collection
 
@@ -32,7 +33,7 @@ What the session evidence holds about the conversation.
 | Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes |
 | Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes |
 | Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No |
-| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Unknown | Unknown | Unknown | Unknown |
+| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Unknown |
 
 Why not Yes:
 
@@ -41,7 +42,7 @@ Why not Yes:
 - **Model detection, Cursor:** the primary model comes from the session-start hook. The transcript does not tell the models, so the evidence does not list the other models.
 - **Subagents, Cursor:** the provider does not read subagent conversations.
 - **Subagents, OpenCode:** a subagent runs in a child session with its own export. It is not part of the evidence of its parent session.
-- **Missed session start, all agents:** not verified yet. The end-to-end suite finds the values (spec issue-3635).
+- **Missed session start, Cursor and OpenCode:** not verified yet. The end-to-end suite finds the values (spec issue-3635).
 
 ## Code attribution
 

app/cli/internal/trace/e2e/checks.go

@@ -39,8 +39,11 @@ var errNotCovered = errors.New("not covered")
 type Session struct {
 	Scenario  string
 	Model     string
+	Harness   Harness
 	Workspace *Workspace
-	Output    *RunOutput
+	// Checkout is the checkout where the agent committed and pushed.
+	Checkout string
+	Output   *RunOutput
 	// Stored is the session that the push stored in local storage mode.
 	Stored *StoredSession
 	// HeadMessage is the message of the commit that the agent made.
@@ -59,6 +62,10 @@ type Feature struct {
 	// ControlPlane is true for a feature that needs a control plane. The suite
 	// runs in local storage mode only, so its cells are "not covered" (D-007).
 	ControlPlane bool
+	// NotCovered is the reason why the suite cannot see the feature at all,
+	// for example because neither the user nor the evidence can observe it.
+	// Its cells are "not covered".
+	NotCovered string
 	// ModelDriven is true for a feature that depends on the model following an
 	// instruction. Its cells run more than once (R-006).
 	ModelDriven bool
@@ -69,7 +76,18 @@ type Feature struct {
 // features are the test definitions of the feature rows. A row of the table
 // with no entry here is reported as "no check".
 var features = map[string]Feature{
+	// Setup
+	"Hook installation and deletion": {Checks: []Check{{scenarioSetup, checkHookInstallation}}},
+	"Trace run wrapper": {Checks: []Check{
+		{scenarioTraceRun, checkHookConfigRestored},
+		{scenarioTraceRun, checkGenericSessionStored},
+	}},
+
 	// Session collection
+	"Missed session start": {Checks: []Check{
+		{scenarioNoStart, checkNoSessionStartHook},
+		{scenarioNoStart, checkCommitTrailer},
+	}},
 	"Transcript":           {Checks: []Check{{scenarioEditAndCommit, checkTranscript}}},
 	"Token usage and cost": {Checks: []Check{{scenarioEditAndCommit, checkTokenUsage}}},
 	"Model detection":      {Checks: []Check{{scenarioEditAndCommit, checkModel}}},
@@ -78,6 +96,10 @@ var features = map[string]Feature{
 	// Code attribution
 	"Line attribution":          {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("greeting.txt")}}},
 	"Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}},
+	"Partial commits":           {Checks: []Check{{scenarioPartial, checkBothCommitsCredited}}},
+	"Generated line counts": {
+		NotCovered: "neither the user nor the evidence can see the generated line counts yet",
+	},
 
 	// User experience
 	"Welcome message":         {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}},
@@ -93,10 +115,12 @@ var features = map[string]Feature{
 	"Spec source role, title, and description": {
 		ModelDriven: true, Checks: []Check{{scenarioTicket, checkTicketHeader}},
 	},
+	"Pasted image pointers": {Checks: []Check{{scenarioPastedImage, checkPastedImage}}},
 
 	// Security
 	"Secret redaction":           {Checks: []Check{{scenarioEditAndCommit, checkSecretRedacted}}},
 	"Allow markers in redaction": {Checks: []Check{{scenarioEditAndCommit, checkAllowedSecretKept}}},
+	"Media skipped by redaction": {Checks: []Check{{scenarioMedia, checkMediaKept}}},
 
 	// Git integration
 	"Commit trailer":        {Checks: []Check{{scenarioEditAndCommit, checkCommitTrailer}}},

app/cli/internal/trace/e2e/checks_more.go

@@ -0,0 +1,185 @@
+//
+// Copyright 2026 The Chainloop Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"image/png"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer"
+	"github.com/chainloop-dev/chainloop/app/cli/internal/trace/state"
+	"github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession"
+)
+
+// commandTimeout limits a command that a check runs in the workspace.
+const commandTimeout = 2 * time.Minute
+
+// readHookConfig reads the hook configuration file of the harness.
+func readHookConfig(s *Session) ([]byte, error) {
+	return os.ReadFile(filepath.Join(s.Workspace.Dir, s.Harness.HookConfig().Path))
+}
+
+// checkHookInstallation checks that `trace init` installs the Chainloop
+// hooks next to a hook of the user, that a second init changes nothing, and
+// that `trace uninstall` deletes only the Chainloop hooks.
+func checkHookInstallation(s *Session) error {
+	hc := s.Harness.HookConfig()
+	ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
+	defer cancel()
+
+	installed, err := readHookConfig(s)
+	if err != nil {
+		return err
+	}
+	if !bytes.Contains(installed, []byte(hc.ChainloopMarker)) {
+		return fmt.Errorf("trace init did not install the Chainloop hooks in %s", hc.Path)
+	}
+	if !bytes.Contains(installed, []byte(hc.UserMarker)) {
+		return fmt.Errorf("trace init deleted the hook of the user from %s", hc.Path)
+	}
+
+	if _, err := s.Workspace.TraceInit(ctx, s.Harness); err != nil {
+		return err
+	}
+	again, err := readHookConfig(s)
+	if err != nil {
+		return err
+	}
+	if !bytes.Equal(installed, again) {
+		return fmt.Errorf("a second trace init changed %s", hc.Path)
+	}
+
+	if _, err := s.Workspace.TraceUninstall(ctx); err != nil {
+		return err
+	}
+	after, err := readHookConfig(s)
+	if err != nil {
+		return err
+	}
+	if bytes.Contains(after, []byte(hc.ChainloopMarker)) {
+		return fmt.Errorf("trace uninstall left Chainloop hooks in %s", hc.Path)
+	}
+	if !bytes.Contains(after, []byte(hc.UserMarker)) {
+		return fmt.Errorf("trace uninstall deleted the hook of the user from %s", hc.Path)
+	}
+	return nil
+}
+
+// checkHookConfigRestored checks that the hook configuration is the same as
+// before the wrapped session.
+func checkHookConfigRestored(s *Session) error {
+	got, err := readHookConfig(s)
+	if err != nil {
+		return err
+	}
+	if string(got) != s.Harness.HookConfig().UserConfig {
+		return fmt.Errorf("the hook configuration changed after trace run: %s", got)
+	}
+	return nil
+}
+
+// checkGenericSessionStored checks that `trace run --local` stored the
+// wrapped session as a generic session.
+func checkGenericSessionStored(s *Session) error {
+	if mode := evidence(s).Session.Mode; mode != aicodingsession.ModeGeneric {
+		return fmt.Errorf("the stored session has mode %q, want %q", mode, aicodingsession.ModeGeneric)
+	}
+	return nil
+}
+
+// checkNoSessionStartHook makes sure that the session-start hook did not
+// fire, so that the scenario tests a missed session start.
+func checkNoSessionStartHook(s *Session) error {
+	if _, err := hookMessage(s, "SessionStart"); err == nil {
+		return errors.New("the session-start hook fired, so the scenario does not test a missed start")
+	}
+	return nil
+}
+
+// checkBothCommitsCredited checks that each of the two commits of a file
+// credits the session in its trailer.
+func checkBothCommitsCredited(s *Session) error {
+	ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
+	defer cancel()
+
+	out, err := s.Workspace.run(ctx, s.Checkout, "git", "log", "-2", "--format=%B%x00")
+	if err != nil {
+		return err
+	}
+	sessionID := evidence(s).Session.ID
+	for i, msg := range strings.Split(strings.TrimRight(strings.TrimSpace(out), "\x00"), "\x00") {
+		if !slices.Contains(state.ParseSessionIDsFromTrailer(msg), sessionID) {
+			return fmt.Errorf("commit %d of 2 does not credit the session: %q", 2-i, strings.TrimSpace(msg))
+		}
+	}
+	return nil
+}
+
+// checkPastedImage checks that the pasted image is stored once as an image
+// spec entry, and that the transcript holds a pointer in place of the image.
+func checkPastedImage(s *Session) error {
+	for _, e := range evidence(s).Spec {
+		if e.Kind != aicodingsession.SpecKindImage || e.Title != "Pasted image 1" {
+			continue
+		}
+		content, err := s.Stored.Material(e.Digest)
+		if err != nil {
+			return fmt.Errorf("read the image material: %w", err)
+		}
+		if _, err := png.Decode(bytes.NewReader(content)); err != nil {
+			return fmt.Errorf("the image material is not a valid PNG: %w", err)
+		}
+
+		pasted := base64.StdEncoding.EncodeToString(pastedPNG())
+		hasPointer := false
+		for _, msg := range evidence(s).RawSession["main"] {
+			if bytes.Contains(msg, []byte(pasted)) {
+				return errors.New("the transcript still holds the pasted image inline")
+			}
+			if bytes.Contains(msg, []byte(pointer.Marker)) && bytes.Contains(msg, []byte(e.Digest)) {
+				hasPointer = true
+			}
+		}
+		if !hasPointer {
+			return errors.New("the transcript has no pointer to the pasted image")
+		}
+		return nil
+	}
+	return errors.New(`the evidence has no image spec entry "Pasted image 1"`)
+}
+
+// checkMediaKept checks that the redaction did not change the image that the
+// agent read. Its base64 holds a fake secret, which a redaction that scans
+// media would replace.
+func checkMediaKept(s *Session) error {
+	found, err := storedContains(s, "/"+mediaSecret+"/")
+	if err != nil {
+		return err
+	}
+	if !found {
+		return errors.New("the stored transcript does not hold the image that the agent read, unchanged")
+	}
+	return nil
+}