diff --git a/README.md b/README.md index ea2b9d7..cfbb2c8 100644 --- a/README.md +++ b/README.md @@ -179,9 +179,24 @@ Each case lists how to confirm it, its causes, fix steps tagged `[read]` / **`/incident [what you see]`** works the top case to resolution: confirm → fix the root cause → wait (`wait.until`, without spending model calls) → verify → report. When the agent finds a cause no case covered, it offers to record it with `kb.add`. + +cometcli remembers. Every triage is saved to the node's history, so it can tell you what +changed since the last check and answer "since when?" (`node history --since 7d`). +Every finished incident is saved as a postmortem with its root cause, actions, +transaction hashes and a timeline from the audit log. Triage points out when the same +problem comes back. Add your own cases in `~/.cometcli/kb/` — format in [docs/EXTENDING.md](docs/EXTENDING.md#knowledge-base-cases). +Across nodes, `cometcli fleet triage` compares them all at once. It points out a node +behind the others, a chain-wide halt, version mismatches, two nodes running one +consensus key, and validators cut off from their sentries. `cometcli watch` does this on +an interval for you. It works new incidents read-only and reports them to Slack, +Discord or Telegram, or with `--mode fix` sends each approval to Telegram with +Approve/Deny buttons. To check how well all of this works, `cometcli drill` breaks a +throwaway local network on purpose and scores whether the agent finds and fixes each +fault. + Jail recovery is careful about fees: `val.unjail` refuses while the jail period runs, the node isn't synced, the signer's saved state is ahead of the chain, the node signs with a different consensus key than the one registered, self-delegation is below the @@ -222,22 +237,25 @@ Tools are grouped by domain. Required args also bind positionally — | Domain | Highlights | |---|---| -| `node` | `triage`, `status`, `peers`, `health`, `consensus`, `logs`, `config --action lint`, `service status\|restart`, `version-check` | +| `node` | `triage`, `history`, `status`, `peers`, `health`, `consensus`, `logs`, `config --action lint`, `service status\|restart`, `version-check` | | `val` | `status`, `signing`, `jail-check`, `consensus`, `rewards`, `votes` · on-chain: `unjail`, `withdraw`, `vote`, `edit`, `create` | | `chain` | `validators` (`--status jailed`), `params`, `gov`, `upgrade-plan`, `pool`, `balance` | | `gov` | `propose` (text / upgrade / messages), `deposit` | +| `incident` | `list`, `show`, `record` — postmortems of past incidents | | `kb` | `search`, `show`, `add` — the knowledge base of failure cases | | `wait` | `until` — synced, height, unjailable, signing, in-consensus, tx-committed, proposal-status, or any triage signal | | `evm` | `chainid`, `parity` (comet↔JSON-RPC drift), `gasprice`, `txpool` | | `tx` | `send`, `delegate`, `undelegate`, `redelegate`, `get` — every tx tool takes `--gas-price`, `--gas-limit`, `--fee-denom`, `--seq` | | `keys` | `add` (`--recover`/`--privkey-hex`), `list`, `show`, `rm`, `convert` (bech32↔0x) | | `sec` | `exposure` (listening ports), `perms` (key-file permissions), `doublesign` (priv_validator_state check) | -| `mon` | `snapshot`, `watch` (live TUI), `alerts` (rules → stdout/Slack/Discord/Telegram) | +| `mon` | `snapshot`, `watch` (live TUI), `record` (signal history), `alerts` (rules → stdout/Slack/Discord/Telegram) | | `upgrade` | `check`, `prepare` (download, verify, stage for cosmovisor), `watch` | | `snap` | `list`, `prune`, `statesync` | | `runbook` | `list`, `show`, `run` — builtins + your own YAML | | `net` | `add-peer`, `rm-peer` | -| `fleet` | `status`, `exec` (a read-only tool on every profile), `shell` (one command on every host, one approval) | +| `fleet` | `triage` (every node, compared), `status`, `exec` (a read-only tool on every profile), `shell` (one command on every host, one approval) | +| `watch` | the autonomous watcher: triage on an interval, work new incidents, remote approvals | +| `drill` | `testnet up\|down`, `list`, `run` — scored incident drills on a throwaway network | Global flags: `--profile`, `--json`, `-y/--yes` (auto-approve observe/diagnose prompts — never on-chain). @@ -309,6 +327,8 @@ credentials # API keys saved by `config set-key` (0600) settings.json # permission rules, hooks, MCP servers (also per project: .cometcli/) COMET.md # memory the agent always reads (also per project) kb/ # your own knowledge-base cases +history/ # triage signal history per profile (30 days) +incidents/ # incident records and postmortems per profile keys/ # ops keyring (file backend) sessions/ # saved conversations (cometcli -c / -r) runbooks/ # your custom YAML playbooks diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index d64fb3e..21442ca 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -99,7 +99,8 @@ down is reported, never fatal. Signals are flat names (`val.jailed`, `host.disk_used_pct`, `logs.apphash`, `node.key_mismatch`…). Each **case** says which signals point to it, how to confirm it, causes, fix steps tagged -`[read]` / `[change]` / `[tx]`, how to verify, and what never to do. In node mode, +`[read]` / `[change]` / `[tx]`, how to verify, and what never to do. In any session +(name the node — `/incident val01 is down` — and cometcli switches to it), **`/incident [what you see]`** runs the method: triage → confirm the top case → fix the root cause first (approvals as usual) → wait with `wait.until` → verify → re-triage → report. When the agent finds a cause no case covered, it offers to record one with @@ -116,6 +117,31 @@ format in [EXTENDING.md](EXTENDING.md#knowledge-base-cases). cometcli wait until --condition signal --value "host.disk_used_pct < 85 && proc.running == true" ``` +### History and incident records + +Every `node.triage` run is saved to the node's signal history +(`~/.cometcli/history//`, kept 30 days), and triage starts by saying what +changed since the previous run (`node.peers 8→1, val.jailed false→true`). For trends: + +```bash +cometcli node history # key health signals, last 24h +cometcli node history --signals val.,node.peers --since 7d +cometcli mon record --interval 5m # sample regularly, not only when someone triages +``` + +Each answer gives first → last, min/max, a trend line and when the value changed. + +When an incident is finished, the agent saves it with `incident.record`: title, root +cause, matching case, evidence, actions and outcome. The timeline (prompts, tool calls, +approvals) and transaction hashes are filled in from the audit log. Records live in +`~/.cometcli/incidents//` as JSON plus a Markdown postmortem. Triage +mentions a node's past incidents and flags a case that keeps coming back. + +```bash +cometcli incident list # last 30 days (--since 90d) +cometcli incident show --id 20261007-0533-val-jailed-downtime +``` + ### Incident tools Tools built for running an incident to resolution — usable directly or by the agent: @@ -287,6 +313,8 @@ cometcli evm txpool # pending/queued EVM tx counts ## Fleet (all profiles at once) ```bash +cometcli fleet triage # every node triaged at once, then compared +cometcli fleet triage --chain mychain-1 # one chain only cometcli fleet status # health matrix: height, peers, signing, disk cometcli fleet exec --tool node.status # run a read-only tool everywhere cometcli fleet exec --tool node.logs --args '{"lines":20}' @@ -294,6 +322,88 @@ cometcli fleet exec --tool val.signing --profiles val01,val02 cometcli fleet shell "uptime" # shell on every host (one approval) ``` +`fleet triage` gives one row per node (height, peers, signing, jailed, top case), then +what only a comparison shows: + +- a node behind the others on its chain +- a chain-wide halt: every node stalled, so the problem is the chain, not one node +- version mismatches between nodes +- two nodes running the same consensus key (double-sign risk) +- validators sharing a host +- a validator whose sentries are down + +Tell cometcli which sentries a validator peers through so it can check them: +`cometcli profile add val01 --sentries sentry1,sentry2`. In a session, questions about +several nodes ("how is my network?") start with `fleet.triage`, in general mode too. + +## Watching on your behalf + +```bash +cometcli watch # every node, every 5m, diagnose mode +cometcli watch --mode fix --interval 2m # may act — approvals go to Telegram +cometcli watch --once --mode notify # one sweep, report only (cron-friendly) +``` + +Each sweep is a fleet triage. A problem that newly appears (a critical or high case on +a node, or a chain halt, double-sign risk or cut-off validator) becomes an incident: + +| mode | what happens | +|---|---| +| `notify` | the finding is reported | +| `diagnose` (default) | the agent works the incident read-only and reports its findings | +| `fix` | the agent may act; **each approval is sent to Telegram with Approve / Deny buttons** | + +The watcher reports findings, agent reports and recoveries. It doesn't work the same +problem again within the cooldown (default 1h), and remembers what it saw across +restarts. + +```yaml +# ~/.cometcli/config.yaml +watch: + interval: 5m + mode: diagnose # notify | diagnose | fix + profiles: [val01, val02] # default: all + cooldown: 1h + approval_timeout: 15m # an unanswered approval is a "no" + allow_tx: false # transactions are refused in watch mode unless true + telegram_users: [123456] # only these Telegram users' presses count + alerts: + slack_webhook: https://hooks.slack.com/… # reports only + discord_webhook: https://discord.com/api/webhooks/… + telegram_chat_id: "-100123456" # reports and approvals +``` + +Save the Telegram bot token with `cometcli config set-key TELEGRAM_BOT_TOKEN`, or put it +in `watch.alerts.telegram_token`. Only presses from that chat, and from +`telegram_users` if set, count. Slack and Discord get reports but can't answer +approvals. + +## Incident drills + +Check how well cometcli handles real faults, on a throwaway network: + +```bash +cometcli drill testnet up --image # 4 local validators in docker, fast slashing +cometcli drill list # the scenarios +cometcli drill run # all of them (--scenario oom,partition to pick) +cometcli drill testnet down # remove containers, data and drill profiles +``` + +Each scenario breaks a drill node for real: + +- stopped until it's jailed for downtime +- `config.toml` broken (crash loop) +- `halt-height` set +- container memory limit too low (OOM) +- network detached + +For each one, the drill checks whether triage named the right root cause, then lets the +agent work it with `/incident`, verifies the fix on the node itself, and puts the node +back to health. The scoreboard shows triage x/y and fixed x/y, with steps, tokens and +time per scenario. Results are saved in `~/.cometcli/drills/`, so model or prompt +changes can be compared run to run. Approvals are granted automatically, **but only on +drill profiles** (`metadata.drill: "true"`); anything else is refused. + ## Monitoring ```bash @@ -409,8 +519,8 @@ saved after each turn to `~/.cometcli/sessions/` (0600, already redacted). | `/memory`, `/remember [--user\|--node] ` | memory files loaded; add a line to COMET.md | | `/mcp`, `/agents` | connected MCP servers; subagents for the task tool | | `/` | commands from `.cometcli/commands/*.md` — see [EXTENDING.md](EXTENDING.md) | -| `/incident [what you see]` | node mode: triage → known case → fix root cause → wait → verify → report | -| `/recover-jail [notes]` | node mode: run the full jail-recovery procedure | +| `/incident [what you see]` | triage → known case → fix root cause → wait → verify → record → report; name the node to switch to it | +| `/recover-jail [notes]` | the full jail-recovery procedure (name the node, or the agent picks it) | | `/one [profile\|off]` | switch the session to node mode for a profile, or back to general (conversation kept) | ### General tools and permissions diff --git a/internal/agent/builtin_cmds.go b/internal/agent/builtin_cmds.go index db8ad2b..758039f 100644 --- a/internal/agent/builtin_cmds.go +++ b/internal/agent/builtin_cmds.go @@ -10,13 +10,14 @@ var builtinCommands = map[string]*settings.Command{ Description: "Work an incident to resolution: triage, match a known case, fix the root cause, wait, verify, report", Body: `Work this incident on the node to resolution. Keep the operator's checklist current with todo_write; don't stop between steps unless something needs their decision. -1. node.triage (since=2h if the problem started earlier). Note which sources were unavailable. +1. node.triage (since=2h if the problem started earlier). Note which sources were unavailable, what changed since the last check, and past incidents on this node — a repeat means the last fix missed something (incident.show it). For "since when?", use node.history. 2. Take the top matched case: kb.show it, run its confirm steps. If confirmation fails, move to the next case or kb.search the symptoms/log lines. No case at all → investigate from the signals and logs (node.logs, bash read-only). 3. If several cases match, fix the one that CAUSES the others first (disk full → crash → behind → jailed: disk first). 4. Apply the fix steps by CALLING the tools. [change] and [tx] steps are approved at the tool's own prompt: state in one line what you're doing and why, then make the call in the same step — never stop to ask for approval in text. Never do anything a case lists under NEVER. 5. Wait for the effect with wait.until (synced, signing, in-consensus, or condition=signal value=""). If progress stalls, re-triage instead of waiting longer. 6. Verify with the case's verify steps, then node.triage again: the case must no longer match. -7. Report: root cause with evidence, actions taken (tx hashes), final state. If you found a cause no case described, propose a kb.add case (with a test fixture) for the operator to approve. +7. Record it with incident.record (title, root cause, case id, evidence, actions, outcome) — the timeline and tx hashes are filled in for you. +8. Report: root cause with evidence, actions taken (tx hashes), final state. If you found a cause no case described, propose a kb.add case (with a test fixture) for the operator to approve. What the operator sees: $ARGUMENTS`, }, diff --git a/internal/agent/commands.go b/internal/agent/commands.go index 37fa266..103d98a 100644 --- a/internal/agent/commands.go +++ b/internal/agent/commands.go @@ -7,6 +7,7 @@ import ( "path/filepath" "sort" "strings" + "unicode" "github.com/abhijitkrm/cometcli/internal/config" "github.com/abhijitkrm/cometcli/internal/runbook" @@ -232,7 +233,7 @@ func RunCommand(a *Agent, c *toolkit.Context, reg *toolkit.Registry, line string return CmdResult{}, err } if len(f) == 1 { - txt := "general mode — shell and files on this machine; /one for node work" + txt := "general mode — shell and files on this machine; ask about a node by name and the agent switches to it (/one pins one)" if a.Node() { txt = "node mode: " + a.Ctx.Profile.Name + " — /one off for general mode" } @@ -468,14 +469,53 @@ func RunCommand(a *Agent, c *toolkit.Context, reg *toolkit.Registry, line string if err := needAgent(); err != nil { return CmdResult{}, err } + prompt, text := cmd.Expand(args, nil, nil), "/"+name if builtinNeedsNode[name] && !a.Node() { - return CmdResult{}, fmt.Errorf("/%s needs node mode — /one first", name) + p, err := a.nodeFor(args) + switch { + case err != nil: + return CmdResult{}, err + case p != nil: + // the operator named the node: work on it + a.SwitchScope(p) + text += " — working on " + p.Name + default: + // let the model pick the profile itself + prompt = "This is about one of the operator's nodes: call use_node with the right profile first.\n\n" + prompt + } } - return CmdResult{Text: "/" + name, Prompt: cmd.Expand(args, nil, nil)}, nil + return CmdResult{Text: text, Prompt: prompt}, nil } return CmdResult{}, ErrUnknownCommand } +// nodeFor finds the profile a node-only command is about: one named in +// its arguments, or nil when none is (the model then picks with use_node). +// It errors only when there are no profiles at all. +func (a *Agent) nodeFor(args string) (*config.Profile, error) { + if a.Ctx == nil || a.Ctx.Cfg == nil || len(a.Ctx.Cfg.Profiles) == 0 { + return nil, fmt.Errorf("no node profiles yet — add one with `cometcli init` or `cometcli profile add`") + } + words := map[string]bool{} + for _, w := range strings.FieldsFunc(strings.ToLower(args), func(r rune) bool { + return r != '-' && r != '_' && r != '.' && !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) { + words[strings.Trim(w, ".")] = true + } + var names []string + for n := range a.Ctx.Cfg.Profiles { + if words[strings.ToLower(n)] { + names = append(names, n) + } + } + if len(names) != 1 { // none, or ambiguous: the model decides + return nil, nil + } + p := a.Ctx.Cfg.Profiles[names[0]] + p.Name = names[0] + return p, nil +} + // memoryTarget picks the file /remember writes: --user, --node, or the // project's COMET.md (the user file without a project). func (a *Agent) memoryTarget(rest *string) (string, error) { diff --git a/internal/agent/prompt.go b/internal/agent/prompt.go index d5959cd..0ee7b4d 100644 --- a/internal/agent/prompt.go +++ b/internal/agent/prompt.go @@ -42,8 +42,11 @@ Incidents and "is something wrong?" questions — work the method: height, or signal="") instead of guessing; if progress stalls, re-triage. 5. Verify with the case's verify steps, then re-run node.triage. -6. Report cause + evidence, actions (tx hashes), final state. If no case - fit and you found the cause, offer to record it with kb.add. +6. Save it with incident.record, then report cause + evidence, actions + (tx hashes), final state. If no case fit and you found the cause, + offer to record it with kb.add. +Triage shows what changed since the last check and this node's past +incidents; node.history answers "since when?" and "is it getting worse?". `) b.WriteString(strings.Replace(generalToolsText, "General tools:", "General tools (they run on the node's host — local or over SSH):", 1)) diff --git a/internal/agent/scope.go b/internal/agent/scope.go index 26ce244..682347a 100644 --- a/internal/agent/scope.go +++ b/internal/agent/scope.go @@ -47,7 +47,7 @@ ENVIRONMENT: %s/%s, host %s, working directory %s, date %s if ap, err := c.Cfg.ActiveProfile(""); err == nil && ap != nil { active = " Active profile: " + ap.Name + "." } - fmt.Fprintf(&b, "NODE PROFILES: %s.%s For any question about a node, validator or chain (jailing, sync, peers, upgrades, governance, txs), call use_node with the right profile yourself and continue — never ask the operator to switch. Any profile on a chain answers chain-wide questions; pick the active one when nothing points elsewhere.\n", strings.Join(names, "; "), active) + fmt.Fprintf(&b, "NODE PROFILES: %s.%s For any question about a node, validator or chain (jailing, sync, peers, upgrades, governance, txs), call use_node with the right profile yourself and continue — never ask the operator to switch. Any profile on a chain answers chain-wide questions; pick the active one when nothing points elsewhere. For questions about several nodes or the whole network (\"how is my fleet?\"), start with fleet.triage.\n", strings.Join(names, "; "), active) } return b.String() } diff --git a/internal/agent/toolsearch.go b/internal/agent/toolsearch.go index d0a112e..6950296 100644 --- a/internal/agent/toolsearch.go +++ b/internal/agent/toolsearch.go @@ -22,6 +22,7 @@ var coreTools = map[string]bool{ "val.status": true, "val.signing": true, "val.jail-check": true, "val.consensus": true, "wait.until": true, "chain.validators": true, "node.triage": true, "kb.show": true, "kb.search": true, + "node.history": true, "incident.record": true, "fleet.triage": true, } var toolSearchDef = ToolDef{ @@ -66,7 +67,7 @@ func (a *Agent) advertised(t toolkit.Tool) bool { return true } if !a.Node() { - return false // node tools need a node: /one + return toolkit.IsFleetWide(t) // other node tools need a node (use_node) } if !a.deferred() || coreTools[t.Name()] { return true diff --git a/internal/agent/usenode_test.go b/internal/agent/usenode_test.go index a058259..43a444a 100644 --- a/internal/agent/usenode_test.go +++ b/internal/agent/usenode_test.go @@ -66,3 +66,36 @@ func TestModelSwitchesToNodeItself(t *testing.T) { t.Fatal("unknown profile accepted") } } + +func TestNodeOnlyCommandsInGeneralMode(t *testing.T) { + a := newTestAgent(t, &mockProvider{}) + a.Ctx.Profile = nil + a.Ctx.Cfg = &config.Config{Profiles: map[string]*config.Profile{ + "val01": {ChainID: "mychain-1", Role: "validator"}, + "val02": {ChainID: "mychain-1", Role: "validator"}, + }} + // names a node: switch to it + res, err := RunCommand(a, a.Ctx, a.Reg, "/incident val01 is down") + if err != nil || !a.Node() || a.Ctx.Profile.Name != "val01" || !strings.Contains(res.Text, "working on val01") { + t.Fatalf("named node: %+v %v node=%v", res, err, a.Node()) + } + if !strings.Contains(res.Prompt, "node.triage") || !strings.Contains(res.Prompt, "val01 is down") { + t.Fatalf("procedure prompt: %q", res.Prompt) + } + // no node named: the model picks with use_node + a.SwitchScope(nil) + res, err = RunCommand(a, a.Ctx, a.Reg, "/incident blocks stopped") + if err != nil || a.Node() || !strings.Contains(res.Prompt, "call use_node") { + t.Fatalf("unnamed node: %+v %v", res, err) + } + // both named: ambiguous, the model decides + res, _ = RunCommand(a, a.Ctx, a.Reg, "/incident val01 and val02 disagree") + if a.Node() || !strings.Contains(res.Prompt, "call use_node") { + t.Fatal("ambiguous names switched anyway") + } + // no profiles at all: explain how to add one + a.Ctx.Cfg = &config.Config{} + if _, err := RunCommand(a, a.Ctx, a.Reg, "/incident val01 is down"); err == nil || !strings.Contains(err.Error(), "cometcli init") { + t.Fatalf("no profiles: %v", err) + } +} diff --git a/internal/cli/agent.go b/internal/cli/agent.go index 4a09cda..3d5f239 100644 --- a/internal/cli/agent.go +++ b/internal/cli/agent.go @@ -13,7 +13,7 @@ func agentFlags(cmd *cobra.Command) { cmd.Flags().StringSlice("autopilot", nil, "tiers that skip the confirm prompt (only local-change; on-chain never)") cmd.Flags().Bool("no-stream", false, "disable token streaming") cmd.Flags().Int("budget", 0, "max tool calls per turn (0 = unlimited)") - cmd.Flags().Int("max-iter", 0, "max model iterations per turn (default 50)") + cmd.Flags().Int("max-iter", 0, "max model steps per request (default 200, or agent.max_turns)") sessionFlags(cmd) } diff --git a/internal/cli/drill.go b/internal/cli/drill.go new file mode 100644 index 0000000..ab8ed56 --- /dev/null +++ b/internal/cli/drill.go @@ -0,0 +1,133 @@ +package cli + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/abhijitkrm/cometcli/internal/agent" + "github.com/abhijitkrm/cometcli/internal/audit" + "github.com/abhijitkrm/cometcli/internal/config" + "github.com/abhijitkrm/cometcli/internal/drill" + "github.com/abhijitkrm/cometcli/internal/toolkit" +) + +func drillCmd(reg *toolkit.Registry) *cobra.Command { + cmd := &cobra.Command{ + Use: "drill", + Short: "Incident drills: inject real faults on a throwaway docker network, score how the agent handles them", + } + logf := func(cmd *cobra.Command) func(string) { + return func(s string) { fmt.Fprintf(cmd.OutOrStdout(), "%s %s\n", time.Now().Format("15:04:05"), s) } + } + + var image string + var validators int + tn := &cobra.Command{Use: "testnet", Short: "Create or remove the throwaway drill network"} + up := &cobra.Command{ + Use: "up", + Short: "Create a local validator network in docker (fast slashing: 20-block window, 30s jail) with drill profiles", + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + n := &drill.Net{Image: image, Validators: validators} + if err := n.Up(cmd.Context(), cfg, logf(cmd)); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "drill network up: profiles drill0..drill%d on chain %s — run `cometcli drill run`; remove with `cometcli drill testnet down`\n", n.Validators-1, n.ChainID) + return nil + }, + } + up.Flags().StringVar(&image, "image", "", "evmd docker image to run (required)") + up.Flags().IntVar(&validators, "validators", 4, "validators (4 keeps the chain live with one node down)") + down := &cobra.Command{ + Use: "down", + Short: "Remove the drill network, its data and its profiles", + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + if err := (&drill.Net{}).Down(cmd.Context(), cfg); err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), "drill network removed") + return nil + }, + } + tn.AddCommand(up, down) + + list := &cobra.Command{ + Use: "list", + Short: "List the drill scenarios", + Run: func(cmd *cobra.Command, _ []string) { + for _, s := range drill.Scenarios { + fmt.Fprintf(cmd.OutOrStdout(), "%-14s %s (expects %s)\n", s.Name, s.Desc, strings.Join(s.Expect, " | ")) + } + }, + } + + var only []string + var node int + var model string + run := &cobra.Command{ + Use: "run", + Short: "Run drills: inject each fault, let the agent work it with /incident, verify on the node, score", + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + n := &drill.Net{} + n.Defaults() + if node < 0 { + node = 3 + } + p := cfg.Profiles[drill.Profile(node)] + if !drill.IsDrill(p) { + return fmt.Errorf("no drill profile %s — `cometcli drill testnet up --image ` first", drill.Profile(node)) + } + p.Name = drill.Profile(node) + factory := func(ctx context.Context, approve toolkit.Approver) (*agent.Agent, error) { + lg, _ := audit.Open("drill") + c := &toolkit.Context{Context: ctx, Cfg: cfg, Audit: lg, Approver: approve} + a, err := agent.New(c, reg) + if err != nil { + return nil, err + } + if model != "" { + a.Model = model + } + return a, nil + } + var results []drill.Result + for _, s := range drill.Scenarios { + if len(only) > 0 && !contains(only, s.Name) { + continue + } + fmt.Fprintf(cmd.OutOrStdout(), "\n== %s on %s\n", s.Name, p.Name) + e := &drill.Env{Net: n, Cfg: cfg, Reg: reg, Node: node, Profile: p} + r := drill.RunScenario(cmd.Context(), e, s, factory, logf(cmd)) + results = append(results, r) + fmt.Fprintf(cmd.OutOrStdout(), " triage %s (%v) · fixed %v · %d steps · %d tokens · %s\n", r.TriageCase, r.TriageOK, r.Fixed, r.Steps, r.Tokens, r.Duration.Round(time.Second)) + } + if len(results) == 0 { + return fmt.Errorf("no scenarios matched — cometcli drill list") + } + path, _ := drill.Save(results) + fmt.Fprintf(cmd.OutOrStdout(), "\n%s\nresults: %s\n", drill.Scoreboard(results), path) + return nil + }, + } + run.Flags().StringSliceVar(&only, "scenario", nil, "scenarios to run (default all)") + run.Flags().IntVar(&node, "node", -1, "validator index to break (default the last)") + run.Flags().StringVar(&model, "model", "", "model for the agent (default: your agent settings)") + + cmd.AddCommand(tn, list, run) + return cmd +} diff --git a/internal/cli/profile.go b/internal/cli/profile.go index 36310c9..c2e6355 100644 --- a/internal/cli/profile.go +++ b/internal/cli/profile.go @@ -130,6 +130,9 @@ func profileAddCmd() *cobra.Command { if fl.Changed("ssh-host") { np.Transport.Host = p.Transport.Host } + if fl.Changed("sentries") { + np.Sentries = p.Sentries + } if fl.Changed("ssh-user") { np.Transport.User = p.Transport.User } @@ -205,6 +208,7 @@ func profileAddCmd() *cobra.Command { f.StringVar(&p.Transport.KeyFile, "ssh-key", "", "ssh private key path") f.StringVar(&p.Service.Type, "service", "", "systemd | docker | launchd | none") f.StringVar(&p.Service.Unit, "unit", "", "service unit/container name") + f.StringSliceVar(&p.Sentries, "sentries", nil, "profiles of the sentry nodes this validator peers through (comma-separated)") f.StringVar(&signerKey, "signer", "", "ops key name in keyring") f.StringVar(&p.Signer.Backend, "signer-backend", "", "os | file | test") f.StringVar(&feeDenom, "fee-denom", "", "fee denom (e.g. atest)") diff --git a/internal/cli/root.go b/internal/cli/root.go index 4457758..919cd7d 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -76,7 +76,7 @@ status, cometcli doctor, cometcli tx unjail) and a tool the agent calls.`, for _, c := range toolGroupCommands(reg) { root.AddCommand(c) } - root.AddCommand(profileCmd(), auditCmd(), versionCmd(), initCmd(), sessionsCmd(), oneCmd(reg), configCmd(), trustCmd()) + root.AddCommand(profileCmd(), auditCmd(), versionCmd(), initCmd(), sessionsCmd(), oneCmd(reg), configCmd(), trustCmd(), watchCmd(reg), drillCmd(reg)) root.AddCommand(extra...) return root } diff --git a/internal/cli/watch.go b/internal/cli/watch.go new file mode 100644 index 0000000..443c078 --- /dev/null +++ b/internal/cli/watch.go @@ -0,0 +1,172 @@ +package cli + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/abhijitkrm/cometcli/internal/agent" + "github.com/abhijitkrm/cometcli/internal/audit" + "github.com/abhijitkrm/cometcli/internal/config" + "github.com/abhijitkrm/cometcli/internal/monitor" + "github.com/abhijitkrm/cometcli/internal/toolkit" + "github.com/abhijitkrm/cometcli/internal/tools/triage" + "github.com/abhijitkrm/cometcli/internal/watch" +) + +func watchCmd(reg *toolkit.Registry) *cobra.Command { + var interval, mode, cooldown string + var profiles []string + var once bool + cmd := &cobra.Command{ + Use: "watch", + Short: "Watch the fleet on your behalf: triage on an interval, work new incidents, report and ask for approvals remotely", + Long: `Sweeps every node (fleet triage) on an interval. A newly appearing critical or +high problem becomes an incident: + + notify report the finding + diagnose the agent investigates read-only and reports (default) + fix the agent may act; every approval is sent to Telegram with + Approve / Deny buttons (unanswered = denied) + +Findings, reports and recoveries go to the watch.alerts channels (Slack, +Discord, Telegram). Transactions are refused unless watch.allow_tx is set. +Configure under 'watch:' in ~/.cometcli/config.yaml; flags override it.`, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + wc := cfg.Watch + pick := func(flag, conf, def string) string { + if cmd.Flags().Changed(flag) { + return map[string]string{"interval": interval, "mode": mode, "cooldown": cooldown}[flag] + } + if conf != "" { + return conf + } + return def + } + iv, err := time.ParseDuration(pick("interval", wc.Interval, "5m")) + if err != nil || iv < 30*time.Second { + return fmt.Errorf("interval: want a duration ≥ 30s") + } + cd, err := time.ParseDuration(pick("cooldown", wc.Cooldown, "1h")) + if err != nil { + return fmt.Errorf("cooldown: %w", err) + } + m, err := watch.ParseMode(pick("mode", wc.Mode, "")) + if err != nil { + return err + } + names := wc.Profiles + if cmd.Flags().Changed("profiles") { + names = profiles + } + selected := map[string]*config.Profile{} + var list []*config.Profile + var all []string + for n := range cfg.Profiles { + all = append(all, n) + } + sort.Strings(all) + for _, n := range all { + if len(names) > 0 && !contains(names, n) { + continue + } + p := cfg.Profiles[n] + p.Name = n + selected[n] = p + list = append(list, p) + } + if len(list) == 0 { + return fmt.Errorf("no profiles to watch — cometcli init") + } + + alerts := wc.Alerts + if alerts.TelegramToken == "" { + alerts.TelegramToken = config.Credential("TELEGRAM_BOT_TOKEN") + } + var notify []watch.Notifier + var channels []string + for _, s := range monitor.Sinks(alerts) { + notify = append(notify, s) + channels = append(channels, s.Name()) + } + var approver toolkit.Approver + if m == watch.ModeFix { + if alerts.TelegramToken == "" || alerts.TelegramChatID == "" { + fmt.Fprintln(cmd.ErrOrStderr(), "fix mode needs Telegram for approvals (watch.alerts.telegram_token or the TELEGRAM_BOT_TOKEN credential, and telegram_chat_id) — running read-only") + } else { + at, _ := time.ParseDuration(wc.ApprovalTimeout) + users := map[int64]bool{} + for _, u := range wc.TelegramUsers { + users[u] = true + } + tg := &watch.Telegram{Token: alerts.TelegramToken, ChatID: alerts.TelegramChatID, Users: users, Timeout: at} + approver = watch.RemoteApprover(tg.Ask, wc.AllowTx) + } + } + statePath, _ := config.Path("watch-state.json") + glog, _ := audit.Open("watch") + w := &watch.Watcher{ + Mode: m, Cooldown: cd, Profiles: selected, Notify: notify, Approver: approver, + StatePath: statePath, Out: cmd.OutOrStdout(), + Collect: func(ctx context.Context) []triage.NodeReport { + return triage.CollectFleet(&toolkit.Context{Context: ctx, Cfg: cfg, Audit: glog}, list, iv) + }, + Run: func(ctx context.Context, p *config.Profile, desc string, readOnly bool, approve toolkit.Approver) (string, error) { + lg, _ := audit.Open(p.Name) + if lg != nil { + defer lg.Close() + } + c := &toolkit.Context{Context: ctx, Profile: p, Cfg: cfg, Audit: lg, Approver: approve} + a, err := agent.New(c, reg) + if err != nil { + return "", err + } + defer a.Close() + if readOnly { + a.Policy.Mode = agent.ModeReadOnly + } + res, err := agent.RunCommand(a, a.Ctx, reg, "/incident "+desc) + if err != nil { + return "", err + } + return a.Run(ctx, res.Prompt) + }, + } + if len(channels) == 0 { + channels = []string{"this terminal only"} + } + var watched []string + for _, p := range list { + watched = append(watched, p.Name) + } + fmt.Fprintf(cmd.OutOrStdout(), "watching %d nodes (%s) every %s · mode %s · cooldown %s · reports to %s\n", + len(list), strings.Join(watched, ", "), iv, m, cd, strings.Join(channels, ", ")) + w.Loop(cmd.Context(), iv, once) + return nil + }, + } + f := cmd.Flags() + f.StringVar(&interval, "interval", "", "time between sweeps (default 5m, or watch.interval)") + f.StringVar(&mode, "mode", "", "notify | diagnose | fix (default diagnose, or watch.mode)") + f.StringVar(&cooldown, "cooldown", "", "before the same issue is worked again (default 1h)") + f.StringSliceVar(&profiles, "profiles", nil, "profiles to watch (default all, or watch.profiles)") + f.BoolVar(&once, "once", false, "run one sweep and exit") + return cmd +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} diff --git a/internal/config/config.go b/internal/config/config.go index f2ce061..6c24f01 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -44,6 +44,29 @@ type Config struct { // (no node) and as the base a profile's agent section overrides. Agent AgentConf `yaml:"agent,omitempty"` Profiles map[string]*Profile `yaml:"profiles"` + // Watch configures `cometcli watch`, the autonomous fleet watcher. + Watch WatchConf `yaml:"watch,omitempty"` +} + +// WatchConf configures the autonomous watcher. +type WatchConf struct { + Interval string `yaml:"interval,omitempty"` // between sweeps (default 5m) + Mode string `yaml:"mode,omitempty"` // notify | diagnose (default) | fix + Profiles []string `yaml:"profiles,omitempty"` // default: all + Cooldown string `yaml:"cooldown,omitempty"` // before the same issue is worked again (default 1h) + // ApprovalTimeout is how long a remote approval waits before it's a + // "no" (default 15m). + ApprovalTimeout string `yaml:"approval_timeout,omitempty"` + // AllowTx lets fix mode request transaction approvals remotely; + // without it, transactions are always refused in watch mode. + AllowTx bool `yaml:"allow_tx,omitempty"` + // Alerts are where findings, reports and approvals go. Telegram is + // the approval channel (token: here or the TELEGRAM_BOT_TOKEN + // credential); Slack and Discord are notify-only. + Alerts Alerts `yaml:"alerts,omitempty"` + // TelegramUsers, when set, are the only Telegram user ids whose + // button presses count. + TelegramUsers []int64 `yaml:"telegram_users,omitempty"` } // AgentFor resolves the agent settings for a profile (nil = general @@ -127,6 +150,9 @@ type Profile struct { Agent AgentConf `yaml:"agent"` Alerts Alerts `yaml:"alerts"` Metadata map[string]string `yaml:"metadata,omitempty"` + // Sentries names the profiles of the sentry nodes this validator + // peers through (fleet triage checks it isn't cut off). + Sentries []string `yaml:"sentries,omitempty"` } // IsValidator reports whether this profile is a consensus-signing validator. diff --git a/internal/drill/drill_test.go b/internal/drill/drill_test.go new file mode 100644 index 0000000..c08112b --- /dev/null +++ b/internal/drill/drill_test.go @@ -0,0 +1,78 @@ +package drill + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/abhijitkrm/cometcli/internal/config" + "github.com/abhijitkrm/cometcli/internal/toolkit" +) + +func TestGenesisAndConfigPatching(t *testing.T) { + n := &Net{Dir: t.TempDir(), Validators: 2, Image: "evmd:test"} + n.Defaults() + for i := 0; i < 2; i++ { + _ = os.MkdirAll(filepath.Join(n.home(i), "config"), 0o755) + _ = os.WriteFile(filepath.Join(n.home(i), "config", "config.toml"), []byte("addr_book_strict = true\n"), 0o644) + _ = os.WriteFile(filepath.Join(n.home(i), "config", "app.toml"), []byte("minimum-gas-prices = \"0stake\"\naddress = \"localhost:9090\"\n"), 0o644) + } + g := `{"app_state":{"slashing":{"params":{"signed_blocks_window":"100","downtime_jail_duration":"600s"}}, + "gov":{"params":{"voting_period":"172800s"}},"evm":{"params":{"evm_denom":"aevmos"}}}}` + _ = os.WriteFile(filepath.Join(n.home(0), "config", "genesis.json"), []byte(g), 0o644) + denom, err := n.patchGenesis() + if err != nil || denom != "aevmos" { + t.Fatalf("denom %q %v", denom, err) + } + raw, _ := os.ReadFile(filepath.Join(n.home(1), "config", "genesis.json")) + var got map[string]any + _ = json.Unmarshal(raw, &got) + sl := got["app_state"].(map[string]any)["slashing"].(map[string]any)["params"].(map[string]any) + if sl["signed_blocks_window"] != "20" || sl["downtime_jail_duration"] != "30s" { + t.Fatalf("slashing not patched (or genesis not copied): %v", sl) + } + if err := n.patchConfigs(1, denom); err != nil { + t.Fatal(err) + } + app, _ := os.ReadFile(filepath.Join(n.home(1), "config", "app.toml")) + if !strings.Contains(string(app), `"1000000000aevmos"`) || !strings.Contains(string(app), "0.0.0.0:9090") { + t.Fatalf("app.toml:\n%s", app) + } + compose := n.composeFile("501:20") + for _, want := range []string{"container_name: cometcli-drill-1", "127.0.0.1:47067:26657", "image: evmd:test", "172.30.9.3"} { + if !strings.Contains(compose, want) { + t.Errorf("compose lacks %q", want) + } + } +} + +func TestApprovalsOnlyForDrillProfiles(t *testing.T) { + if ok, err := drillApprover(&toolkit.Context{Profile: &config.Profile{Name: "val01"}}, "restart", toolkit.TierLocalChange, nil); ok || err == nil { + t.Fatal("approved a change on a real profile") + } + if ok, err := drillApprover(&toolkit.Context{}, "rm -rf", toolkit.TierLocalChange, nil); ok || err == nil { + t.Fatal("approved a change on this machine (no profile)") + } + drill := &config.Profile{Name: "drill3", Metadata: map[string]string{"drill": "true"}} + if ok, _ := drillApprover(&toolkit.Context{Profile: drill}, "unjail", toolkit.TierOnChain, nil); !ok { + t.Fatal("drill profile not approved") + } + res := RunScenario(context.Background(), &Env{Profile: &config.Profile{Name: "val01"}}, Scenarios[0], nil, func(string) {}) + if !strings.Contains(res.Err, "not a drill profile") { + t.Fatalf("ran a scenario on a real profile: %+v", res) + } +} + +func TestScoreboard(t *testing.T) { + s := Scoreboard([]Result{ + {Scenario: "jail-downtime", TriageCase: "val-jailed-downtime", TriageOK: true, Fixed: true, Steps: 9, Tokens: 41000, Duration: 3 * time.Minute}, + {Scenario: "oom", TriageCase: "node-crash-loop", Fixed: false, FixDetail: "memory limit still 120 MB", Steps: 30}, + }) + if !strings.Contains(s, "score: triage 1/2 · fixed 1/2") || !strings.Contains(s, "memory limit still 120 MB") { + t.Fatalf("scoreboard:\n%s", s) + } +} diff --git a/internal/drill/run.go b/internal/drill/run.go new file mode 100644 index 0000000..807807d --- /dev/null +++ b/internal/drill/run.go @@ -0,0 +1,179 @@ +package drill + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/abhijitkrm/cometcli/internal/agent" + "github.com/abhijitkrm/cometcli/internal/config" + "github.com/abhijitkrm/cometcli/internal/toolkit" + "github.com/abhijitkrm/cometcli/internal/tools/triage" +) + +// Result is one scored drill. +type Result struct { + Scenario string `json:"scenario"` + Model string `json:"model"` + TriageCase string `json:"triage_case"` + TriageOK bool `json:"triage_ok"` // triage named an expected root case + Fixed bool `json:"fixed"` // verified on the node after the agent finished + FixDetail string `json:"fix_detail"` + Steps int `json:"steps"` + Tokens int `json:"tokens"` + Duration time.Duration `json:"duration_ns"` + Report string `json:"report"` + Err string `json:"error,omitempty"` +} + +// AgentFactory builds an agent in general mode with the given approver. +type AgentFactory func(ctx context.Context, approve toolkit.Approver) (*agent.Agent, error) + +// drillApprover approves everything — but only on drill profiles. +func drillApprover(c *toolkit.Context, prompt string, tier toolkit.Tier, _ map[string]any) (bool, error) { + if c == nil || !IsDrill(c.Profile) { + return false, fmt.Errorf("drill approvals only apply to drill profiles") + } + return true, nil +} + +// topCase is triage's leading root case for the node. +func topCase(ctx context.Context, e *Env) string { + c := &toolkit.Context{Context: ctx, Profile: e.Profile, Cfg: e.Cfg} + defer c.Close() + r := triage.Collect(c, 10*time.Minute) + for _, h := range triage.LoadKB(c).Match(r.Signals, r.Chain) { + if h.Case.Kind != "advisory" && h.SymptomOf == "" { + return h.Case.ID + } + } + return "" +} + +// RunScenario injects the fault, has the agent work it, scores the +// outcome and recovers the node. log reports progress. +func RunScenario(ctx context.Context, e *Env, s Scenario, newAgent AgentFactory, log func(string)) (res Result) { + res.Scenario = s.Name // named result: the deferred timing and recovery error must reach the caller + if !IsDrill(e.Profile) { + res.Err = "refusing: " + e.Profile.Name + " is not a drill profile" + return res + } + start := time.Now() + defer func() { + res.Duration = time.Since(start) + log("recovering " + e.Profile.Name) + if err := e.Recover(ctx, func(ctx context.Context) error { return e.unjail(ctx) }); err != nil { + res.Err = strings.TrimSpace(res.Err + "; recovery: " + err.Error()) + } + }() + log(fmt.Sprintf("injecting: %s", s.Desc)) + if err := s.Inject(ctx, e); err != nil { + res.Err = "inject: " + err.Error() + return res + } + if s.Ready != nil && !s.Ready(ctx, e) { + res.Err = "the fault never became visible" + return res + } + res.TriageCase = topCase(ctx, e) + for _, want := range s.Expect { + res.TriageOK = res.TriageOK || res.TriageCase == want + } + log(fmt.Sprintf("triage says %s (%v); the agent takes over", orDash(res.TriageCase), res.TriageOK)) + + a, err := newAgent(ctx, drillApprover) + if err != nil { + res.Err = "agent: " + err.Error() + return res + } + defer a.Close() + res.Model = a.Provider.Name() + "/" + a.Model + cmd, err := agent.RunCommand(a, a.Ctx, a.Reg, "/incident "+fmt.Sprintf(s.Prompt, e.Profile.Name)) + if err != nil { + res.Err = "incident: " + err.Error() + return res + } + runCtx, cancel := context.WithTimeout(ctx, 15*time.Minute) + report, err := a.Run(runCtx, cmd.Prompt) + cancel() + res.Report = report + res.Steps = a.Rounds() + u, _ := a.Usage() + res.Tokens = u.Input + u.Output + if err != nil { + res.Err = "agent run: " + err.Error() + } + log("verifying on the node") + ok := waitFor(ctx, 3*time.Minute, func() bool { v, _ := s.Verify(ctx, e); return v }) + res.Fixed = ok + _, res.FixDetail = s.Verify(ctx, e) + return res +} + +// unjail sends MsgUnjail for the drill node through the normal tool. +func (e *Env) unjail(ctx context.Context) error { + t, ok := e.Reg.Get("val.unjail") + if !ok { + return fmt.Errorf("val.unjail not registered") + } + c := &toolkit.Context{Context: ctx, Profile: e.Profile, Cfg: e.Cfg, Approver: drillApprover} + defer c.Close() + _, err := t.Run(c, toolkit.Args{}) + return err +} + +// Save writes results to ~/.cometcli/drills/