From 283fb708a4c4b1aec20d55c00db271f60222fd70 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 30 Oct 2026 10:22:56 +0000 Subject: [PATCH] test: add an opt-in emulate.dev end-to-end tier for the Slack adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `mise run emulate` (emulate@0.12.3) with a seed in testdata/emulate/seed.yaml, and TestEmulate* tests in adapters/slack that skip unless CHAT_EMULATE_SLACK_URL is set. They cover a signed Events API message event through the adapter's HTTP handler to a thread reply that conversations.replies shows, and a simulated button click through the adapter's HTTP handler to the interaction handler. A separate Emulate workflow runs the tier, so it cannot turn CI red. CONTRIBUTING.md explains how to run it and lists emulate's known gaps, including the Linear schema differences that block Linear tests (#148). Closes #147 _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `high`_ --- .github/workflows/emulate.yaml | 56 +++++++ CONTRIBUTING.md | 46 ++++++ adapters/linear/README.md | 1 + adapters/slack/README.md | 1 + adapters/slack/emulate_test.go | 284 +++++++++++++++++++++++++++++++++ mise.toml | 9 ++ testdata/emulate/seed.yaml | 57 +++++++ 7 files changed, 454 insertions(+) create mode 100644 .github/workflows/emulate.yaml create mode 100644 adapters/slack/emulate_test.go create mode 100644 testdata/emulate/seed.yaml diff --git a/.github/workflows/emulate.yaml b/.github/workflows/emulate.yaml new file mode 100644 index 0000000..0c3c373 --- /dev/null +++ b/.github/workflows/emulate.yaml @@ -0,0 +1,56 @@ +# The emulate.dev test tier: the Slack adapter against a local emulator. It +# lives in its own workflow so that a failure here does not mark CI red. See +# "Emulator Tests" in CONTRIBUTING.md. +name: Emulate + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + emulate: + name: emulate (slack) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + + - name: Install mise-pinned toolchain + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Start emulate + run: | + nohup mise run emulate > "$RUNNER_TEMP/emulate.log" 2>&1 & + for _ in $(seq 90); do + if curl -fsS -X POST -H 'Authorization: Bearer xoxb-emulate-bot' \ + http://127.0.0.1:4710/api/auth.test > /dev/null 2>&1; then + exit 0 + fi + sleep 1 + done + echo "emulate did not start within 90 seconds" + cat "$RUNNER_TEMP/emulate.log" + exit 1 + + - name: Test + run: mise run test:emulate + env: + GOFLAGS: -race + + - name: Show emulate log + if: always() + run: cat "$RUNNER_TEMP/emulate.log" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c874feb..660d5dc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,6 +92,52 @@ Some docs are checked by `documentation_test.go`: - A few key phrases must stay in the README, reference, explanation, and package docs. If you reword one, update the test in the same change. +## Emulator Tests + +An opt-in tier runs the Slack adapter end to end against +[emulate.dev](https://emulate.dev) (`vercel-labs/emulate`, pinned to 0.12.3), +a local, stateful emulator of the Slack and Linear APIs. It covers flows that +the fakes do not cover end to end: + +- a signed Events API message event, through the adapter's HTTP handler and + the mention handler, to a thread reply that `conversations.replies` shows; +- a button click from emulate's block-action simulator, through the adapter's + HTTP handler, to the interaction handler. + +You need Node.js 22 or later for `npx`. Run from the repository root: + +```sh +mise run emulate # terminal 1: Slack on :4710, Linear on :4711 +mise run test:emulate # terminal 2: sets CHAT_EMULATE_SLACK_URL, runs TestEmulate* +``` + +The seed is `testdata/emulate/seed.yaml`. The block-action test listens on +`127.0.0.1:4799`, the seeded interactivity URL. Without +`CHAT_EMULATE_SLACK_URL`, the tests skip and `go test ./...` is unchanged. The +`Emulate` workflow runs the tier on each pull request, separately from `CI`. + +emulate is not the source of truth for Slack's rules. The fakes +(`adapters/slack/slackapi/slackapitest` and `linearAPIServer` in +`adapters/linear/linear_test.go`) and the live tests (`TestLive*` in +`adapters/slack/slackapi/live_test.go`) are. Known gaps in emulate 0.12.3: + +- No Socket Mode: `apps.connections.open` returns 404. +- No slash commands. +- No Events API request URL in the CLI or the seed. The event test posts the + user message to emulate, then signs and sends its `event_callback` itself. +- emulate accepts a JSON body for `conversations.history` and + `conversations.replies`. Real Slack returns `invalid_arguments`, so this tier + would have missed #98. +- Scope checks are relaxed unless the seed sets `strict_scopes: true`. This + seed leaves them relaxed. +- No Linear tests. emulate's Linear schema differs from Linear's, so the + adapter cannot reply: `User` has no `organization` (single-install `Init` + fails), `CommentCreateInput` has no `parentId`, `AgentActivityCreateInput` is + `{sessionId, type, body}` instead of `{agentSessionId, content}`, Comment + webhooks have no `data.user`, and `AgentSessionEvent` webhooks carry the + session in `data` instead of `agentSession`. The seed keeps a Linear team for + manual checks. Issue #148 tracks Linear tests. + ## Validating Adapter Changes Live CI has no live Slack or Linear tests. Before you claim that a change works diff --git a/adapters/linear/README.md b/adapters/linear/README.md index c79beeb..7dfad3d 100644 --- a/adapters/linear/README.md +++ b/adapters/linear/README.md @@ -135,3 +135,4 @@ The adapter owns its own rate-limit retry (`RetryPolicy`) and Linear-specific ex ## Verification - Run: `go test ./adapters/linear/...` +- The emulate.dev tier has no Linear tests yet, because emulate's Linear schema differs from Linear's. See "Emulator Tests" in `CONTRIBUTING.md` and issue #148. diff --git a/adapters/slack/README.md b/adapters/slack/README.md index 0ad774a..3cfc864 100644 --- a/adapters/slack/README.md +++ b/adapters/slack/README.md @@ -109,3 +109,4 @@ The code has compile-time assertions for `chat.Listener`, `chat.HistoryReader`, ## Verification - Run: `go test ./adapters/slack/...` +- Emulator tier (opt-in): `mise run emulate`, then `mise run test:emulate`. It runs `TestEmulate*` in `emulate_test.go`, which skip without `CHAT_EMULATE_SLACK_URL`. See "Emulator Tests" in `CONTRIBUTING.md`. [emulate_test.go:newEmulateSlack] diff --git a/adapters/slack/emulate_test.go b/adapters/slack/emulate_test.go new file mode 100644 index 0000000..d087394 --- /dev/null +++ b/adapters/slack/emulate_test.go @@ -0,0 +1,284 @@ +// The emulate tests run the Slack adapter end to end against emulate.dev +// (github.com/vercel-labs/emulate), a local, stateful Slack emulator. They +// skip unless this variable is set: +// +// CHAT_EMULATE_SLACK_URL base URL of the emulated Slack, for example http://127.0.0.1:4710 +// +// Start the emulator in one terminal and run the tests in another, from the +// repository root: +// +// mise run emulate +// mise run test:emulate +// +// The emulate constants below must match testdata/emulate/seed.yaml. emulate +// is not the source of truth for Slack's rules: slackapitest and the TestLive +// tests are. CONTRIBUTING.md lists the gaps that emulate does not cover. + +package slack_test + +import ( + "bytes" + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/coder/chat" + "github.com/coder/chat/adapters/slack" + "github.com/coder/chat/adapters/slack/slackapi" + "github.com/coder/chat/adapters/slack/slackapi/slackapitest" + "github.com/coder/chat/state/memory" +) + +const ( + emulateSigningSecret = "emulate-signing-secret" + emulateBotToken = "xoxb-emulate-bot" + emulateUserToken = "xoxp-emulate-alice" + emulateUserName = "alice" + emulateChannelName = "chat-emulate" + // emulateInteractivityAddr is the host and port of the interactivity_url + // in the seed. emulate sends block-action clicks there. + emulateInteractivityAddr = "127.0.0.1:4799" +) + +// emulateSlack is one test's connection to the emulated Slack workspace. +type emulateSlack struct { + baseURL string + bot *chat.Chat + adapter *slack.Adapter + botAPI *slackapi.Client // Web API client with the bot token + userAPI *slackapi.Client // Web API client with the user token + userID string + channel string +} + +// newEmulateSlack skips the test when CHAT_EMULATE_SLACK_URL is unset. +// Otherwise it builds a runtime with the Slack adapter pointed at the emulator +// and looks up the seeded user and channel. +func newEmulateSlack(t *testing.T, ctx context.Context) *emulateSlack { + t.Helper() + + baseURL := strings.TrimRight(os.Getenv("CHAT_EMULATE_SLACK_URL"), "/") + if baseURL == "" { + t.Skip("set CHAT_EMULATE_SLACK_URL and run `mise run emulate` to run the emulate.dev Slack tests") + } + apiURL := baseURL + "/api" + + adapter, err := slack.New(ctx, slack.Options{ + SigningSecret: emulateSigningSecret, + BotToken: emulateBotToken, + APIBaseURL: apiURL, + }) + if err != nil { + t.Fatalf("new slack adapter: %v", err) + } + // chat.New calls Init, which reads the bot identity from auth.test. + bot, err := chat.New(ctx, chat.WithState(memory.New()), chat.WithAdapter(adapter)) + if err != nil { + t.Fatalf("new chat runtime against %s: %v", baseURL, err) + } + + botAPI := slackapi.New(slackapi.Options{Token: emulateBotToken, BaseURL: apiURL}) + userAPI := botAPI.WithToken(emulateUserToken) + user, err := userAPI.AuthTest(ctx) + if err != nil { + t.Fatalf("auth.test with the user token: %v", err) + } + if user.UserID == "" || user.BotID != "" { + t.Fatalf("auth.test with the user token = %+v, want the seeded human user %q", user, emulateUserName) + } + + var list struct { + Channels []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"channels"` + } + if err := botAPI.Call(ctx, "conversations.list", nil, &list); err != nil { + t.Fatalf("conversations.list: %v", err) + } + var channel string + for _, c := range list.Channels { + if c.Name == emulateChannelName { + channel = c.ID + } + } + if channel == "" { + t.Fatalf("seeded channel #%s not found; is emulate running with testdata/emulate/seed.yaml?", emulateChannelName) + } + // emulate adds the seeded users to the seeded channels, but not the bot user + // that the OAuth app creates. + if err := botAPI.Call(ctx, "conversations.join", map[string]string{"channel": channel}, nil); err != nil { + t.Fatalf("conversations.join: %v", err) + } + + return &emulateSlack{ + baseURL: baseURL, + bot: bot, + adapter: adapter, + botAPI: botAPI, + userAPI: userAPI, + userID: user.UserID, + channel: channel, + } +} + +// TestEmulateSlackEventReply sends a signed Events API message event to the +// adapter's HTTP handler. The mention handler replies in the thread, and +// conversations.replies on the emulator must show the reply. +// +// emulate cannot register an Events API request URL from its CLI or seed, so +// the test posts the user message to the emulator and then signs and sends the +// event_callback for it itself. +func TestEmulateSlackEventReply(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + e := newEmulateSlack(t, ctx) + botUser := e.adapter.BotActor() + + e.bot.OnNewMention(func(ctx context.Context, ev *chat.MessageEvent) error { + _, err := ev.Thread.Post(ctx, chat.Text("pong: "+ev.Message.Text)) + return err + }) + webhook, err := e.bot.Webhook("slack") + if err != nil { + t.Fatalf("webhook: %v", err) + } + srv := httptest.NewServer(webhook) + defer srv.Close() + + text := "<@" + botUser.ID + "> ping " + strconv.FormatInt(time.Now().UnixNano(), 10) + posted, err := e.userAPI.PostMessage(ctx, slackapi.PostMessageRequest{Channel: e.channel, Text: text}) + if err != nil { + t.Fatalf("post user message: %v", err) + } + + body, err := json.Marshal(map[string]any{ + "type": "event_callback", + "team_id": botUser.Tenant, + "api_app_id": "A0EMULATE1", + "event_id": "Ev" + strings.ReplaceAll(posted.TS, ".", ""), + "event_time": time.Now().Unix(), + "event": map[string]any{ + "type": "message", + "channel": e.channel, + "channel_type": "channel", + "user": e.userID, + "text": text, + "ts": posted.TS, + "event_ts": posted.TS, + }, + }) + if err != nil { + t.Fatalf("encode event: %v", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL, bytes.NewReader(body)) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + slackapitest.SignRequest(req, body, emulateSigningSecret, time.Now()) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("deliver event: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("event delivery status = %d, want 200", resp.StatusCode) + } + + // The default dispatch mode runs the handler before the webhook returns, so + // the reply is already stored. + page, err := e.botAPI.ConversationReplies(ctx, slackapi.ConversationRepliesRequest{Channel: e.channel, TS: posted.TS}) + if err != nil { + t.Fatalf("conversations.replies: %v", err) + } + want := "pong: " + text + for _, m := range page.Messages { + if m.TS != posted.TS && m.ThreadTS == posted.TS && m.Text == want && m.BotID != "" { + return + } + } + t.Fatalf("conversations.replies has no bot reply %q in thread %s: %+v", want, posted.TS, page.Messages) +} + +// TestEmulateSlackBlockAction clicks a button with emulate's block-action +// simulator. emulate sends the signed interaction to the seeded +// interactivity_url, where the adapter's HTTP handler must call the +// interaction handler with the clicked action and the clicking user. +func TestEmulateSlackBlockAction(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + e := newEmulateSlack(t, ctx) + + got := make(chan *chat.Interaction, 1) + e.bot.OnInteraction(func(_ context.Context, ev *chat.InteractionEvent) error { + got <- ev.Interaction + return nil + }) + webhook, err := e.bot.Webhook("slack") + if err != nil { + t.Fatalf("webhook: %v", err) + } + ln, err := net.Listen("tcp", emulateInteractivityAddr) + if err != nil { + t.Fatalf("listen on the seeded interactivity address %s: %v", emulateInteractivityAddr, err) + } + srv := httptest.NewUnstartedServer(webhook) + srv.Listener.Close() + srv.Listener = ln + srv.Start() + defer srv.Close() + + value := "approve-" + strconv.FormatInt(time.Now().UnixNano(), 10) + var posted slackapi.PostMessageResponse + if err := e.botAPI.Call(ctx, "chat.postMessage", map[string]any{ + "channel": e.channel, + "text": "Approve?", + "blocks": []any{map[string]any{ + "type": "actions", + "elements": []any{map[string]any{ + "type": "button", + "action_id": "emulate_approve", + "value": value, + "text": map[string]any{"type": "plain_text", "text": "Approve"}, + }}, + }}, + }, &posted); err != nil { + t.Fatalf("post button message: %v", err) + } + + // The simulator is not a Web API method, so this client calls it from the + // emulator root instead of /api. + simulator := slackapi.New(slackapi.Options{Token: emulateUserToken, BaseURL: e.baseURL}) + if err := simulator.Call(ctx, "_slack/simulate/block-actions", map[string]string{ + "channel": e.channel, + "message_ts": posted.TS, + "action_id": "emulate_approve", + "user": emulateUserName, + }, nil); err != nil { + t.Fatalf("simulate block action: %v", err) + } + + select { + case interaction := <-got: + if interaction.ActionID != "emulate_approve" || interaction.Value != value { + t.Fatalf("interaction action = %q value = %q, want emulate_approve %q", interaction.ActionID, interaction.Value, value) + } + if interaction.Actor.ID != e.userID || interaction.Actor.BotKind != chat.BotHuman { + t.Fatalf("interaction actor = %+v, want human %s", interaction.Actor, e.userID) + } + case <-ctx.Done(): + t.Fatal("interaction handler was not called") + } +} diff --git a/mise.toml b/mise.toml index 9673fe6..f897bea 100644 --- a/mise.toml +++ b/mise.toml @@ -18,6 +18,15 @@ run = [ "clawpatch map", ] +[tasks.emulate] +description = "Start the emulate.dev Slack (:4710) and Linear (:4711) emulators for the emulate test tier" +run = "npx --yes emulate@0.12.3 start --service slack,linear --port 4710 --seed testdata/emulate/seed.yaml" + +[tasks."test:emulate"] +description = "Run the emulate.dev tests against a running `mise run emulate`" +env = { CHAT_EMULATE_SLACK_URL = "http://127.0.0.1:4710" } +run = "go test -run TestEmulate -count=1 -v ./adapters/slack/" + [tasks."test:adapters"] description = "Run adapter-focused tests" run = [ diff --git a/testdata/emulate/seed.yaml b/testdata/emulate/seed.yaml new file mode 100644 index 0000000..765817b --- /dev/null +++ b/testdata/emulate/seed.yaml @@ -0,0 +1,57 @@ +# Seed for the emulate.dev test tier (`mise run emulate`). See CONTRIBUTING.md. +# The Slack emulate tests in adapters/slack/emulate_test.go depend on these +# values: change both together. emulate generates the Slack user, bot, and +# channel IDs at startup, so the tests look them up by name. +slack: + team: + name: Chat SDK Emulate + domain: chat-sdk-emulate + # Signs every interactivity callback. The tests also sign the Events API + # deliveries that they send with it. + signing_secret: emulate-signing-secret + users: + - name: alice + real_name: Alice + channels: + - name: chat-emulate + oauth_apps: + - app_id: A0EMULATE1 + client_id: "1000.2000" + client_secret: emulate-client-secret + name: Chat SDK Bot + bot_name: chat-bot + redirect_uris: [] + # The block-action test listens on this address. + interactivity_url: http://127.0.0.1:4799/slack/interactions + tokens: + - token: xoxb-emulate-bot + type: bot + user: chat-bot + app_id: A0EMULATE1 + - token: xoxp-emulate-alice + type: user + user: alice + +# No Go test uses Linear yet: emulate's Linear schema differs from Linear's +# (see CONTRIBUTING.md). The seed keeps a team so that manual checks have data. +linear: + organization: + name: Chat SDK Emulate + url_key: chat-sdk-emulate + users: + - email: alice@example.com + name: Alice + teams: + - key: EMU + name: Emulate + oauth_apps: + - client_id: lin_emulate_client + client_secret: lin_emulate_secret + name: Chat SDK Bot + redirect_uris: [] + scopes: [read, write, app:mentionable, app:assignable] + actor: app + tokens: + - token: lin_emulate_alice + user: alice@example.com + scopes: [read, write, issues:create, comments:create, admin]