From 7b0bd5a96f84ef916b2e63a491414ac9b075c697 Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:41:06 -0700 Subject: [PATCH 1/5] Run broker-backed transport tests in CI Start RabbitMQ, NATS and Kafka as service containers and run each transport's Category=Explicit tests against them in a separate step. The NATS and Kafka tests now read NATS_URL and KAFKA_BOOTSTRAP_SERVERS (defaulting to localhost) instead of hard-coded Docker addresses, like the RabbitMQ tests' RABBITMQ_URI. Part of #74; the NetMQ tests join once #80 is merged. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dotnet.yml | 33 +++++++++++++++++++ Obvs.Kafka.Tests/TestBroker.cs | 10 ++++++ Obvs.Kafka.Tests/TestMessageSendAndReceive.cs | 9 ++--- Obvs.NATS.Tests/TestBroker.cs | 10 ++++++ Obvs.NATS.Tests/TestServiceBus.cs | 2 +- Obvs.NATS.Tests/TestSourceAndPublisher.cs | 2 +- 6 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 Obvs.Kafka.Tests/TestBroker.cs create mode 100644 Obvs.NATS.Tests/TestBroker.cs diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index 4ac797ec..4c489ac9 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -17,6 +17,27 @@ jobs: contents: read id-token: write # NuGet Trusted Publishing + # Brokers for the transport tests marked Category=Explicit + services: + rabbitmq: + image: rabbitmq:4.4-alpine + ports: ['5672:5672'] + options: >- + --health-cmd "rabbitmq-diagnostics -q check_port_connectivity" + --health-interval 5s --health-timeout 5s --health-retries 20 + nats: + image: nats:2.15 + ports: ['4222:4222'] + kafka: + image: apache/kafka:4.4.0 + ports: ['9092:9092'] + env: + # Assign partitions to the test's new consumer group straight away + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + options: >- + --health-cmd "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092" + --health-interval 5s --health-timeout 10s --health-retries 20 + steps: - name: Checkout uses: actions/checkout@v7 @@ -42,6 +63,18 @@ jobs: run: dotnet build --no-restore --configuration Release -p:Version=${{ steps.gitversion.outputs.semVer }} -p:ContinuousIntegrationBuild=true -p:Deterministic=true - name: Test run: dotnet test --no-build --configuration Release --verbosity normal --filter Category!=Explicit + - name: Create Kafka test topic + # The test consumer reads from the latest offset, so the topic must exist before it subscribes + run: docker exec ${{ job.services.kafka.id }} /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic test-topic-3 + - name: Test transports against brokers + env: + RABBITMQ_URI: amqp://localhost:5672 + NATS_URL: nats://localhost:4222 + KAFKA_BOOTSTRAP_SERVERS: localhost:9092 + run: | + dotnet test Obvs.RabbitMQ.Tests --no-build --configuration Release --verbosity normal --filter Category=Explicit + dotnet test Obvs.NATS.Tests --no-build --configuration Release --verbosity normal --filter Category=Explicit + dotnet test Obvs.Kafka.Tests --no-build --configuration Release --verbosity normal --filter Category=Explicit - name: Pack if: github.event_name == 'push' run: | diff --git a/Obvs.Kafka.Tests/TestBroker.cs b/Obvs.Kafka.Tests/TestBroker.cs new file mode 100644 index 00000000..674d5b38 --- /dev/null +++ b/Obvs.Kafka.Tests/TestBroker.cs @@ -0,0 +1,10 @@ +using System; + +namespace Obvs.Kafka.Tests +{ + internal static class TestBroker + { + // Set KAFKA_BOOTSTRAP_SERVERS to run the Explicit tests against a broker other than a default local install + public static string BootstrapServers => Environment.GetEnvironmentVariable("KAFKA_BOOTSTRAP_SERVERS") ?? "localhost:9092"; + } +} diff --git a/Obvs.Kafka.Tests/TestMessageSendAndReceive.cs b/Obvs.Kafka.Tests/TestMessageSendAndReceive.cs index 39243007..b6061d75 100644 --- a/Obvs.Kafka.Tests/TestMessageSendAndReceive.cs +++ b/Obvs.Kafka.Tests/TestMessageSendAndReceive.cs @@ -44,13 +44,8 @@ public override string ToString() public TestMessageSendAndReceive() { - // Steps to run local kafka broker: - // - Install: Docker for Windows / Docker Desktop - // - Clone: https://github.com/wurstmeister/kafka-docker - // - Edit: docker-compose-single-broker.yml - // - Change: KAFKA_ADVERTISED_HOST_NAME: localhost - // - Run: docker-compose -f "docker-compose-single-broker.yml" up -d --build - var cfg = new KafkaConfiguration("localhost:9092"); + // Run a local broker with: docker run -p 9092:9092 apache/kafka + var cfg = new KafkaConfiguration(TestBroker.BootstrapServers); var sourceConfig = new KafkaSourceConfiguration(); var producerConfig = new KafkaProducerConfiguration(); const string topicName = "test-topic-3"; diff --git a/Obvs.NATS.Tests/TestBroker.cs b/Obvs.NATS.Tests/TestBroker.cs new file mode 100644 index 00000000..e5cf4453 --- /dev/null +++ b/Obvs.NATS.Tests/TestBroker.cs @@ -0,0 +1,10 @@ +using System; + +namespace Obvs.NATS.Tests +{ + internal static class TestBroker + { + // Set NATS_URL to run the Explicit tests against a server other than a default local install + public static string Url => Environment.GetEnvironmentVariable("NATS_URL") ?? "nats://localhost:4222"; + } +} diff --git a/Obvs.NATS.Tests/TestServiceBus.cs b/Obvs.NATS.Tests/TestServiceBus.cs index e7bb5cad..5875d8f1 100644 --- a/Obvs.NATS.Tests/TestServiceBus.cs +++ b/Obvs.NATS.Tests/TestServiceBus.cs @@ -21,7 +21,7 @@ public async Task TestBasicSendReceiveOfAllMessageTypes() settings.ServiceName = "Obvs.NATS.TestService"; settings.Configure(connection => { - connection.Url = "nats://192.168.99.100:32774"; // change to local Docker address:port that maps onto 4222 + connection.Url = TestBroker.Url; connection.IsShared = true; }); settings.Configure(messageProperty => diff --git a/Obvs.NATS.Tests/TestSourceAndPublisher.cs b/Obvs.NATS.Tests/TestSourceAndPublisher.cs index 1838bf6b..8488d00a 100644 --- a/Obvs.NATS.Tests/TestSourceAndPublisher.cs +++ b/Obvs.NATS.Tests/TestSourceAndPublisher.cs @@ -19,7 +19,7 @@ public async Task TestSendAndReceiveMessage() var lazyConnection = new Lazy(() => { var cf = new ConnectionFactory(); - return cf.CreateConnection("nats://192.168.99.100:32774"); // change to local Docker address:port that maps onto 4222 + return cf.CreateConnection(TestBroker.Url); }, LazyThreadSafetyMode.ExecutionAndPublication); const string subjectPrefix = "Obvs.NATS.Tests"; From 582e352952723ea78e67bb72d5f7a06d31ad0dfc Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:42:40 -0700 Subject: [PATCH 2/5] Configure the CI Kafka container as a full single-node KRaft broker Setting KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS alone replaced the image defaults and the broker failed to start (missing process.roles). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dotnet.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index 4c489ac9..a2b4931a 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -31,7 +31,20 @@ jobs: kafka: image: apache/kafka:4.4.0 ports: ['9092:9092'] + # Setting any KAFKA_* variable replaces the image's defaults, so configure a full single-node KRaft broker env: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 1 # Assign partitions to the test's new consumer group straight away KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 options: >- From 6dcd910f036cec63fdaf522c64b987bc2d1865e3 Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:45:49 -0700 Subject: [PATCH 3/5] Warm up the CI Kafka broker before the transport tests The Kafka test consumer starts at the latest offset and Subscribe returns before partitions are assigned, so on a fresh broker it missed the messages while __consumer_offsets was being created. Use one offsets partition and run a warm-up consumer group first. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dotnet.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index a2b4931a..ed624641 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -41,6 +41,7 @@ jobs: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1 @@ -76,9 +77,12 @@ jobs: run: dotnet build --no-restore --configuration Release -p:Version=${{ steps.gitversion.outputs.semVer }} -p:ContinuousIntegrationBuild=true -p:Deterministic=true - name: Test run: dotnet test --no-build --configuration Release --verbosity normal --filter Category!=Explicit - - name: Create Kafka test topic - # The test consumer reads from the latest offset, so the topic must exist before it subscribes - run: docker exec ${{ job.services.kafka.id }} /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic test-topic-3 + - name: Prepare Kafka + # The test consumer starts at the latest offset and doesn't wait for partition assignment, so create its + # topic and warm up the group coordinator (which creates __consumer_offsets) before the tests run + run: | + docker exec ${{ job.services.kafka.id }} /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic test-topic-3 + docker exec ${{ job.services.kafka.id }} /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-topic-3 --group ci-warmup --timeout-ms 10000 || true - name: Test transports against brokers env: RABBITMQ_URI: amqp://localhost:5672 From 12f4f0272050a64c21715221ac173741a531abd2 Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:05:58 -0700 Subject: [PATCH 4/5] Run the broker tests with xUnit v3 test syntax, and add NetMQ Master now runs tests on Microsoft.Testing.Platform (#101), so use --project and --filter-trait. The NetMQ tests no longer clash over ports (#80), so run them too. The broker steps now come before the unit test step, so this branch no longer edits the lines master changed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dotnet.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index ed624641..e0c2ab47 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -75,8 +75,6 @@ jobs: run: dotnet restore - name: Build run: dotnet build --no-restore --configuration Release -p:Version=${{ steps.gitversion.outputs.semVer }} -p:ContinuousIntegrationBuild=true -p:Deterministic=true - - name: Test - run: dotnet test --no-build --configuration Release --verbosity normal --filter Category!=Explicit - name: Prepare Kafka # The test consumer starts at the latest offset and doesn't wait for partition assignment, so create its # topic and warm up the group coordinator (which creates __consumer_offsets) before the tests run @@ -89,9 +87,12 @@ jobs: NATS_URL: nats://localhost:4222 KAFKA_BOOTSTRAP_SERVERS: localhost:9092 run: | - dotnet test Obvs.RabbitMQ.Tests --no-build --configuration Release --verbosity normal --filter Category=Explicit - dotnet test Obvs.NATS.Tests --no-build --configuration Release --verbosity normal --filter Category=Explicit - dotnet test Obvs.Kafka.Tests --no-build --configuration Release --verbosity normal --filter Category=Explicit + dotnet test --project Obvs.RabbitMQ.Tests --no-build --configuration Release -- --filter-trait "Category=Explicit" + dotnet test --project Obvs.NATS.Tests --no-build --configuration Release -- --filter-trait "Category=Explicit" + dotnet test --project Obvs.Kafka.Tests --no-build --configuration Release -- --filter-trait "Category=Explicit" + dotnet test --project Obvs.NetMQ.Tests --no-build --configuration Release -- --filter-trait "Category=Explicit" + - name: Test + run: dotnet test --no-build --configuration Release --verbosity normal --filter Category!=Explicit - name: Pack if: github.event_name == 'push' run: | From a74b489525951ec54dddffa57b1cfd8777bf6862 Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:10:10 -0700 Subject: [PATCH 5/5] Leave the NetMQ tests out of the CI broker step for now They lose the first messages on CI runners (ZeroMQ slow joiner): the publisher socket is created on first publish and sends before subscribers have connected. Tracked separately. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dotnet.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index e0c2ab47..b95a8e17 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -90,7 +90,6 @@ jobs: dotnet test --project Obvs.RabbitMQ.Tests --no-build --configuration Release -- --filter-trait "Category=Explicit" dotnet test --project Obvs.NATS.Tests --no-build --configuration Release -- --filter-trait "Category=Explicit" dotnet test --project Obvs.Kafka.Tests --no-build --configuration Release -- --filter-trait "Category=Explicit" - dotnet test --project Obvs.NetMQ.Tests --no-build --configuration Release -- --filter-trait "Category=Explicit" - name: Test run: dotnet test --no-build --configuration Release --verbosity normal --filter Category!=Explicit - name: Pack