refactor(kvstore): replace Consul backend with Redis

README.md

@@ -247,9 +247,8 @@ go build -o indexer cmd/indexer/main.go
 Start required services before running the indexer (docker-compose provided):
 
 - NATS server (events)
-- Consul (KV) or Badger (embedded)
+- Redis (KV store, Bloom filter, ManualWorker) — or Badger for the embedded KV backend
 - PostgreSQL (wallet address repo)
-- Redis (for Bloom filter or ManualWorker)
 
 ```bash
 docker-compose up -d
@@ -260,7 +259,7 @@ docker-compose up -d
 ## 🔧 Configuration
 
 - **Chains**: configurable (`start_block`, `batch_size`, `poll_interval`)
-- **KVStore**: BadgerDB / in-memory / Consul
+- **KVStore**: Redis / BadgerDB (embedded)
 - **Bloom Filter**: Redis or in-memory
 - **Event Emitter**: NATS streaming
 - **RPC Providers**: failover + rate-limiting
@@ -305,7 +304,7 @@ nats consumer sub transfer transaction-consumer
 # Initialize bloom filter and kvstore
 ./wallet-kv-load run --config configs/config.yaml --batch 10000 --debug
 
-# Migrate from Badger to Consul (edit migrate.yaml first)
+# Migrate KV between Badger and Redis (edit migrate.yaml first)
 ./kv-migrate run --config configs/config.yaml --dry-run
 ```
 

cmd/devtools/blockreliability/main.go

@@ -76,7 +76,7 @@ func main() {
 	}
 
 	// KVStore
-	kv, err := kvstore.NewFromConfig(services.KVS)
+	kv, err := kvstore.NewFromConfig(services.KVS, redisClient.GetClient())
 	if err != nil {
 		logger.Fatal("KVStore connection failed", "err", err)
 	}

cmd/indexer/main.go

@@ -124,8 +124,8 @@ func runIndexer(chains []string, configPath string, debug, manual, catchup, from
 	}
 
 	// start kvstore
-	logger.Info("Connecting to kvstore", "url", services.KVS.Consul.Address)
-	kvstore, err := kvstore.NewFromConfig(services.KVS)
+	logger.Info("Connecting to kvstore", "type", services.KVS.Type)
+	kvstore, err := kvstore.NewFromConfig(services.KVS, redisClient.GetClient())
 	if err != nil {
 		logger.Fatal("Create kvstore failed", "err", err)
 	}

cmd/kv-migrate/main.go

@@ -12,7 +12,6 @@ import (
 	"github.com/fystack/multichain-indexer/pkg/infra"
 	"github.com/fystack/multichain-indexer/pkg/kvstore"
 	"github.com/goccy/go-yaml"
-	"github.com/hashicorp/consul/api"
 )
 
 // ANSI color codes
@@ -44,7 +43,7 @@ type EndpointType string
 
 const (
 	EndpointTypeBadger EndpointType = "badger"
-	EndpointTypeConsul EndpointType = "consul"
+	EndpointTypeRedis  EndpointType = "redis"
 )
 
 type CLI struct {
@@ -62,14 +61,21 @@ type MigrationConfig struct {
 type EndpointConfig struct {
 	Type   enum.KVStoreType    `yaml:"type"`
 	Badger config.BadgerConfig `yaml:"badger,omitempty"`
-	Consul config.ConsulConfig `yaml:"consul,omitempty"`
+	Redis  RedisEndpointConfig `yaml:"redis,omitempty"`
+}
+
+type RedisEndpointConfig struct {
+	URL      string `yaml:"url"`
+	Password string `yaml:"password"`
+	MTLS     bool   `yaml:"mtls"`
+	Prefix   string `yaml:"prefix"`
 }
 
 func main() {
 	var cli CLI
 	ctx := kong.Parse(&cli,
 		kong.Name("kv-migrate"),
-		kong.Description("Migrate keys between Badger and Consul KV stores"))
+		kong.Description("Migrate keys between Badger and Redis KV stores"))
 
 	printBanner()
 
@@ -194,25 +200,15 @@ func buildStore(config EndpointConfig) (infra.KVStore, error) {
 			config.Badger.Prefix,
 			infra.JSON,
 		)
-	case enum.KVStoreTypeConsul:
-		if config.Consul.Address == "" {
-			return nil, fmt.Errorf("consul address is required")
+	case enum.KVStoreTypeRedis:
+		if config.Redis.URL == "" {
+			return nil, fmt.Errorf("redis url is required")
 		}
-		var httpAuth *api.HttpBasicAuth
-		if config.Consul.HttpAuth.Username != "" || config.Consul.HttpAuth.Password != "" {
-			httpAuth = &api.HttpBasicAuth{
-				Username: config.Consul.HttpAuth.Username,
-				Password: config.Consul.HttpAuth.Password,
-			}
+		rc, err := infra.NewRedisClient(config.Redis.URL, config.Redis.Password, "", config.Redis.MTLS)
+		if err != nil {
+			return nil, err
 		}
-		return kvstore.NewConsulClient(kvstore.Options{
-			Scheme:   config.Consul.Scheme,
-			Address:  config.Consul.Address,
-			Folder:   config.Consul.Folder,
-			Codec:    infra.JSON,
-			Token:    config.Consul.Token,
-			HttpAuth: httpAuth,
-		})
+		return kvstore.NewRedisStore(rc.GetClient(), config.Redis.Prefix, infra.JSON)
 	default:
 		return nil, fmt.Errorf("unsupported store type: %s", config.Type)
 	}

cmd/wallet-kv-load/main.go

@@ -15,11 +15,12 @@ import (
 	"github.com/fystack/multichain-indexer/pkg/kvstore"
 	"github.com/fystack/multichain-indexer/pkg/model"
 	"github.com/fystack/multichain-indexer/pkg/repository"
+	"github.com/redis/go-redis/v9"
 	"gorm.io/gorm"
 )
 
 type CLI struct {
-	Run RunCmd `cmd:"" help:"Load wallet addresses from DB into Consul KV."`
+	Run RunCmd `cmd:"" help:"Load wallet addresses from DB into the KV store."`
 }
 
 type RunCmd struct {
@@ -48,11 +49,21 @@ func (c *RunCmd) Run() error {
 		logger.Fatal("Create db connection failed", "err", err)
 	}
 
-	// Build KV store from config; must be consul
-	if cfg.Services.KVS.Type != enum.KVStoreTypeConsul {
-		logger.Fatal("KVStore type must be consul for this command", "type", cfg.Services.KVS.Type)
+	// Build KV store from config. Redis needs a client; badger does not.
+	var redisClient *redis.Client
+	if cfg.Services.KVS.Type == enum.KVStoreTypeRedis {
+		rc, err := infra.NewRedisClient(
+			cfg.Services.Redis.URL,
+			cfg.Services.Redis.Password,
+			string(cfg.Environment),
+			cfg.Services.Redis.MTLS,
+		)
+		if err != nil {
+			logger.Fatal("Create redis client failed", "err", err)
+		}
+		redisClient = rc.GetClient()
 	}
-	store, err := kvstore.NewFromConfig(cfg.Services.KVS)
+	store, err := kvstore.NewFromConfig(cfg.Services.KVS, redisClient)
 	if err != nil {
 		logger.Fatal("Create KV store failed", "err", err)
 	}

configs/config.example.yaml

@@ -515,15 +515,12 @@ services:
     mtls: false # enable mutual TLS with client certificates (production only)
 
   kvstore:
-    type: "consul"
-    consul:
-      scheme: "http"
-      address: "127.0.0.1:8500"
-      folder: "indexer"
-      token: ""
-      # http_auth:                   # optional: enable if consul is secured
-      #   username: ""
-      #   password: ""
+    type: "redis" # redis or badger
+    redis:
+      prefix: "indexer" # key namespace; reuses the shared redis connection above
+    # badger:                        # embedded local backend (no external service)
+    #   directory: "data/badger"
+    #   prefix: "indexer"
 
   bloomfilter:
     type: "redis" # redis or in_memory

docker-compose.yml

@@ -31,20 +31,6 @@ services:
     tty: true
     restart: always
 
-  consul:
-    image: hashicorp/consul:1.20
-    container_name: consul
-    ports:
-      - "8500:8500"
-      - "8601:8600/udp"
-    command: "agent -server -ui -node=server-1 -bootstrap-expect=1 -client=0.0.0.0"
-    healthcheck:
-      test: ["CMD", "consul", "operator", "raft", "list-peers"]
-      interval: 5s
-      timeout: 3s
-      retries: 10
-    restart: always
-
 volumes:
   db_data:
   redis_data:

go.mod

@@ -13,7 +13,6 @@ require (
 	github.com/go-viper/mapstructure/v2 v2.4.0
 	github.com/goccy/go-yaml v1.19.2
 	github.com/golang/protobuf v1.5.4
-	github.com/hashicorp/consul/api v1.32.1
 	github.com/jackc/pgx/v5 v5.7.5
 	github.com/lmittmann/tint v1.1.2
 	github.com/mr-tron/base58 v1.2.0
@@ -38,40 +37,26 @@ replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.16
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/fatih/color v1.16.0 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.12 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/google/flatbuffers v25.2.10+incompatible // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-hclog v1.5.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/hashicorp/serf v0.10.1 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/nats-io/nkeys v0.4.11 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
@@ -87,7 +72,6 @@ require (
 	go.opentelemetry.io/otel/metric v1.38.0 // indirect
 	go.opentelemetry.io/otel/trace v1.38.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
 	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/sys v0.41.0 // indirect
 	golang.org/x/text v0.34.0 // indirect