From cd87941eca7c4829b432d85a42884f009495c112 Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 9 Apr 2026 20:59:25 +0200 Subject: [PATCH 01/75] refactor: better handling of reverse proxy headers Previously, the real IP of the client was not properly handled by certstream-server-go. This commit takes the IP address from the header and stores them in the request context and the client struct. Additionally the user agent is now also preserved. Also each client now gets a generated ID to identify it even if another client from the same IP connects to the server. Also some log statements were changed so that it fits better when printing the client name. --- internal/metrics/prometheus.go | 64 ++++++++++- internal/web/broadcastmanager.go | 28 ++--- internal/web/client.go | 76 ++++++++++-- internal/web/client_test.go | 145 +++++++++++++++++++++++ internal/web/server.go | 191 +++++++++++++++++++++++++++---- 5 files changed, 451 insertions(+), 53 deletions(-) create mode 100644 internal/web/client_test.go diff --git a/internal/metrics/prometheus.go b/internal/metrics/prometheus.go index c846226..caca426 100644 --- a/internal/metrics/prometheus.go +++ b/internal/metrics/prometheus.go @@ -4,12 +4,17 @@ import ( "fmt" "io" "log" + "maps" + "slices" + "strings" "sync" "time" "github.com/VictoriaMetrics/metrics" ) +var illegalCharacters = []string{"{", "}", "\n", `"`} + var Prometheus = NewPrometheusExporter() // PrometheusExporter is holds the metrics for the Prometheus exporter. It also provides helper functions @@ -59,15 +64,39 @@ func (pm *PrometheusExporter) RegisterGaugeMetricInt(label string, callback func } // RegisterClient registers a new gauge metric for the client with the given name. -func (pm *PrometheusExporter) RegisterClient(name string, callback func() float64) { - label := fmt.Sprintf("certstreamservergo_skipped_certs{client=\"%s\"}", name) - metrics.GetOrCreateGauge(label, callback) +func (pm *PrometheusExporter) RegisterClient(id, connIP, connPort, realIP, useragent string, skippedCertsCallback func() float64) { + argMap := make(map[string]string) + argMap["id"] = id + argMap["conn_ip"] = connIP + argMap["conn_port"] = connPort + argMap["real_ip"] = realIP + argMap["useragent"] = useragent + + label := createMetric("certstreamservergo_skipped_certs", argMap) + + // label := fmt.Sprintf("certstreamservergo_skipped_certs{id=\"%s\",conn_ip=\"%s\",conn_port=\"%s\",real_ip=\"%s\",useragent=\"%s\"}", + // id, connIP, connPort, realIP, useragent) + metrics.GetOrCreateGauge(label, skippedCertsCallback) } // UnregisterClient unregisters the metric for the client with the given name. -func (pm *PrometheusExporter) UnregisterClient(name string) { - label := fmt.Sprintf("certstreamservergo_skipped_certs{client=\"%s\"}", name) - metrics.UnregisterMetric(label) +func (pm *PrometheusExporter) UnregisterClient(id, connIP, connPort, realIP, useragent string) { + argMap := make(map[string]string) + argMap["id"] = id + argMap["conn_ip"] = connIP + argMap["conn_port"] = connPort + argMap["real_ip"] = realIP + argMap["useragent"] = useragent + + label := createMetric("certstreamservergo_skipped_certs", argMap) + + // label := fmt.Sprintf("certstreamservergo_skipped_certs{id=\"%s\",conn_ip=\"%s\",conn_port=\"%s\",real_ip=\"%s\",useragent=\"%s\"}", + // id, connIP, connPort, realIP, useragent) + + ok := metrics.UnregisterMetric(label) + if !ok { + log.Printf("failed to unregister metric '%s'", label) + } } // RegisterLog registers a new gauge metric for the given CT log. @@ -111,3 +140,26 @@ func (pm *PrometheusExporter) getCertCountForLog(operatorName, logname string) i return count } + +func createMetric(metricName string, argMap map[string]string) string { + argString := "" + + for _, key := range slices.Sorted(maps.Keys(argMap)) { + key = sanitizeMetricValue(key) + key = strings.ReplaceAll(key, " ", "_") + value := sanitizeMetricValue(argMap[key]) + argString += fmt.Sprintf(`%s="%s",`, key, value) //nolint:perfsprint + } + + argString = strings.TrimRight(argString, ",") + + return fmt.Sprintf("%s{%s}", metricName, argString) +} + +func sanitizeMetricValue(value string) string { + for _, char := range illegalCharacters { + value = strings.ReplaceAll(value, char, "_") + } + + return value +} diff --git a/internal/web/broadcastmanager.go b/internal/web/broadcastmanager.go index 16f5015..7ddadae 100644 --- a/internal/web/broadcastmanager.go +++ b/internal/web/broadcastmanager.go @@ -29,22 +29,22 @@ func (bm *BroadcastManager) registerClient(c *client) { bm.clientLock.Lock() bm.clients = append(bm.clients, c) log.Printf("Clients: %d, Capacity: %d\n", len(bm.clients), cap(bm.clients)) - metrics.Prometheus.RegisterClient(c.name, func() float64 { return float64(c.skippedCerts) }) + metrics.Prometheus.RegisterClient(c.id, c.connectionIP, c.connectionPort, c.realIPFromHeader, c.userAgent, func() float64 { return float64(c.skippedCerts) }) bm.clientLock.Unlock() } // unregisterClient removes a client from the list of clients of the BroadcastManager. // The client will no longer receive certificate broadcasts right after unregistering. -func (bm *BroadcastManager) unregisterClient(targetClient *client) { +func (bm *BroadcastManager) unregisterClient(c *client) { bm.clientLock.Lock() // Close the broadcast channel of the client, otherwise this leads to a memory leak - close(targetClient.broadcastChan) - metrics.Prometheus.UnregisterClient(targetClient.name) + close(c.broadcastChan) + metrics.Prometheus.UnregisterClient(c.id, c.connectionIP, c.connectionPort, c.realIPFromHeader, c.userAgent) // Remove client from internal client list - for i, c := range bm.clients { - if targetClient != c { + for i, storedClient := range bm.clients { + if c != storedClient { continue } @@ -90,18 +90,6 @@ func (bm *BroadcastManager) clientCountByType(subType SubscriptionType) (count i return count } -func (bm *BroadcastManager) GetSkippedCerts() map[string]uint64 { - bm.clientLock.RLock() - defer bm.clientLock.RUnlock() - - skippedCerts := make(map[string]uint64, len(bm.clients)) - for _, c := range bm.clients { - skippedCerts[c.name] = c.skippedCerts - } - - return skippedCerts -} - // broadcaster is run in a goroutine and handles the dispatching of entries to clients. func (bm *BroadcastManager) broadcaster() { for { @@ -124,7 +112,7 @@ func (bm *BroadcastManager) broadcaster() { case SubTypeDomain: data = dataDomain default: - log.Printf("Unknown subscription type '%d' for client '%s'. Skipping this client!\n", c.subType, c.name) + log.Printf("Unknown subscription type '%d'. Skipping client %s\n", c.subType, c.Name()) continue } @@ -134,7 +122,7 @@ func (bm *BroadcastManager) broadcaster() { // Default case is executed if the client's broadcast channel is full. c.skippedCerts++ if c.skippedCerts%1000 == 1 { - log.Printf("Not providing client '%s' with cert because client's buffer is full. The client can't keep up. Skipped certs: %d\n", c.name, c.skippedCerts) + log.Printf("Client can't keep up. Skipped certs: %d | Client: %s\n", c.skippedCerts, c.Name()) } } } diff --git a/internal/web/client.go b/internal/web/client.go index b11a15c..8e1c986 100644 --- a/internal/web/client.go +++ b/internal/web/client.go @@ -3,12 +3,16 @@ package web import ( "fmt" "log" + "math/rand" + "net" "strings" "time" "github.com/gorilla/websocket" ) +const idChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + const ( SubTypeFull SubscriptionType = iota SubTypeLite @@ -19,29 +23,51 @@ type SubscriptionType int // client represents a single client's connection to the server. type client struct { + clientData + + id string conn *websocket.Conn broadcastChan chan []byte - name string subType SubscriptionType skippedCerts uint64 } -func newClient(conn *websocket.Conn, subType SubscriptionType, name string, certBufferSize int) *client { +type clientData struct { + userAgent string + connectionIP string + connectionPort string + realIPFromHeader string +} + +// newClient creates a new client struct that holds information about a connected client. +func newClient(conn *websocket.Conn, subType SubscriptionType, data clientData, certBufferSize int) *client { return &client{ + clientData: data, + id: generateClientID(), conn: conn, broadcastChan: make(chan []byte, certBufferSize), - name: name, subType: subType, } } +// generateClientID generates a random 8-char identifier for the client. +func generateClientID() string { + clientID := make([]byte, 8) + for i := range clientID { + //nolint:gosec + clientID[i] = idChars[rand.Intn(len(idChars))] + } + + return string(clientID) +} + // Each client has a broadcastHandler that runs in the background and sends out the broadcast messages to the client. func (c *client) broadcastHandler() { writeWait := 60 * time.Second pingTicker := time.NewTicker(30 * time.Second) defer func() { - log.Println("Closing broadcast handler for client:", c.conn.RemoteAddr()) + log.Println("Closing broadcast handler for client:", c.Name()) //nolint:gosec pingTicker.Stop() @@ -127,7 +153,7 @@ func (c *client) listenWebsocket() { // If client fails to send ping messages if strings.Contains(strings.ToLower(readErr.Error()), "i/o timeout") { - log.Printf("No ping received from client: %v\n", c.conn.RemoteAddr()) + log.Printf("No ping received from client: %s\n", c.Name()) //nolint:gosec closeMessage := websocket.FormatCloseMessage(websocket.ClosePolicyViolation, "No ping received!") @@ -136,12 +162,48 @@ func (c *client) listenWebsocket() { log.Printf("Error while sending close message: %v\n", writeErr) } } else if strings.Contains(strings.ToLower(readErr.Error()), "an existing connection was forcibly closed by the remote host") { - log.Printf("Connection to client lost: %v\n", c.conn.RemoteAddr()) + log.Printf("Connection to client lost: %s\n", c.Name()) //nolint:gosec } - log.Printf("Disconnecting client %v!\n", c.conn.RemoteAddr()) + log.Printf("Disconnecting client %s!\n", c.Name()) //nolint:gosec break } } } + +// sanitizeInput removes newline and carriage-return characters from a +// string to prevent log-injection attacks (gosec G706). +func sanitizeInput(s string) string { + s = strings.ReplaceAll(s, "\n", "\\n") + s = strings.ReplaceAll(s, "\r", "\\r") + + return s +} + +// Name returns the name/identifier for this client. +func (c *client) Name() string { + var clientName string + + clientName = fmt.Sprintf("[%s] - ", c.id) + + connIP := sanitizeInput(c.connectionIP) + connPort := sanitizeInput(c.connectionPort) + realIP := sanitizeInput(c.realIPFromHeader) + + socket := net.JoinHostPort(connIP, connPort) + + // If the realIP is set and if it differs from the connection IP, return both the connection IP and the real IP. + if realIP != "" && realIP != connIP { + clientName += fmt.Sprintf("%s (via %s)", socket, realIP) + } else { + clientName += socket + } + + if c.userAgent != "" { + ua := sanitizeInput(c.userAgent) + clientName += fmt.Sprintf(" - '%s'", ua) + } + + return clientName +} diff --git a/internal/web/client_test.go b/internal/web/client_test.go new file mode 100644 index 0000000..b1e2b30 --- /dev/null +++ b/internal/web/client_test.go @@ -0,0 +1,145 @@ +package web + +import ( + "fmt" + "testing" +) + +func newTestClient(id, connectionIP, connectionPort, realIP, userAgent string) *client { + c := &client{ + clientData: clientData{ + connectionIP: connectionIP, + connectionPort: connectionPort, + realIPFromHeader: realIP, + userAgent: userAgent, + }, + id: id, + } + return c +} + +// TestClientName tests if the Name() function returns properly formatted names based on various client configurations. +func TestClientName(t *testing.T) { + tests := []struct { + name string + id string + connectionIP string + connectionPort string + realIP string + userAgent string + want string + }{ + { + name: "connection IP only", + id: "M7hBCSxj", + connectionIP: "1.2.3.4", + connectionPort: "1234", + realIP: "", + userAgent: "", + want: "[M7hBCSxj] - 1.2.3.4:1234", + }, + { + name: "realIP same as connectionIP", + id: "Epdy7ZIl", + connectionIP: "1.2.3.4", + connectionPort: "1234", + realIP: "1.2.3.4", + userAgent: "", + want: "[Epdy7ZIl] - 1.2.3.4:1234", + }, + { + name: "realIP differs from connectionIP", + id: "4Y0NE9v0", + connectionIP: "10.0.0.1", + connectionPort: "1234", + realIP: "203.0.113.5", + userAgent: "", + want: "[4Y0NE9v0] - 10.0.0.1:1234 (via 203.0.113.5)", + }, + { + name: "with user agent", + id: "a32J8PW4", + connectionIP: "1.2.3.4", + connectionPort: "1234", + realIP: "", + userAgent: "go-certstream/1.0", + want: "[a32J8PW4] - 1.2.3.4:1234 - 'go-certstream/1.0'", + }, + { + name: "with user agent and proxy", + id: "qiBSXENv", + connectionIP: "10.0.0.1", + connectionPort: "1234", + realIP: "203.0.113.5", + userAgent: "go-certstream/1.0", + want: "[qiBSXENv] - 10.0.0.1:1234 (via 203.0.113.5) - 'go-certstream/1.0'", + }, + { + name: "user agent with newline is escaped", + id: "enf8k2HQ", + connectionIP: "1.2.3.4", + connectionPort: "1234", + realIP: "", + userAgent: "bad\nagent", + want: "[enf8k2HQ] - 1.2.3.4:1234 - 'bad\\nagent'", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newTestClient(tt.id, tt.connectionIP, tt.connectionPort, tt.realIP, tt.userAgent) + got := c.Name() + if got != tt.want { + t.Errorf("Name() = %q, want %q", got, tt.want) + } + }) + } +} + +// TestGenerateClientID tests if generating a client ID produces a string of the correct length and character set. +func TestGenerateClientID(t *testing.T) { + id := generateClientID() + if len(id) != 8 { + t.Errorf("generateClientID() length = %d, want 8", len(id)) + } + + for _, ch := range id { + if !isValidIDChar(ch) { + t.Errorf("generateClientID() contains invalid character: %q", ch) + } + } +} + +// TestGenerateClientIDUniqueness tests if there are collisions within 1000 generated IDs. +func TestGenerateClientIDUniqueness(t *testing.T) { + const iterations = 1000 + seen := make(map[string]struct{}, iterations) + + for i := range iterations { + id := generateClientID() + if _, exists := seen[id]; exists { + t.Logf("collision after %d iterations (non-fatal, low probability expected)", i) + } + seen[id] = struct{}{} + } + + if len(seen) < iterations/2 { + t.Errorf("too many collisions: only %d unique IDs out of %d", len(seen), iterations) + } +} + +func isValidIDChar(ch rune) bool { + return (ch >= 'a' && ch <= 'z') || + (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') +} + +// TestClientNameAlwaysContainsID tests if the id prefix is always present in the Name() output. +func TestClientNameAlwaysContainsID(t *testing.T) { + c := newTestClient("TESTID1X", "5.6.7.8", "", "", "") + got := c.Name() + expected := fmt.Sprintf("[%s]", "TESTID1X") + if len(got) < len(expected) || got[:len(expected)] != expected { + t.Errorf("Name() %q does not start with id prefix %q", got, expected) + } +} diff --git a/internal/web/server.go b/internal/web/server.go index c08d78b..0883bf6 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "strconv" + "strings" "time" "github.com/go-chi/chi/v5" @@ -25,6 +26,14 @@ var ( upgrader websocket.Upgrader ) +type contextKey int + +const ( + // origConnAddrKey is the context key under which the original TCP connection + // address (r.RemoteAddr before any RealIP rewriting) is stored. + origConnAddrKey contextKey = iota +) + // Server is a struct that holds the necessary information to run a webserver. // It is used for the websocket server as well as the metrics server. type Server struct { @@ -45,6 +54,98 @@ func (ws *Server) RegisterPrometheus(url string, callback func(w io.Writer, expo }) } +// getForwardedIP extracts the real client IP from well-known reverse-proxy headers. +// Order of precedence: X-Forwarded-For > X-Real-IP. +// Returns an empty string when no valid IP is found in any header. +func getForwardedIP(r *http.Request) string { + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + parts := strings.Split(xff, ",") + if ip := strings.TrimSpace(parts[0]); ip != "" { + if parsed := net.ParseIP(ip); parsed != nil { + return parsed.String() + } + } + } + + if ip := r.Header.Get("X-Real-IP"); ip != "" { + if parsed := net.ParseIP(strings.TrimSpace(ip)); parsed != nil { + return parsed.String() + } + } + + return "" +} + +// realIPMiddleware returns a middleware that always stores the original r.RemoteAddr +// in the request context (under origConnAddrKey) before anything can overwrite it. +// +// When realIP is true it also rewrites r.RemoteAddr to the forwarded client IP taken +// from proxy headers (X-Forwarded-For / X-Real-IP). +// +// If trustedProxies is non-empty the rewrite is only performed when the actual TCP +// connection IP is in that list; otherwise a warning is logged and r.RemoteAddr is +// left unchanged. An empty trustedProxies list means every connecting IP is trusted, +// which preserves the previous behavior. +func realIPMiddleware(realIP bool, trustedProxies []string) func(next http.Handler) http.Handler { + ipList, cidrList := generateIPList(trustedProxies) + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Always capture the real TCP connection address before anything overwrites it. + origAddr := r.RemoteAddr + ctx := context.WithValue(r.Context(), origConnAddrKey, origAddr) + r = r.WithContext(ctx) + + // If extracting the realIP is not desired, skip the rest of the middleware. + if !realIP { + next.ServeHTTP(w, r) + return + } + + // Parse the actual connection IP to check if the relevant headers can be trusted. + connIPStr, _, err := net.SplitHostPort(origAddr) + if err != nil { + connIPStr = origAddr + } + + connIP := net.ParseIP(connIPStr) + + // Empty trusted_proxies = trust every connection. + trusted := len(ipList) == 0 && len(cidrList) == 0 + + // If trusted_proxies is not empty, check if connIP is contained within trusted_proxies. + if !trusted && connIP != nil { + for _, cidr := range cidrList { + if cidr.Contains(connIP) { + trusted = true + break + } + } + + if !trusted { + for _, trustedIP := range ipList { + if trustedIP.Equal(connIP) { + trusted = true + break + } + } + } + } + + // If the source IP is trusted, replace the RemoteAddr with the IP from the header. + if trusted { + if fwdIP := getForwardedIP(r); fwdIP != "" { + r.RemoteAddr = fwdIP + } + } else { + log.Printf("Warning: connection from %s is not in trusted_proxies, ignoring forwarded IP headers\n", origAddr) //nolint:gosec + } + + next.ServeHTTP(w, r) + }) + } +} + // IPWhitelist returns a middleware that checks if the IP of the client is in the whitelist. func IPWhitelist(whitelist []string) func(next http.Handler) http.Handler { // build a list of whitelisted IPs and CIDRs @@ -82,7 +183,14 @@ func IPWhitelist(whitelist []string) func(next http.Handler) http.Handler { return } - ipString, _, err := net.SplitHostPort(r.RemoteAddr) + // Use the original TCP connection IP stored in context so that the + // whitelist check is not affected by forwarded-for headers. + addrToCheck := r.RemoteAddr + if origAddr, ok := r.Context().Value(origConnAddrKey).(string); ok && origAddr != "" { + addrToCheck = origAddr + } + + ipString, _, err := net.SplitHostPort(addrToCheck) if err != nil { http.Error(w, "InternalServerError", http.StatusInternalServerError) return @@ -104,12 +212,33 @@ func IPWhitelist(whitelist []string) func(next http.Handler) http.Handler { } } - log.Printf("IP %s not in whitelist, rejecting request\n", r.RemoteAddr) + log.Printf("IP %s not in whitelist, rejecting request\n", addrToCheck) //nolint:gosec http.Error(w, "Forbidden", http.StatusForbidden) }) } } +// generateIPList parses and returns two slices of net.IP and net.IPNet from a string slice. +// The input string slice needs to contain IP addresses or CIDR notations. Invalid entries are ignored. +func generateIPList(inputIPList []string) ([]net.IP, []net.IPNet) { + var ipList []net.IP + var cidrList []net.IPNet + + for _, element := range inputIPList { + _, ipNet, err := net.ParseCIDR(element) + if err == nil { + cidrList = append(cidrList, *ipNet) + continue + } + + if ip := net.ParseIP(element); ip != nil { + ipList = append(ipList, ip) + } + } + + return ipList, cidrList +} + // initFullWebsocket is called when a client connects to the /full-stream endpoint. // It upgrades the connection to a websocket and starts a goroutine to listen for messages from the client. func initFullWebsocket(w http.ResponseWriter, r *http.Request) { @@ -119,7 +248,7 @@ func initFullWebsocket(w http.ResponseWriter, r *http.Request) { return } - setupClient(connection, SubTypeFull, r.RemoteAddr) + setupClient(connection, SubTypeFull, r) } // initLiteWebsocket is called when a client connects to the / endpoint. @@ -131,7 +260,7 @@ func initLiteWebsocket(w http.ResponseWriter, r *http.Request) { return } - setupClient(connection, SubTypeLite, r.RemoteAddr) + setupClient(connection, SubTypeLite, r) } // initDomainWebsocket is called when a client connects to the /domains-only endpoint. @@ -143,30 +272,30 @@ func initDomainWebsocket(w http.ResponseWriter, r *http.Request) { return } - setupClient(connection, SubTypeDomain, r.RemoteAddr) + setupClient(connection, SubTypeDomain, r) } // upgradeConnection upgrades the connection to a websocket and returns the connection. func upgradeConnection(w http.ResponseWriter, r *http.Request) (*websocket.Conn, error) { var remoteAddr string - xForwardedFor := r.Header.Get("X-Forwarded-For") - if xForwardedFor != "" { - remoteAddr = fmt.Sprintf("'%s' (X-Forwarded-For: '%s')", r.RemoteAddr, xForwardedFor) + ipFromHeader, _ := r.Context().Value(origConnAddrKey).(string) + if ipFromHeader != "" && ipFromHeader != r.RemoteAddr { + remoteAddr = fmt.Sprintf("'%s' (via proxy '%s')", ipFromHeader, r.RemoteAddr) } else { remoteAddr = fmt.Sprintf("'%s'", r.RemoteAddr) } - log.Printf("Starting new websocket for %s - %s\n", remoteAddr, r.URL) + log.Printf("Starting new websocket for %s - URI: '%s'\n", remoteAddr, r.URL) //nolint:gosec connection, err := upgrader.Upgrade(w, r, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("error while upgrading connection: %w", err) } defaultCloseHandler := connection.CloseHandler() connection.SetCloseHandler(func(code int, text string) error { - log.Printf("Stopping websocket for %s - %s\n", remoteAddr, r.URL) + log.Printf("Stopping websocket for %s - URI: '%s'\n", remoteAddr, r.URL) //nolint:gosec return defaultCloseHandler(code, text) }) @@ -174,8 +303,29 @@ func upgradeConnection(w http.ResponseWriter, r *http.Request) (*websocket.Conn, } // setupClient initializes a client struct and starts the broadcastHandler and websocket listener. -func setupClient(connection *websocket.Conn, subscriptionType SubscriptionType, name string) { - c := newClient(connection, subscriptionType, name, config.AppConfig.General.BufferSizes.Websocket) +func setupClient(connection *websocket.Conn, subType SubscriptionType, r *http.Request) { + // Extract data from request + origConnAddr, _ := r.Context().Value(origConnAddrKey).(string) + + hostIP, hostPort, err := net.SplitHostPort(origConnAddr) + if err != nil { + log.Printf("Error while trying to parse remote address: %s\n", origConnAddr) //nolint:gosec + } + + // Only pass the real IP from header if the RemoteAddr was altered + realIPFromHeader := "" + if r.RemoteAddr != origConnAddr { + realIPFromHeader = r.RemoteAddr + } + + data := clientData{ + userAgent: r.Header.Get("User-Agent"), + connectionIP: hostIP, + connectionPort: hostPort, + realIPFromHeader: realIPFromHeader, + } + + c := newClient(connection, subType, data, config.AppConfig.General.BufferSizes.Websocket) go c.broadcastHandler() go c.listenWebsocket() @@ -239,10 +389,10 @@ func NewMetricsServer(networkIf string, port int, certPath, keyPath string) *Ser keyPath: keyPath, } metricsServer.routes.Use(middleware.Recoverer) - - if config.AppConfig.Prometheus.RealIP { - metricsServer.routes.Use(middleware.RealIP) - } + metricsServer.routes.Use(realIPMiddleware( + config.AppConfig.Prometheus.RealIP, + config.AppConfig.Prometheus.TrustedProxies, + )) // Enable IP whitelist if configured if len(config.AppConfig.Prometheus.Whitelist) > 0 { @@ -274,9 +424,10 @@ func NewWebsocketServer(networkIf string, port int, certPath, keyPath string) *S }, } - if config.AppConfig.Webserver.RealIP { - websocketServer.routes.Use(middleware.RealIP) - } + websocketServer.routes.Use(realIPMiddleware( + config.AppConfig.Webserver.RealIP, + config.AppConfig.Webserver.TrustedProxies, + )) // Enable IP whitelist if configured if len(config.AppConfig.Webserver.Whitelist) > 0 { From 887cb7163d5096321e44089089e656c1f705774c Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 9 Apr 2026 21:07:47 +0200 Subject: [PATCH 02/75] refactor: use helper method for generating an IP whitelist --- internal/web/server.go | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/internal/web/server.go b/internal/web/server.go index 0883bf6..f3e1d80 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -147,30 +147,10 @@ func realIPMiddleware(realIP bool, trustedProxies []string) func(next http.Handl } // IPWhitelist returns a middleware that checks if the IP of the client is in the whitelist. +// It always checks against the original TCP connection IP (stored in context by +// newRealIPMiddleware), so the whitelist cannot be bypassed via forwarded headers. func IPWhitelist(whitelist []string) func(next http.Handler) http.Handler { - // build a list of whitelisted IPs and CIDRs - log.Println("Building IP whitelist...") - - var ipList []net.IP - var cidrList []net.IPNet - - for _, element := range whitelist { - _, ipNet, err := net.ParseCIDR(element) - if err != nil { - var ip net.IP - if ip = net.ParseIP(element); ip == nil { - log.Println("Invalid IP in metrics whitelist: ", element) - - continue - } - - ipList = append(ipList, ip) - - continue - } - - cidrList = append(cidrList, *ipNet) - } + ipList, cidrList := generateIPList(whitelist) log.Println("IP whitelist: ", ipList) log.Println("CIDR whitelist: ", cidrList) From 75ac2dfdffbbf8ce11e6660546f47432d8d3b828 Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 9 Apr 2026 21:13:26 +0200 Subject: [PATCH 03/75] docs: add trusted_proxies option and comments to sample config --- config.sample.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/config.sample.yaml b/config.sample.yaml index 99966fe..682c8eb 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -3,8 +3,19 @@ webserver: listen_addr: "0.0.0.0" listen_port: 8080 # If you want to use a reverse proxy in front of the server, set this to true - # It will use the X-Forwarded-For header to get the real IP of the client + # It will use the X-Forwarded-For / X-Real-IP headers to get the real IP of the client real_ip: false + # List of trusted proxy IPs or CIDR ranges whose X-Forwarded-For / X-Real-IP headers + # are honored when real_ip is true. When left empty every connecting IP is trusted. + # Restricting this to your actual proxy IPs prevents clients from spoofing their address + # by injecting a forged forwarded header. Note: X-Forwarded-For can contain a chain of + # IPs (e.g. "client, proxy1, proxy2"); only the first (leftmost) entry is used as the real client IP. + trusted_proxies: + - "172.16.0.0/12" + # If you want to restrict access to the server to specific IPs or CIDR ranges, you can add them to the whitelist. + # If the whitelist is empty, all IPs are allowed to access the server. + whitelist: + - "127.0.0.1/8" full_url: "/full-stream" lite_url: "/" domains_only_url: "/domains-only" @@ -20,6 +31,9 @@ prometheus: metrics_url: "/metrics" expose_system_metrics: false real_ip: false + # List of trusted proxy IPs or CIDR ranges (see webserver.trusted_proxies for details). + #trusted_proxies: + # - "127.0.0.1" whitelist: - "127.0.0.1/8" From 00e72504bab4c2a7cd35aa18fb0732f208711532 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 20 Apr 2026 01:54:56 +0200 Subject: [PATCH 04/75] fix(static-ct): defer partial tile fetching Optimize tile processing by delaying the fetch of partial tiles. Certstream will wait up to a minute before forcefully fetching the partial tile to ensure slow-growing logs are still processed. Certstream will now: - Track when a partial tile is first seen - Wait up to one minute before forcing a partial fetch - Skip fetching partial tiles within the deferral window - Reset tracking when a full tile was downloaded fixes #104 --- internal/certificatetransparency/ct-tiled.go | 56 ++++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/internal/certificatetransparency/ct-tiled.go b/internal/certificatetransparency/ct-tiled.go index bb593d2..1834d50 100644 --- a/internal/certificatetransparency/ct-tiled.go +++ b/internal/certificatetransparency/ct-tiled.go @@ -261,12 +261,25 @@ func ConvertTileLeafToRawLogEntry(leaf TileLeaf, index uint64) *ct.RawLogEntry { return rawEntry } + +// DefaultMaxPartialWait is the default maximum time to wait before forcefully +// fetching a partial tile that has not yet grown into a full tile. +const DefaultMaxPartialWait = 1 * time.Minute + type StaticCTClient struct { url string httpClient *http.Client backoff backoff.Backoff userAgent string ctIndex uint64 + + // Deferred partial-tile state. + // partialTileIndex holds the tile index of the currently tracked partial tile. + // partialTileFirstSeen is when that partial tile was first observed; zero means + // no partial tile is being tracked. + partialTileIndex uint64 + partialTileFirstSeen time.Time + maxPartialWait time.Duration } func NewStaticCTClient(url string, httpClient *http.Client, userAgent string, startIndex uint64) *StaticCTClient { @@ -281,6 +294,7 @@ func NewStaticCTClient(url string, httpClient *http.Client, userAgent string, st }, userAgent: userAgent, ctIndex: startIndex, + maxPartialWait: DefaultMaxPartialWait, } } @@ -332,22 +346,54 @@ func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert fun endTile := currentTreeSize / TileSize // Process full tiles + fetchedFullTiles := false for tileIndex := startTile; tileIndex < endTile; tileIndex++ { if err := s.processTile(ctx, tileIndex, 0, foundCert, foundPrecert); err != nil { return false, fmt.Errorf("processing tile %d: %w", tileIndex, err) } + + fetchedFullTiles = true + } + + // When the current end tile has advanced past the tracked partial tile, that tile + // has since become a full tile and been processed; reset tracking so we start + // fresh for the new partial tile (if any). + if endTile > s.partialTileIndex { + s.partialTileFirstSeen = time.Time{} } - // Process partial tile if exists + // Process partial tiles. partialSize := currentTreeSize % TileSize if partialSize > 0 { - if err := s.processTile(ctx, endTile, partialSize, foundCert, foundPrecert); err != nil { - log.Printf("Warning: error processing partial tile %d: %s\n", endTile, err) - // Don't return error for partial tiles as they might be incomplete + switch { + case s.partialTileFirstSeen.IsZero() || s.partialTileIndex != endTile: + // First time we see this partial tile – start the deferral clock. + s.partialTileIndex = endTile + s.partialTileFirstSeen = time.Now() + log.Println("Deferring fetch of partial tile", endTile, "with size", partialSize) + + case time.Since(s.partialTileFirstSeen) >= s.maxPartialWait: + // The partial tile has been pending too long – fetch it now to prevent + // extreme processing delays on slow-growing logs. + log.Println("Forcefully fetching partial tile", endTile, "with size", partialSize) + + if err := s.processTile(ctx, endTile, partialSize, foundCert, foundPrecert); err != nil { + log.Printf("Warning: error processing partial tile %d: %s\n", endTile, err) + } + + // Reset tracking; the tile will be re-observed on the next poll if it + // still hasn't grown into a full tile. + s.partialTileFirstSeen = time.Time{} + + default: + // Still within the deferral window – skip. } + } else { + // currentTreeSize is an exact multiple of TileSize; no partial tile exists. + s.partialTileFirstSeen = time.Time{} } - return true, nil + return fetchedFullTiles, nil } // processTile processes a single tile from the tiled log. From cd48ef08f1a5910f093603b85626d4f9109605db Mon Sep 17 00:00:00 2001 From: Rico Date: Sat, 2 May 2026 00:54:06 +0200 Subject: [PATCH 05/75] build: reformat goreleaser config --- .github/goreleaser.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/goreleaser.yml b/.github/goreleaser.yml index 406e3d6..175e5fd 100644 --- a/.github/goreleaser.yml +++ b/.github/goreleaser.yml @@ -15,13 +15,13 @@ builds: - darwin - windows goarch: - - 386 + - "386" - amd64 - arm - arm64 ignore: - goos: darwin - goarch: 386 + goarch: "386" - goos: darwin goarch: arm - goos: windows @@ -29,9 +29,9 @@ builds: - goos: windows goarch: arm64 - goos: windows - goarch: 386 + goarch: "386" checksum: - name_template: '{{.ProjectName}}_{{.Version}}_checksums.txt' + name_template: "{{.ProjectName}}_{{.Version}}_checksums.txt" changelog: disable: true @@ -41,8 +41,8 @@ dockers_v2: - "ghcr.io/d-rickyy-b/{{.ProjectName}}" dockerfile: docker/Dockerfile.goreleaser tags: - - "{{.Tag}}" - - "{{ if not .Prerelease }}latest{{ end }}" + - "{{.Tag}}" + - "{{ if not .Prerelease }}latest{{ end }}" platforms: - linux/amd64 - linux/arm64 @@ -58,7 +58,7 @@ dockers_v2: "org.opencontainers.image.source": "https://github.com/d-Rickyy-b/certstream-server-go" archives: - - formats: [ "binary" ] + - formats: ["binary"] name_template: >- {{- .ProjectName }}_ {{- .Version}}_ From dacd033e8c27874b70df050e9392818a70e25d2f Mon Sep 17 00:00:00 2001 From: Rico Date: Sat, 2 May 2026 01:17:16 +0200 Subject: [PATCH 06/75] build: run build workflow on pushes to dev branch --- .github/workflows/release_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml index ea64c15..eaa35b3 100644 --- a/.github/workflows/release_build.yml +++ b/.github/workflows/release_build.yml @@ -4,6 +4,8 @@ on: push: tags: - "*" + branches: + - dev jobs: build: From 4b3547f980f3f3bd034421577cf377984072e31d Mon Sep 17 00:00:00 2001 From: Rico Date: Sat, 2 May 2026 01:53:09 +0200 Subject: [PATCH 07/75] refactor: stop background task when context is canceled --- internal/certificatetransparency/ct-watcher.go | 2 +- internal/metrics/logmetrics.go | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index 75b5420..d3bb313 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -75,7 +75,7 @@ func (w *Watcher) Start() { // Start background job to save CTIndexes at regular intervals storageInterval := time.Second * 30 - go metrics.Metrics.SaveCertIndexesAtInterval(storageInterval, ctIndexFilePath) + go metrics.Metrics.SaveCertIndexesAtInterval(w.context, storageInterval, ctIndexFilePath) } // initialize the watcher with currently available logs diff --git a/internal/metrics/logmetrics.go b/internal/metrics/logmetrics.go index 1638897..0204fe8 100644 --- a/internal/metrics/logmetrics.go +++ b/internal/metrics/logmetrics.go @@ -1,6 +1,7 @@ package metrics import ( + "context" "encoding/json" "fmt" "log" @@ -239,13 +240,18 @@ func (m *LogMetrics) createCTIndexFile(ctIndexFilePath string) error { // We first create a temp file and write the index data to it. Only then do we move the temp file to the actual // permanent index file. This prevents the last good index file from being clobbered if the program was shutdown/killed // in-between the write operation. -func (m *LogMetrics) SaveCertIndexesAtInterval(interval time.Duration, ctIndexFilePath string) { +func (m *LogMetrics) SaveCertIndexesAtInterval(ctx context.Context, interval time.Duration, ctIndexFilePath string) { ticker := time.NewTicker(interval) defer ticker.Stop() - for range ticker.C { - if err := m.SaveCertIndexes(ctIndexFilePath); err != nil { - log.Printf("Error saving CT indexes at '%s': %s\n", ctIndexFilePath, err) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := m.SaveCertIndexes(ctIndexFilePath); err != nil { + log.Printf("Error saving CT indexes at '%s': %s\n", ctIndexFilePath, err) + } } } } From 31d74d8bf9afd1c2b519b5c741a7ae1638aac4a9 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 11 May 2026 02:33:28 +0200 Subject: [PATCH 08/75] test(logmetrics): add various unit tests --- internal/metrics/logmetrics_test.go | 203 ++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/internal/metrics/logmetrics_test.go b/internal/metrics/logmetrics_test.go index 00e9e69..984e7f4 100644 --- a/internal/metrics/logmetrics_test.go +++ b/internal/metrics/logmetrics_test.go @@ -1,14 +1,94 @@ package metrics import ( + "encoding/json" + "os" "path/filepath" "testing" "time" ) +func TestCreateCTIndexFile_InitializesNilIndex(t *testing.T) { + m := &LogMetrics{metrics: make(CTMetrics), index: nil} + path := filepath.Join(t.TempDir(), "ct_index.json") + + err := m.createCTIndexFile(path) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + + if m.index == nil { + t.Fatal("expected index to be initialized, but it is nil") + } + + data, readErr := os.ReadFile(path) + if readErr != nil { + t.Fatalf("expected file to exist, got read error: %v", readErr) + } + + var parsed CTCertIndex + if jsonErr := json.Unmarshal(data, &parsed); jsonErr != nil { + t.Fatalf("expected valid JSON, got: %v", jsonErr) + } + + if len(parsed) != 0 { + t.Fatalf("expected empty index, got %d entries", len(parsed)) + } +} + +func TestCreateCTIndexFile_WritesExistingIndexData(t *testing.T) { + index := CTCertIndex{ + "https://ct.example.com/log": 42, + "https://ct.other.com/log": 100, + } + m := &LogMetrics{metrics: make(CTMetrics), index: index} + path := filepath.Join(t.TempDir(), "ct_index.json") + + err := m.createCTIndexFile(path) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + + data, readErr := os.ReadFile(path) + if readErr != nil { + t.Fatalf("expected file to exist, got read error: %v", readErr) + } + + var parsed CTCertIndex + if jsonErr := json.Unmarshal(data, &parsed); jsonErr != nil { + t.Fatalf("expected valid JSON, got: %v", jsonErr) + } + + for key, expectedVal := range index { + gotVal, ok := parsed[key] + if !ok { + t.Errorf("expected key %q in file, but not found", key) + continue + } + if gotVal != expectedVal { + t.Errorf("for key %q: expected %d, got %d", key, expectedVal, gotVal) + } + } +} + +func TestCreateCTIndexFile_ReturnsErrorForInvalidPath(t *testing.T) { + m := &LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} + // Use a path inside a non-existent directory to trigger os.Create failure. + path := filepath.Join(t.TempDir(), "nonexistent", "ct_index.json") + + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for invalid path, but did not panic") + } + }() + + _ = m.createCTIndexFile(path) +} + func TestLoadCTIndex_DoesNotDeadlockWhenFileMissing(t *testing.T) { metrics := LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} ctIndexPath := filepath.Join(t.TempDir(), "ct_index.json") + os.WriteFile(ctIndexPath, []byte("{}"), 0644) done := make(chan struct{}) go func() { @@ -23,3 +103,126 @@ func TestLoadCTIndex_DoesNotDeadlockWhenFileMissing(t *testing.T) { t.Fatalf("LoadCTIndex appears to deadlock when index file is missing") } } + +func TestLoadCTIndex_CreatesFileWhenMissing(t *testing.T) { + m := &LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} + path := filepath.Join(t.TempDir(), "ct_index.json") + + m.LoadCTIndex(path) + + if _, err := os.Stat(path); os.IsNotExist(err) { + t.Fatal("expected LoadCTIndex to create the file when missing, but file does not exist") + } + + if len(m.index) != 0 { + t.Fatalf("expected empty index after loading missing file, got %d entries", len(m.index)) + } +} + +func TestLoadCTIndex_LoadsValidFile(t *testing.T) { + m := &LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} + path := filepath.Join(t.TempDir(), "ct_index.json") + + want := CTCertIndex{ + "https://ct.example.com/log": 42, + "https://ct.other.com/log": 999, + } + data, _ := json.Marshal(want) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + m.LoadCTIndex(path) + + for url, expectedIdx := range want { + got := m.GetCTIndex(url) + if got != expectedIdx { + t.Errorf("url %q: expected index %d, got %d", url, expectedIdx, got) + } + } +} + +func TestLoadCTIndex_LoadsEmptyJSONFile(t *testing.T) { + m := &LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} + path := filepath.Join(t.TempDir(), "ct_index.json") + + if err := os.WriteFile(path, []byte("{}"), 0o644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + m.LoadCTIndex(path) + + if len(m.index) != 0 { + t.Fatalf("expected empty index, got %d entries", len(m.index)) + } +} + +func TestLoadCTIndex_LoadsEmptyFile(t *testing.T) { + m := &LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} + path := filepath.Join(t.TempDir(), "ct_index.json") + + if err := os.WriteFile(path, []byte(""), 0o644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + defer func() { + if r := recover(); r == nil { + t.Errorf("expected panic, got none") + } + }() + + m.LoadCTIndex(path) +} + +func TestLoadCTIndex_OverwritesPreexistingIndex(t *testing.T) { + m := &LogMetrics{ + metrics: make(CTMetrics), + index: CTCertIndex{"https://old.example.com/log": 1}, + } + path := filepath.Join(t.TempDir(), "ct_index.json") + + want := CTCertIndex{"https://new.example.com/log": 77} + data, _ := json.Marshal(want) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + m.LoadCTIndex(path) + + if got := m.GetCTIndex("https://new.example.com/log"); got != 77 { + t.Errorf("expected new entry with index 77, got %d", got) + } + if got := m.GetCTIndex("https://old.example.com/log"); got != 0 { + t.Errorf("expected old entry to be gone (0), got %d", got) + } +} + +func TestLoadCTIndex_PanicsOnInvalidJSON(t *testing.T) { + m := &LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} + path := filepath.Join(t.TempDir(), "ct_index.json") + + if err := os.WriteFile(path, []byte("not valid json"), 0o644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for invalid JSON, but did not panic") + } + }() + + m.LoadCTIndex(path) +} + +func TestLoadCTIndex_PanicsWhenDirectoryMissing(t *testing.T) { + m := &LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} + path := filepath.Join(t.TempDir(), "nonexistent", "ct_index.json") + + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic when index file directory is missing, but did not panic") + } + }() + + m.LoadCTIndex(path) +} From 80fcbbeb938ff751c7fb34a9318369c364e05c84 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 11 May 2026 02:34:00 +0200 Subject: [PATCH 09/75] fix(logmetrics): prevent nil pointer in m.index --- internal/metrics/logmetrics.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/metrics/logmetrics.go b/internal/metrics/logmetrics.go index 0204fe8..eb0e33c 100644 --- a/internal/metrics/logmetrics.go +++ b/internal/metrics/logmetrics.go @@ -198,6 +198,13 @@ func (m *LogMetrics) LoadCTIndex(ctIndexFilePath string) { } } + m.index = make(CTCertIndex) + + if len(bytes) == 0 || string(bytes) == "" { + log.Panicln("CT index file is empty!") + return + } + jerr := json.Unmarshal(bytes, &m.index) if jerr != nil { log.Printf("Error unmarshalling CT index file: '%s'\n", ctIndexFilePath) From 30c441f65405dbf259b53fa7cc02636e5e5cb35d Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 11 May 2026 02:34:24 +0200 Subject: [PATCH 10/75] refactor: merge multiple log lines into one --- internal/metrics/logmetrics.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/metrics/logmetrics.go b/internal/metrics/logmetrics.go index eb0e33c..4f45102 100644 --- a/internal/metrics/logmetrics.go +++ b/internal/metrics/logmetrics.go @@ -215,8 +215,7 @@ func (m *LogMetrics) LoadCTIndex(ctIndexFilePath string) { } func (m *LogMetrics) createCTIndexFile(ctIndexFilePath string) error { - log.Printf("Specified CT index file does not exist: '%s'\n", ctIndexFilePath) - log.Println("Creating CT index file now!") + log.Printf("Specified CT index file does not exist, creating now: '%s'\n", ctIndexFilePath) file, createErr := os.Create(ctIndexFilePath) if createErr != nil { From 3ec925127c2aba6635693a7e45c1d95885aeec0c Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 12 May 2026 23:57:57 +0200 Subject: [PATCH 11/75] refactor: update batch size from 100 to 256 for rfc6962 logs references #97 Despite having worked fine on my machine, the report stated that they had issues downloading all certificates in time. This value might become configurable in the future. --- internal/certificatetransparency/ct-watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index d3bb313..765ae75 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -445,7 +445,7 @@ func (w *worker) runStandardWorker(ctx context.Context) error { certScanner := scanner.NewScanner(jsonClient, scanner.ScannerOptions{ FetcherOptions: scanner.FetcherOptions{ - BatchSize: 100, + BatchSize: 256, ParallelFetch: 1, StartIndex: int64(w.ctIndex), Continuous: true, From 36f197eaedfe24610b13d3376141677542be1025 Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 12 May 2026 23:58:27 +0200 Subject: [PATCH 12/75] refactor: add logging for unregistering clients --- internal/web/broadcastmanager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/web/broadcastmanager.go b/internal/web/broadcastmanager.go index 7ddadae..5c092d9 100644 --- a/internal/web/broadcastmanager.go +++ b/internal/web/broadcastmanager.go @@ -37,6 +37,7 @@ func (bm *BroadcastManager) registerClient(c *client) { // The client will no longer receive certificate broadcasts right after unregistering. func (bm *BroadcastManager) unregisterClient(c *client) { bm.clientLock.Lock() + log.Println("Unregistering client:", targetClient.conn.RemoteAddr()) // Close the broadcast channel of the client, otherwise this leads to a memory leak close(c.broadcastChan) From ffaaedbf9823f272943691c37d1eff07dab98cbb Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 12 May 2026 23:58:56 +0200 Subject: [PATCH 13/75] refactor: reformat code --- internal/certificatetransparency/ct-tiled.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/certificatetransparency/ct-tiled.go b/internal/certificatetransparency/ct-tiled.go index 52f3dca..a756b21 100644 --- a/internal/certificatetransparency/ct-tiled.go +++ b/internal/certificatetransparency/ct-tiled.go @@ -224,6 +224,7 @@ func ConvertTileLeafToRawLogEntry(leaf TileLeaf, index uint64) *ct.RawLogEntry { return rawEntry } + type StaticCTClient struct { url string httpClient *http.Client @@ -242,8 +243,8 @@ func NewStaticCTClient(url string, httpClient *http.Client, userAgent string, st Factor: 1.3, Jitter: true, }, - userAgent: userAgent, - ctIndex: startIndex, + userAgent: userAgent, + ctIndex: startIndex, } } From 3c074932d8801f9c01091ab8b8d2876b9015bdc4 Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 00:03:10 +0200 Subject: [PATCH 14/75] test: handle errors in test cases properly --- internal/metrics/logmetrics_test.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/metrics/logmetrics_test.go b/internal/metrics/logmetrics_test.go index 984e7f4..53a5b46 100644 --- a/internal/metrics/logmetrics_test.go +++ b/internal/metrics/logmetrics_test.go @@ -88,7 +88,10 @@ func TestCreateCTIndexFile_ReturnsErrorForInvalidPath(t *testing.T) { func TestLoadCTIndex_DoesNotDeadlockWhenFileMissing(t *testing.T) { metrics := LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} ctIndexPath := filepath.Join(t.TempDir(), "ct_index.json") - os.WriteFile(ctIndexPath, []byte("{}"), 0644) + writeErr := os.WriteFile(ctIndexPath, []byte("{}"), 0644) + if writeErr != nil { + t.Fatalf("failed to write test file: %v", writeErr) + } done := make(chan struct{}) go func() { @@ -127,7 +130,10 @@ func TestLoadCTIndex_LoadsValidFile(t *testing.T) { "https://ct.example.com/log": 42, "https://ct.other.com/log": 999, } - data, _ := json.Marshal(want) + data, marshalErr := json.Marshal(want) + if marshalErr != nil { + t.Fatalf("failed to marshal test data: %v", marshalErr) + } if err := os.WriteFile(path, data, 0o644); err != nil { t.Fatalf("failed to write test file: %v", err) } @@ -182,7 +188,11 @@ func TestLoadCTIndex_OverwritesPreexistingIndex(t *testing.T) { path := filepath.Join(t.TempDir(), "ct_index.json") want := CTCertIndex{"https://new.example.com/log": 77} - data, _ := json.Marshal(want) + data, marshalErr := json.Marshal(want) + if marshalErr != nil { + t.Fatalf("failed to marshal test data: %v", marshalErr) + } + if err := os.WriteFile(path, data, 0o644); err != nil { t.Fatalf("failed to write test file: %v", err) } From 96ace02725dd10c6910bf16ba0856c4cb75641a9 Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 00:03:28 +0200 Subject: [PATCH 15/75] docs: add changes to changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37e858f..71a28ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated http client settings to prevent timeouts and other connectivity issues - Updated http server settings to allow for higher delays - Minor code improvements and refactoring, mostly style related +- Updated batch size from 100 to 256 ### Removed ### Fixed +- Calculate correct starting tile index for static ct logs - Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client +- Respect ct_index path in config file for the `create-index` command ### Docs ## [1.9.0] - 2026-04-03 From 27bb5c65aab6290c750267bfcfcbdde5ed2f72ad Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 00:06:35 +0200 Subject: [PATCH 16/75] refactor: use proper variable name --- internal/web/broadcastmanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/web/broadcastmanager.go b/internal/web/broadcastmanager.go index 5c092d9..e2e4350 100644 --- a/internal/web/broadcastmanager.go +++ b/internal/web/broadcastmanager.go @@ -37,7 +37,7 @@ func (bm *BroadcastManager) registerClient(c *client) { // The client will no longer receive certificate broadcasts right after unregistering. func (bm *BroadcastManager) unregisterClient(c *client) { bm.clientLock.Lock() - log.Println("Unregistering client:", targetClient.conn.RemoteAddr()) + log.Println("Unregistering client:", c.conn.RemoteAddr()) // Close the broadcast channel of the client, otherwise this leads to a memory leak close(c.broadcastChan) From e53d4689536262ee5e00009673c64f06e12312cf Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 02:13:55 +0200 Subject: [PATCH 17/75] refactor: reformat code and use slices package --- internal/certificatetransparency/ct-tiled.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/certificatetransparency/ct-tiled.go b/internal/certificatetransparency/ct-tiled.go index 0c285ae..2da69fb 100644 --- a/internal/certificatetransparency/ct-tiled.go +++ b/internal/certificatetransparency/ct-tiled.go @@ -7,6 +7,7 @@ import ( "io" "log" "net/http" + "slices" "strconv" "strings" "time" @@ -36,7 +37,7 @@ type TileLeaf struct { } var ( - EntryTypeCert uint16 = 0 + EntryTypeCert uint16 EntryTypePrecert uint16 = 1 ) @@ -54,7 +55,7 @@ func encodeTilePath(index uint64) string { // Build path from groups in reverse var builder strings.Builder - for i := len(groups) - 1; i >= 0; i-- { + for i, v := range slices.Backward(groups) { if i < len(groups)-1 { builder.WriteByte('/') } @@ -63,7 +64,7 @@ func encodeTilePath(index uint64) string { builder.WriteByte('x') } - fmt.Fprintf(&builder, "%03d", groups[i]) + fmt.Fprintf(&builder, "%03d", v) } return builder.String() From 8e7c090442a46a3bfcb0fe0ea9fb2b1151a91eef Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 02:14:26 +0200 Subject: [PATCH 18/75] test(ct-tiled): add test for encodeTilePath --- .../certificatetransparency/ct-tiled_test.go | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 internal/certificatetransparency/ct-tiled_test.go diff --git a/internal/certificatetransparency/ct-tiled_test.go b/internal/certificatetransparency/ct-tiled_test.go new file mode 100644 index 0000000..1fcf45b --- /dev/null +++ b/internal/certificatetransparency/ct-tiled_test.go @@ -0,0 +1,70 @@ +package certificatetransparency + +import ( + "fmt" + "testing" +) + +func Test_encodeTilePath(t *testing.T) { + tests := []struct { + name string + index uint64 + want string + }{ + { + name: "zero index", + index: 0, + want: "000", + }, + { + name: "single group padded", + index: 1, + want: "001", + }, + { + name: "single group max", + index: 999, + want: "999", + }, + { + name: "example from spec", + index: 1_234_067, + want: "x001/x234/067", + }, + { + name: "two groups exact thousand", + index: 1_000, + want: "x001/000", + }, + { + name: "two groups with remainder", + index: 1_001, + want: "x001/001", + }, + { + name: "two groups arbitrary", + index: 123_456, + want: "x123/456", + }, + { + name: "three groups", + index: 1_000_000, + want: "x001/x000/000", + }, + { + name: "four groups", + index: 12_123_456_789, + want: "x012/x123/x456/789", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := encodeTilePath(tt.index) + fmt.Println(got, tt.want) + if got != tt.want { + t.Errorf("encodeTilePath() = %v, want %v", got, tt.want) + } + }) + } +} From cbbd438ec1c0755201d770c97bf329489ef26def Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 02:46:59 +0200 Subject: [PATCH 19/75] fix: properly handle uint64-int64 conversions Handle gosec G115 warnings. --- internal/certificatetransparency/ct-parser.go | 5 +++++ internal/certificatetransparency/ct-tiled.go | 7 +++++++ internal/certificatetransparency/ct-watcher.go | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/internal/certificatetransparency/ct-parser.go b/internal/certificatetransparency/ct-parser.go index 2bed1fa..02bd580 100644 --- a/internal/certificatetransparency/ct-parser.go +++ b/internal/certificatetransparency/ct-parser.go @@ -29,6 +29,11 @@ func parseData(entry *ct.RawLogEntry, operatorName, logName, ctURL, logType stri certLink = fmt.Sprintf("%s/ct/v1/get-entries?start=%d&end=%d", ctURL, entry.Index, entry.Index) } + // Handle gosec G115 warning + if entry.Index < 0 { + return models.Data{}, fmt.Errorf("invalid cert index: %d", entry.Index) + } + // Create main data structure data := models.Data{ CertIndex: uint64(entry.Index), diff --git a/internal/certificatetransparency/ct-tiled.go b/internal/certificatetransparency/ct-tiled.go index 2da69fb..6b2dfdd 100644 --- a/internal/certificatetransparency/ct-tiled.go +++ b/internal/certificatetransparency/ct-tiled.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log" + "math" "net/http" "slices" "strconv" @@ -189,6 +190,11 @@ func ParseTileData(data []byte) ([]TileLeaf, error) { // ConvertTileLeafToRawLogEntry converts a TileLeaf to ct.RawLogEntry for compatibility. func ConvertTileLeafToRawLogEntry(leaf TileLeaf, index uint64) *ct.RawLogEntry { + if index > math.MaxInt64 { + log.Printf("index (%d) exceeds math.MaxInt64, skipping\n", index) + return nil + } + rawEntry := &ct.RawLogEntry{ Index: int64(index), Leaf: ct.MerkleTreeLeaf{ @@ -334,6 +340,7 @@ func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert fun // First time we see this partial tile – start the deferral clock. s.partialTileIndex = endTile s.partialTileFirstSeen = time.Now() + log.Println("Deferring fetch of partial tile", endTile, "with size", partialSize) case time.Since(s.partialTileFirstSeen) >= s.maxPartialWait: diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index 765ae75..61833d6 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log" + "math" "net/http" "path/filepath" "strings" @@ -310,6 +311,7 @@ func (w *Watcher) CreateIndexFile(filePath string) error { log.Println("Fetching checkpoint for", normalizedURL) staticCTClient := NewStaticCTClient(transparencyLog.MonitoringURL, httpClient, UserAgent, 0) + checkpoint, fetchErr := staticCTClient.FetchCheckpoint(w.context) if fetchErr != nil { log.Printf("Could not get checkpoint for '%s': %s\n", transparencyLog.MonitoringURL, fetchErr) @@ -443,6 +445,12 @@ func (w *worker) runStandardWorker(ctx context.Context) error { w.ctIndex = sth.TreeSize } + // Handle gosec G115 warning + if w.ctIndex > math.MaxInt64 { + log.Printf("index (%d) exceeds math.MaxInt64, skipping\n", w.ctIndex) + return nil + } + certScanner := scanner.NewScanner(jsonClient, scanner.ScannerOptions{ FetcherOptions: scanner.FetcherOptions{ BatchSize: 256, From 5984ab90183d5b30269b2cac4e2a5b9410ac28df Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 02:47:20 +0200 Subject: [PATCH 20/75] refactor(ct-watcher): wrap error --- internal/certificatetransparency/ct-watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index 61833d6..0583ff1 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -326,7 +326,7 @@ func (w *Watcher) CreateIndexFile(filePath string) error { saveErr := metrics.Metrics.SaveCertIndexes(filePath) if saveErr != nil { - return saveErr + return fmt.Errorf("failed to save cert index: %w", saveErr) } log.Println("Index file saved to", filePath) From df0ce9e288160ec49aae5715d0573e35305e03dc Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 02:48:33 +0200 Subject: [PATCH 21/75] refactor: decrease code nesting by reversing condition --- internal/web/client.go | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/internal/web/client.go b/internal/web/client.go index 8e1c986..6626c2a 100644 --- a/internal/web/client.go +++ b/internal/web/client.go @@ -146,29 +146,31 @@ func (c *client) listenWebsocket() { for { // ignore any message sent from clients - we only handle errors (aka. disconnects) _, _, readErr := c.conn.ReadMessage() - if readErr != nil { - if websocket.IsUnexpectedCloseError(readErr, websocket.CloseGoingAway, websocket.CloseNormalClosure) { - log.Printf("Unexpected websocket close error: %v\n", readErr) - } + if readErr == nil { + continue + } + + if websocket.IsUnexpectedCloseError(readErr, websocket.CloseGoingAway, websocket.CloseNormalClosure) { + log.Printf("Unexpected websocket close error: %v\n", readErr) + } - // If client fails to send ping messages - if strings.Contains(strings.ToLower(readErr.Error()), "i/o timeout") { - log.Printf("No ping received from client: %s\n", c.Name()) //nolint:gosec + // If client fails to send ping messages + if strings.Contains(strings.ToLower(readErr.Error()), "i/o timeout") { + log.Printf("No ping received from client: %s\n", c.Name()) //nolint:gosec - closeMessage := websocket.FormatCloseMessage(websocket.ClosePolicyViolation, "No ping received!") + closeMessage := websocket.FormatCloseMessage(websocket.ClosePolicyViolation, "No ping received!") - writeErr := c.conn.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(5*time.Second)) - if writeErr != nil { - log.Printf("Error while sending close message: %v\n", writeErr) - } - } else if strings.Contains(strings.ToLower(readErr.Error()), "an existing connection was forcibly closed by the remote host") { - log.Printf("Connection to client lost: %s\n", c.Name()) //nolint:gosec + writeErr := c.conn.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(5*time.Second)) + if writeErr != nil { + log.Printf("Error while sending close message: %v\n", writeErr) } + } else if strings.Contains(strings.ToLower(readErr.Error()), "an existing connection was forcibly closed by the remote host") { + log.Printf("Connection to client lost: %s\n", c.Name()) //nolint:gosec + } - log.Printf("Disconnecting client %s!\n", c.Name()) //nolint:gosec + log.Printf("Disconnecting client %s!\n", c.Name()) //nolint:gosec - break - } + break } } From aac6364b33b859097d366aa79b61188cc7e28084 Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 02:48:44 +0200 Subject: [PATCH 22/75] refactor: reformat test code --- internal/metrics/logmetrics_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/metrics/logmetrics_test.go b/internal/metrics/logmetrics_test.go index 53a5b46..3de01c1 100644 --- a/internal/metrics/logmetrics_test.go +++ b/internal/metrics/logmetrics_test.go @@ -88,7 +88,7 @@ func TestCreateCTIndexFile_ReturnsErrorForInvalidPath(t *testing.T) { func TestLoadCTIndex_DoesNotDeadlockWhenFileMissing(t *testing.T) { metrics := LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} ctIndexPath := filepath.Join(t.TempDir(), "ct_index.json") - writeErr := os.WriteFile(ctIndexPath, []byte("{}"), 0644) + writeErr := os.WriteFile(ctIndexPath, []byte("{}"), 0o644) if writeErr != nil { t.Fatalf("failed to write test file: %v", writeErr) } From 3d94136f77940c9fadb3795f3d38ee851e5eb9ed Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 02:54:03 +0200 Subject: [PATCH 23/75] refactor: disable tile fetching logging --- internal/certificatetransparency/ct-tiled.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/certificatetransparency/ct-tiled.go b/internal/certificatetransparency/ct-tiled.go index 6b2dfdd..15e972c 100644 --- a/internal/certificatetransparency/ct-tiled.go +++ b/internal/certificatetransparency/ct-tiled.go @@ -341,12 +341,12 @@ func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert fun s.partialTileIndex = endTile s.partialTileFirstSeen = time.Now() - log.Println("Deferring fetch of partial tile", endTile, "with size", partialSize) + // log.Println("Deferring fetch of partial tile", endTile, "with size", partialSize) case time.Since(s.partialTileFirstSeen) >= s.maxPartialWait: // The partial tile has been pending too long – fetch it now to prevent // extreme processing delays on slow-growing logs. - log.Println("Forcefully fetching partial tile", endTile, "with size", partialSize) + // log.Println("Forcefully fetching partial tile", endTile, "with size", partialSize) if err := s.processTile(ctx, endTile, partialSize, foundCert, foundPrecert); err != nil { log.Printf("Warning: error processing partial tile %d: %s\n", endTile, err) From 258c28d51281dbeb5855d38a6cbbf57f8089cc79 Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 02:54:22 +0200 Subject: [PATCH 24/75] chore: update golangci config --- .golangci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 6f5e17f..c58405f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -36,7 +36,7 @@ linters: - godot # - godox - gomoddirectives - - gomodguard + - gomodguard_v2 - goprintffuncname - gosec - govet @@ -109,6 +109,7 @@ linters: - err error - ctx context.Context - v *viper.Viper + - v uint64 - c *client - cs *Certstream - k x509.KeyUsage @@ -136,6 +137,7 @@ linters: - varnamelen - goconst - err113 + - gosec path: _test\.go - linters: - gochecknoinits From 15fcfcc74bc9d798ae5a7f70fae277eaefc41595 Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 03:22:42 +0200 Subject: [PATCH 25/75] build: update goreleaser config to properly handle dev branch builds --- .github/goreleaser.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/goreleaser.yml b/.github/goreleaser.yml index 175e5fd..56cfae5 100644 --- a/.github/goreleaser.yml +++ b/.github/goreleaser.yml @@ -41,8 +41,8 @@ dockers_v2: - "ghcr.io/d-rickyy-b/{{.ProjectName}}" dockerfile: docker/Dockerfile.goreleaser tags: - - "{{.Tag}}" - - "{{ if not .Prerelease }}latest{{ end }}" + - '{{ if eq .Branch "dev" }}dev{{ else }}{{ .Tag }}{{ end }}' + - '{{ if and (ne .Branch "dev") (not .Prerelease) }}latest{{ end }}' platforms: - linux/amd64 - linux/arm64 From fa78b6befc0a39d385c53c7ed2d0589d1b7580f0 Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 13 May 2026 03:32:31 +0200 Subject: [PATCH 26/75] build: add own dev runner config to github action release_build --- .github/workflows/release_build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml index eaa35b3..1f78c5e 100644 --- a/.github/workflows/release_build.yml +++ b/.github/workflows/release_build.yml @@ -47,9 +47,19 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Run GoReleaser + if: startsWith(github.ref, 'refs/tags/') uses: goreleaser/goreleaser-action@v7 with: version: latest args: release --clean --config .github/goreleaser.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run GoReleaser (dev snapshot) + if: github.ref == 'refs/heads/dev' + uses: goreleaser/goreleaser-action@v7 + with: + version: dev + args: release --snapshot --clean --config .github/goreleaser.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f6219d9f4dfbc55279df2ee0b75c08e6d9ad15ab Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 14 May 2026 01:35:32 +0200 Subject: [PATCH 27/75] build: revert goreleaser to version "latest" Thought this was the build tag. --- .github/workflows/release_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml index 1f78c5e..b4cc7bb 100644 --- a/.github/workflows/release_build.yml +++ b/.github/workflows/release_build.yml @@ -59,7 +59,7 @@ jobs: if: github.ref == 'refs/heads/dev' uses: goreleaser/goreleaser-action@v7 with: - version: dev + version: latest args: release --snapshot --clean --config .github/goreleaser.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 973d48bc73c93efb53036fe0a1539498b6037987 Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 30 Jun 2026 23:19:33 +0200 Subject: [PATCH 28/75] chore: add task for building arm binary --- taskfile.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/taskfile.yml b/taskfile.yml index c52644a..3fb3fc3 100644 --- a/taskfile.yml +++ b/taskfile.yml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://taskfile.dev/schema.json -version: '3' +version: "3" tasks: docker: @@ -20,6 +20,14 @@ tasks: cmds: - go build ./cmd/certstream-server-go + build-arm: + desc: Build the server binary for ARM architecture. + cmds: + - go build ./cmd/certstream-server-go + env: + GOARCH: arm64 + GOOS: linux + lint: desc: Run linters. cmds: From e3962de60fb03690ab1dcf7118e203f72cf50e83 Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 5 Jul 2026 16:45:38 +0200 Subject: [PATCH 29/75] docs: update changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71a28ff..cfc7136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,18 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Option to configure trusted_proxies for which the X-Forwarded-For and X-Real-IP headers are used as client IP (b062edca) - Added multiple unit tests for ensuring correct functionality + ### Changed - Updated weak cipher suites to stronger ones (939517cd) - Updated http client settings to prevent timeouts and other connectivity issues - Updated http server settings to allow for higher delays - Minor code improvements and refactoring, mostly style related -- Updated batch size from 100 to 256 +- Updated batch size from 100 to 256 (#97) + ### Removed + ### Fixed - Calculate correct starting tile index for static ct logs - Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client - Respect ct_index path in config file for the `create-index` command + ### Docs ## [1.9.0] - 2026-04-03 From c8bd735ad3c84be47d819a8fd1669107116cb3ff Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 5 Jul 2026 18:36:38 +0200 Subject: [PATCH 30/75] fix: ensure all additionalLogs entries are processed correctly The previous logic skipped processing of additional logs altogether if there were duplicates in the list. --- CHANGELOG.md | 1 + .../certificatetransparency/ct-watcher.go | 26 +++-- .../ct-watcher_test.go | 102 +++++++++--------- 3 files changed, 68 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfc7136..e9b639a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Calculate correct starting tile index for static ct logs - Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client - Respect ct_index path in config file for the `create-index` command +- Ensure all additionalLog entries are processed correctly ### Docs diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index 0583ff1..c3a0b68 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -120,7 +120,7 @@ func (w *Watcher) updateLogs() { log.Println("Checking for new ct logs...") - // Track all URLs that should be monitored after reconciliation + // Track all URLs that should be monitored after reconciliation. monitoredURLs := make(map[string]struct{}) newCTs := 0 @@ -621,8 +621,7 @@ func getAllLogs(logListFetcher LogListFetcher) (loglist3.LogList, error) { } } -logFound: - // + // Add additional logs provided via the config, if any. for _, additionalLog := range config.AppConfig.General.AdditionalLogs { customLog := loglist3.Log{ URL: additionalLog.URL, @@ -638,17 +637,21 @@ logFound: } operatorFound = true + logFound := false // Check if user provided log is already in our loglist for _, ctlog := range operator.Logs { if ctlog.URL == additionalLog.URL { // Log already exists, skip it. - break logFound + logFound = true + break } } - // This works, since allLogs.Operators is a slice of pointers. - operator.Logs = append(operator.Logs, &customLog) + if !logFound { + // This works, since allLogs.Operators is a slice of pointers. + operator.Logs = append(operator.Logs, &customLog) + } break } @@ -670,20 +673,23 @@ logFound: operatorFound := false - tiledLogFound: for _, operator := range allLogs.Operators { if operator.Name == additionalLog.Operator { operatorFound = true + logFound := false for _, tl := range operator.TiledLogs { if tl.MonitoringURL == additionalLog.URL { // Log already exists, skip it. - break tiledLogFound + logFound = true + break } } - // This works, since allLogs.Operators is a slice of pointers. - operator.TiledLogs = append(operator.TiledLogs, &customLog) + if !logFound { + // This works, since allLogs.Operators is a slice of pointers. + operator.TiledLogs = append(operator.TiledLogs, &customLog) + } break } diff --git a/internal/certificatetransparency/ct-watcher_test.go b/internal/certificatetransparency/ct-watcher_test.go index 2a990e7..a43652c 100644 --- a/internal/certificatetransparency/ct-watcher_test.go +++ b/internal/certificatetransparency/ct-watcher_test.go @@ -190,9 +190,9 @@ func TestGetAllLogs_FetcherError(t *testing.T) { // --- Tests for additional classic logs --- -// TestGetAllLogs_AdditionalLog_NewOperator verifies that an additional log whose operator +// TestGetAllLogs_AdditionalLogs_NewOperator verifies that an additional log whose operator // does not yet exist in the list creates a new operator entry. -func TestGetAllLogs_AdditionalLog_NewOperator(t *testing.T) { +func TestGetAllLogs_AdditionalLogs_NewOperator(t *testing.T) { t.Cleanup(func() { config.AppConfig = config.Config{} }) @@ -221,9 +221,9 @@ func TestGetAllLogs_AdditionalLog_NewOperator(t *testing.T) { } } -// TestGetAllLogs_AdditionalLog_ExistingOperator verifies that an additional log is appended +// TestGetAllLogs_AdditionalLogs_ExistingOperator verifies that an additional log is appended // to an already-existing operator. -func TestGetAllLogs_AdditionalLog_ExistingOperator(t *testing.T) { +func TestGetAllLogs_AdditionalLogs_ExistingOperator(t *testing.T) { t.Cleanup(func() { config.AppConfig = config.Config{} }) @@ -260,45 +260,43 @@ func TestGetAllLogs_AdditionalLog_ExistingOperator(t *testing.T) { } } -// TestGetAllLogs_AdditionalLog_NoDuplicate verifies that adding a log that already exists -// (same URL) does not create a duplicate entry. -func TestGetAllLogs_AdditionalLog_NoDuplicate(t *testing.T) { +// TestGetAllLogs_AdditionalLogs_Multiple verifies that multiple additional logs across +// different operators are all added correctly. +func TestGetAllLogs_AdditionalLogs_Multiple(t *testing.T) { t.Cleanup(func() { config.AppConfig = config.Config{} }) - existingURL := "ct.googleapis.com/logs/xenon2024/" - mockList := buildLogList([]struct { - name string - logs []string - tiledLogs []string - }{ - {"Google", []string{existingURL}, nil}, - }) - - config.AppConfig.General.DisableDefaultLogs = false + config.AppConfig.General.DisableDefaultLogs = true config.AppConfig.General.AdditionalLogs = []config.LogConfig{ - {Operator: "Google", URL: existingURL, Description: "Duplicate"}, + {Operator: "OperatorA", URL: "loga.example.com/log1/", Description: "Log A1"}, + {Operator: "OperatorA", URL: "loga.example.com/log2/", Description: "Log A2"}, + {Operator: "OperatorB", URL: "logb.example.com/log1/", Description: "Log B1"}, } - result, err := getAllLogs(newMockListFetcher(t, mockList, nil)) + result, err := getAllLogs(newMockListFetcher(t, emptyLogList, nil)) if err != nil { t.Fatalf("unexpected error: %v", err) } - op := findOperator(result, "Google") - if op == nil { - t.Fatal("expected operator 'Google' to be present") + if countLogs(result) != 3 { + t.Errorf("expected 3 logs total, got %d", countLogs(result)) } - if len(op.Logs) != 1 { - t.Errorf("expected 1 log (no duplicate), got %d", len(op.Logs)) + opA := findOperator(result, "OperatorA") + if opA == nil || len(opA.Logs) != 2 { + t.Errorf("expected 2 logs for 'OperatorA', got %v", opA) + } + + opB := findOperator(result, "OperatorB") + if opB == nil || len(opB.Logs) != 1 { + t.Errorf("expected 1 log for 'OperatorB', got %v", opB) } } -// TestGetAllLogs_MultipleAdditionalLogs verifies that multiple additional logs across -// different operators are all added correctly. -func TestGetAllLogs_MultipleAdditionalLogs(t *testing.T) { +// TestGetAllLogs_AdditionalLogs_Duplicates verifies that multiple additional logs across +// different operators are all added correctly, even if there are duplicates. +func TestGetAllLogs_AdditionalLogs_Duplicates(t *testing.T) { t.Cleanup(func() { config.AppConfig = config.Config{} }) @@ -307,6 +305,8 @@ func TestGetAllLogs_MultipleAdditionalLogs(t *testing.T) { config.AppConfig.General.AdditionalLogs = []config.LogConfig{ {Operator: "OperatorA", URL: "loga.example.com/log1/", Description: "Log A1"}, {Operator: "OperatorA", URL: "loga.example.com/log2/", Description: "Log A2"}, + {Operator: "OperatorA", URL: "loga.example.com/log2/", Description: "Log A2"}, + {Operator: "OperatorB", URL: "logb.example.com/log1/", Description: "Log B1"}, {Operator: "OperatorB", URL: "logb.example.com/log1/", Description: "Log B1"}, } @@ -332,9 +332,9 @@ func TestGetAllLogs_MultipleAdditionalLogs(t *testing.T) { // --- Tests for additional tiled logs --- -// TestGetAllLogs_AdditionalTiledLog_NewOperator verifies that an additional tiled log +// TestGetAllLogs_AdditionalTiledLogs_NewOperator verifies that an additional tiled log // whose operator does not yet exist creates a new operator entry. -func TestGetAllLogs_AdditionalTiledLog_NewOperator(t *testing.T) { +func TestGetAllLogs_AdditionalTiledLogs_NewOperator(t *testing.T) { t.Cleanup(func() { config.AppConfig = config.Config{} }) @@ -363,9 +363,9 @@ func TestGetAllLogs_AdditionalTiledLog_NewOperator(t *testing.T) { } } -// TestGetAllLogs_AdditionalTiledLog_ExistingOperator verifies that an additional tiled log +// TestGetAllLogs_AdditionalTiledLogs_ExistingOperator verifies that an additional tiled log // is appended to an already-existing operator. -func TestGetAllLogs_AdditionalTiledLog_ExistingOperator(t *testing.T) { +func TestGetAllLogs_AdditionalTiledLogs_ExistingOperator(t *testing.T) { t.Cleanup(func() { config.AppConfig = config.Config{} }) @@ -402,39 +402,39 @@ func TestGetAllLogs_AdditionalTiledLog_ExistingOperator(t *testing.T) { } } -// TestGetAllLogs_AdditionalTiledLog_NoDuplicate verifies that adding a tiled log that -// already exists does not create a duplicate entry. -func TestGetAllLogs_AdditionalTiledLog_NoDuplicate(t *testing.T) { +// TestGetAllLogs_AdditionalLogs_Duplicates verifies that multiple additional logs across +// different operators are all added correctly, even if there are duplicates. +func TestGetAllLogs_AdditionalTiledLogs_Duplicates(t *testing.T) { t.Cleanup(func() { config.AppConfig = config.Config{} }) - existingURL := "tiled.googleapis.com/logs/existing/" - mockList := buildLogList([]struct { - name string - logs []string - tiledLogs []string - }{ - {"Google", nil, []string{existingURL}}, - }) - - config.AppConfig.General.DisableDefaultLogs = false + config.AppConfig.General.DisableDefaultLogs = true config.AppConfig.General.AdditionalTiledLogs = []config.LogConfig{ - {Operator: "Google", URL: existingURL, Description: "Duplicate Tiled"}, + {Operator: "OperatorA", URL: "loga.example.com/log1/", Description: "Log A1"}, + {Operator: "OperatorA", URL: "loga.example.com/log2/", Description: "Log A2"}, + {Operator: "OperatorA", URL: "loga.example.com/log2/", Description: "Log A2"}, + {Operator: "OperatorB", URL: "logb.example.com/log1/", Description: "Log B1"}, + {Operator: "OperatorB", URL: "logb.example.com/log1/", Description: "Log B1"}, } - result, err := getAllLogs(newMockListFetcher(t, mockList, nil)) + result, err := getAllLogs(newMockListFetcher(t, emptyLogList, nil)) if err != nil { t.Fatalf("unexpected error: %v", err) } - op := findOperator(result, "Google") - if op == nil { - t.Fatal("expected operator 'Google' to be present") + if countTiledLogs(result) != 3 { + t.Errorf("expected 3 logs total, got %d", countTiledLogs(result)) } - if len(op.TiledLogs) != 1 { - t.Errorf("expected 1 tiled log (no duplicate), got %d", len(op.TiledLogs)) + opA := findOperator(result, "OperatorA") + if opA == nil || len(opA.TiledLogs) != 2 { + t.Errorf("expected 2 logs for 'OperatorA', got %v", opA) + } + + opB := findOperator(result, "OperatorB") + if opB == nil || len(opB.TiledLogs) != 1 { + t.Errorf("expected 1 log for 'OperatorB', got %v", opB) } } From 55627b793d3099648b7ecafae5eeeba240de447d Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 5 Jul 2026 18:37:12 +0200 Subject: [PATCH 31/75] test: remove print statement from unit test --- internal/certificatetransparency/ct-tiled_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/certificatetransparency/ct-tiled_test.go b/internal/certificatetransparency/ct-tiled_test.go index 1fcf45b..4c246d1 100644 --- a/internal/certificatetransparency/ct-tiled_test.go +++ b/internal/certificatetransparency/ct-tiled_test.go @@ -1,7 +1,6 @@ package certificatetransparency import ( - "fmt" "testing" ) @@ -61,7 +60,6 @@ func Test_encodeTilePath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := encodeTilePath(tt.index) - fmt.Println(got, tt.want) if got != tt.want { t.Errorf("encodeTilePath() = %v, want %v", got, tt.want) } From 092efbb730c3e38a3c9c3fd70b28833ed528acb3 Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 5 Jul 2026 18:45:00 +0200 Subject: [PATCH 32/75] refactor: improve LogListFetcher by using interface and struct Extract the mechanism for retrieving the Certificate Transparency log list into a distinct interface and HTTP-specific implementation. This improves modularity and makes the fetching process more easily testable. Using a struct combined with an interface makes the code a lot cleaner than the previous implementation, passing a function parameter. --- .../certificatetransparency/ct-loglist.go | 61 +++++++++++++++++++ .../certificatetransparency/ct-watcher.go | 47 +------------- .../ct-watcher_test.go | 15 +++-- 3 files changed, 75 insertions(+), 48 deletions(-) create mode 100644 internal/certificatetransparency/ct-loglist.go diff --git a/internal/certificatetransparency/ct-loglist.go b/internal/certificatetransparency/ct-loglist.go new file mode 100644 index 0000000..82fe9ee --- /dev/null +++ b/internal/certificatetransparency/ct-loglist.go @@ -0,0 +1,61 @@ +package certificatetransparency + +import ( + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/google/certificate-transparency-go/loglist3" +) + +var googleLogListFetcher = NewHTTPLogListFetcher(loglist3.LogListURL) + +type LogListFetcher interface { + Fetch() (loglist3.LogList, error) +} + +type HTTPLogListFetcher struct { + LogListURL string +} + +func NewHTTPLogListFetcher(url string) *HTTPLogListFetcher { + return &HTTPLogListFetcher{LogListURL: url} +} + +// Fetch fetches the list of all CT logs from the provided LogList URL. +func (f *HTTPLogListFetcher) Fetch() (loglist3.LogList, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + httpClient := newHTTPClient() + + req, newReqErr := http.NewRequestWithContext(ctx, http.MethodGet, f.LogListURL, nil) + if newReqErr != nil { + return loglist3.LogList{}, fmt.Errorf("failed to create loglist request: %w", newReqErr) + } + + // Download the list of all logs from ctLogInfo and decode JSON + resp, reqErr := httpClient.Do(req) + if reqErr != nil { + return loglist3.LogList{}, fmt.Errorf("failed to execute loglist request: %w", reqErr) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return loglist3.LogList{}, fmt.Errorf("%w: unexpected status code %d", ErrRequestFailed, resp.StatusCode) + } + + bodyBytes, readErr := io.ReadAll(resp.Body) + if readErr != nil { + return loglist3.LogList{}, fmt.Errorf("failed reading response body: %w", readErr) + } + + allLogs, parseErr := loglist3.NewFromJSON(bodyBytes) + if parseErr != nil { + return loglist3.LogList{}, fmt.Errorf("failed parsing response body: %w", parseErr) + } + + return *allLogs, nil +} diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index c3a0b68..1a3e6e9 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "log" "math" "net/http" @@ -565,46 +564,6 @@ func certHandler(entryChan chan models.Entry) { } } -// LogListFetcher defines a function type for fetching a log list. This allows us to inject different -// implementations (e.g. for testing). -type LogListFetcher func() (loglist3.LogList, error) - -// googleLogListFetcher fetches the list of all CT logs from Google Chromes CT LogList. -func googleLogListFetcher() (loglist3.LogList, error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - httpClient := newHTTPClient() - - req, newReqErr := http.NewRequestWithContext(ctx, http.MethodGet, loglist3.LogListURL, nil) - if newReqErr != nil { - return loglist3.LogList{}, fmt.Errorf("failed to create loglist request: %w", newReqErr) - } - - // Download the list of all logs from ctLogInfo and decode JSON - resp, reqErr := httpClient.Do(req) - if reqErr != nil { - return loglist3.LogList{}, fmt.Errorf("failed to execute loglist request: %w", reqErr) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return loglist3.LogList{}, fmt.Errorf("%w: unexpected status code %d", ErrRequestFailed, resp.StatusCode) - } - - bodyBytes, readErr := io.ReadAll(resp.Body) - if readErr != nil { - return loglist3.LogList{}, fmt.Errorf("failed reading response body: %w", readErr) - } - - allLogs, parseErr := loglist3.NewFromJSON(bodyBytes) - if parseErr != nil { - return loglist3.LogList{}, fmt.Errorf("failed parsing response body: %w", parseErr) - } - - return *allLogs, nil -} - // getAllLogs returns a list of all CT logs - those from the Google list, if not disabled - // and additional logs provided via the config. func getAllLogs(logListFetcher LogListFetcher) (loglist3.LogList, error) { @@ -614,10 +573,10 @@ func getAllLogs(logListFetcher LogListFetcher) (loglist3.LogList, error) { if !config.AppConfig.General.DisableDefaultLogs { var err error - allLogs, err = logListFetcher() + allLogs, err = logListFetcher.Fetch() if err != nil { - log.Printf("Error fetching log list from Google: %s\n", err) - return loglist3.LogList{}, fmt.Errorf("failed to fetch log list from Google: %w", err) + log.Printf("Error fetching log list: %s\n", err) + return loglist3.LogList{}, fmt.Errorf("failed to fetch log list: %w", err) } } diff --git a/internal/certificatetransparency/ct-watcher_test.go b/internal/certificatetransparency/ct-watcher_test.go index a43652c..b4e8bfd 100644 --- a/internal/certificatetransparency/ct-watcher_test.go +++ b/internal/certificatetransparency/ct-watcher_test.go @@ -37,13 +37,20 @@ func buildLogList(operators []struct { return ll } +type mockLogListFetcher struct { + loglist loglist3.LogList + err error +} + +func (m *mockLogListFetcher) Fetch() (loglist3.LogList, error) { + return m.loglist, m.err +} + // newMockListFetcher returns a function that returns the passed loglist. func newMockListFetcher(t *testing.T, loglist loglist3.LogList, err error) LogListFetcher { t.Helper() - return func() (loglist3.LogList, error) { - return loglist, err - } + return &mockLogListFetcher{loglist: loglist, err: err} } // countLogs returns the total number of classic (non-tiled) logs across all operators. @@ -182,7 +189,7 @@ func TestGetAllLogs_FetcherError(t *testing.T) { config.AppConfig.General.DisableDefaultLogs = false - _, err := getAllLogs(newMockListFetcher(t, emptyLogList, errors.New("network error"))) + _, err := getAllLogs(newMockListFetcher(t, emptyLogList, errors.New("mock error - can be ignored in test output"))) if err == nil { t.Fatal("expected error when fetcher fails, got nil") } From e57d3f65ddc6065f57f21955f3f15de429789e2f Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 5 Jul 2026 18:50:15 +0200 Subject: [PATCH 33/75] refactor: centralize log list logic Move CT log list management to a dedicated module. The associated test file is also renamed. --- .../certificatetransparency/ct-loglist.go | 113 ++++++++++++++++++ ...{ct-watcher_test.go => ct-loglist_test.go} | 0 .../certificatetransparency/ct-watcher.go | 110 ----------------- 3 files changed, 113 insertions(+), 110 deletions(-) rename internal/certificatetransparency/{ct-watcher_test.go => ct-loglist_test.go} (100%) diff --git a/internal/certificatetransparency/ct-loglist.go b/internal/certificatetransparency/ct-loglist.go index 82fe9ee..e962d95 100644 --- a/internal/certificatetransparency/ct-loglist.go +++ b/internal/certificatetransparency/ct-loglist.go @@ -4,9 +4,12 @@ import ( "context" "fmt" "io" + "log" "net/http" + "strings" "time" + "github.com/d-Rickyy-b/certstream-server-go/internal/config" "github.com/google/certificate-transparency-go/loglist3" ) @@ -59,3 +62,113 @@ func (f *HTTPLogListFetcher) Fetch() (loglist3.LogList, error) { return *allLogs, nil } + +// getAllLogs returns a list of all CT logs - those from the Google list, if not disabled - +// and additional logs provided via the config. +func getAllLogs(logListFetcher LogListFetcher) (loglist3.LogList, error) { + var allLogs loglist3.LogList + + // Ability to disable default logs, if the user only wants to monitor custom logs. + if !config.AppConfig.General.DisableDefaultLogs { + var err error + + allLogs, err = logListFetcher.Fetch() + if err != nil { + log.Printf("Error fetching log list: %s\n", err) + return loglist3.LogList{}, fmt.Errorf("failed to fetch log list: %w", err) + } + } + + // Add additional logs provided via the config, if any. + for _, additionalLog := range config.AppConfig.General.AdditionalLogs { + customLog := loglist3.Log{ + URL: additionalLog.URL, + Description: additionalLog.Description, + } + + operatorFound := false + + for _, operator := range allLogs.Operators { + // Only compare logs with the same operator + if operator.Name != additionalLog.Operator { + continue + } + + operatorFound = true + logFound := false + + // Check if user provided log is already in our loglist + for _, ctlog := range operator.Logs { + if ctlog.URL == additionalLog.URL { + // Log already exists, skip it. + logFound = true + break + } + } + + if !logFound { + // This works, since allLogs.Operators is a slice of pointers. + operator.Logs = append(operator.Logs, &customLog) + } + + break + } + + if !operatorFound { + newOperator := loglist3.Operator{ + Name: additionalLog.Operator, + Logs: []*loglist3.Log{&customLog}, + } + allLogs.Operators = append(allLogs.Operators, &newOperator) + } + } + + for _, additionalLog := range config.AppConfig.General.AdditionalTiledLogs { + customLog := loglist3.TiledLog{ + MonitoringURL: additionalLog.URL, + Description: additionalLog.Description, + } + + operatorFound := false + + for _, operator := range allLogs.Operators { + if operator.Name == additionalLog.Operator { + operatorFound = true + logFound := false + + for _, tl := range operator.TiledLogs { + if tl.MonitoringURL == additionalLog.URL { + // Log already exists, skip it. + logFound = true + break + } + } + + if !logFound { + // This works, since allLogs.Operators is a slice of pointers. + operator.TiledLogs = append(operator.TiledLogs, &customLog) + } + + break + } + } + + if !operatorFound { + newOperator := loglist3.Operator{ + Name: additionalLog.Operator, + TiledLogs: []*loglist3.TiledLog{&customLog}, + } + allLogs.Operators = append(allLogs.Operators, &newOperator) + } + } + + return allLogs, nil +} + +func normalizeCtlogURL(input string) string { + input = strings.TrimPrefix(input, "https://") + input = strings.TrimPrefix(input, "http://") + input = strings.TrimSuffix(input, "/") + + return input +} diff --git a/internal/certificatetransparency/ct-watcher_test.go b/internal/certificatetransparency/ct-loglist_test.go similarity index 100% rename from internal/certificatetransparency/ct-watcher_test.go rename to internal/certificatetransparency/ct-loglist_test.go diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index 1a3e6e9..b4eb458 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -564,116 +564,6 @@ func certHandler(entryChan chan models.Entry) { } } -// getAllLogs returns a list of all CT logs - those from the Google list, if not disabled - -// and additional logs provided via the config. -func getAllLogs(logListFetcher LogListFetcher) (loglist3.LogList, error) { - var allLogs loglist3.LogList - - // Ability to disable default logs, if the user only wants to monitor custom logs. - if !config.AppConfig.General.DisableDefaultLogs { - var err error - - allLogs, err = logListFetcher.Fetch() - if err != nil { - log.Printf("Error fetching log list: %s\n", err) - return loglist3.LogList{}, fmt.Errorf("failed to fetch log list: %w", err) - } - } - - // Add additional logs provided via the config, if any. - for _, additionalLog := range config.AppConfig.General.AdditionalLogs { - customLog := loglist3.Log{ - URL: additionalLog.URL, - Description: additionalLog.Description, - } - - operatorFound := false - - for _, operator := range allLogs.Operators { - // Only compare logs with the same operator - if operator.Name != additionalLog.Operator { - continue - } - - operatorFound = true - logFound := false - - // Check if user provided log is already in our loglist - for _, ctlog := range operator.Logs { - if ctlog.URL == additionalLog.URL { - // Log already exists, skip it. - logFound = true - break - } - } - - if !logFound { - // This works, since allLogs.Operators is a slice of pointers. - operator.Logs = append(operator.Logs, &customLog) - } - - break - } - - if !operatorFound { - newOperator := loglist3.Operator{ - Name: additionalLog.Operator, - Logs: []*loglist3.Log{&customLog}, - } - allLogs.Operators = append(allLogs.Operators, &newOperator) - } - } - - for _, additionalLog := range config.AppConfig.General.AdditionalTiledLogs { - customLog := loglist3.TiledLog{ - MonitoringURL: additionalLog.URL, - Description: additionalLog.Description, - } - - operatorFound := false - - for _, operator := range allLogs.Operators { - if operator.Name == additionalLog.Operator { - operatorFound = true - logFound := false - - for _, tl := range operator.TiledLogs { - if tl.MonitoringURL == additionalLog.URL { - // Log already exists, skip it. - logFound = true - break - } - } - - if !logFound { - // This works, since allLogs.Operators is a slice of pointers. - operator.TiledLogs = append(operator.TiledLogs, &customLog) - } - - break - } - } - - if !operatorFound { - newOperator := loglist3.Operator{ - Name: additionalLog.Operator, - TiledLogs: []*loglist3.TiledLog{&customLog}, - } - allLogs.Operators = append(allLogs.Operators, &newOperator) - } - } - - return allLogs, nil -} - -func normalizeCtlogURL(input string) string { - input = strings.TrimPrefix(input, "https://") - input = strings.TrimPrefix(input, "http://") - input = strings.TrimSuffix(input, "/") - - return input -} - // newHTTPClient creates a new http.Client with reasonable timeouts and connection settings for interacting with CT logs. func newHTTPClient() *http.Client { httpClient := http.Client{ From 170fbebf395eee6ff625350a9dfa077146e93221 Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 5 Jul 2026 22:26:09 +0200 Subject: [PATCH 34/75] feat: add ability to exclude log operators or specific logs fixes #109 --- .golangci.yml | 1 + CHANGELOG.md | 1 + config.docker.yaml | 7 ++ config.sample.yaml | 7 ++ .../certificatetransparency/ct-loglist.go | 92 ++++++++++++--- .../ct-loglist_test.go | 107 ++++++++++++++++++ internal/config/config.go | 23 +++- internal/config/config_test.go | 51 +++++++++ 8 files changed, 274 insertions(+), 15 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index c58405f..22d92c7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -138,6 +138,7 @@ linters: - goconst - err113 - gosec + - dupl path: _test\.go - linters: - gochecknoinits diff --git a/CHANGELOG.md b/CHANGELOG.md index e9b639a..5232c3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Option to configure trusted_proxies for which the X-Forwarded-For and X-Real-IP headers are used as client IP (b062edca) - Added multiple unit tests for ensuring correct functionality +- Add `excluded_logs` config option to skip specific logs and operators. Works both for classic and tiled logs. ### Changed - Updated weak cipher suites to stronger ones (939517cd) diff --git a/config.docker.yaml b/config.docker.yaml index efbc743..64c5903 100644 --- a/config.docker.yaml +++ b/config.docker.yaml @@ -26,6 +26,13 @@ general: # DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. disable_default_logs: false + # Exclude logs from being monitored. + # You can either exclude by operator (all logs of that operator) + # or by a specific log URL. + #excluded_logs: + # - operator: "Google" + # - url: "https://ct.googleapis.com/logs/us1/argon2024/" + # Google regularly updates the log list. If this option is set to true, the server will remove all logs no longer listed in the Google log list. # This option defaults to true. See https://github.com/d-Rickyy-b/certstream-server-go/issues/51 drop_old_logs: true diff --git a/config.sample.yaml b/config.sample.yaml index 682c8eb..74fe9a6 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -52,6 +52,13 @@ general: # operator: "Cloudflare" # description: "Cloudflare 'Raio2025h2b'" + # Exclude logs from being monitored. + # You can either exclude by operator (all logs of that operator) + # or by a specific log URL. + #excluded_logs: + # - operator: "Google" + # - url: "https://ct.googleapis.com/logs/us1/argon2024/" + # To optimize the performance of the server, you can overwrite the size of different buffers # For low CPU, low memory machines, you should reduce the buffer sizes to save memory in case the CPU is maxed. buffer_sizes: diff --git a/internal/certificatetransparency/ct-loglist.go b/internal/certificatetransparency/ct-loglist.go index e962d95..02bc253 100644 --- a/internal/certificatetransparency/ct-loglist.go +++ b/internal/certificatetransparency/ct-loglist.go @@ -6,6 +6,7 @@ import ( "io" "log" "net/http" + "slices" "strings" "time" @@ -67,6 +68,19 @@ func (f *HTTPLogListFetcher) Fetch() (loglist3.LogList, error) { // and additional logs provided via the config. func getAllLogs(logListFetcher LogListFetcher) (loglist3.LogList, error) { var allLogs loglist3.LogList + excludedOperators := []string{} + excludedLogs := []string{} + + // Extract excluded logs from the config. + for _, excludedLog := range config.AppConfig.General.ExcludedLogs { + if excludedLog.URL == "" { + // Exclude whole operator + excludedOperators = append(excludedOperators, excludedLog.Operator) + } else { + // Exclude only this specific URL (normalize for comparison) + excludedLogs = append(excludedLogs, normalizeCtlogURL(excludedLog.URL)) + } + } // Ability to disable default logs, if the user only wants to monitor custom logs. if !config.AppConfig.General.DisableDefaultLogs { @@ -132,25 +146,27 @@ func getAllLogs(logListFetcher LogListFetcher) (loglist3.LogList, error) { operatorFound := false for _, operator := range allLogs.Operators { - if operator.Name == additionalLog.Operator { - operatorFound = true - logFound := false + if operator.Name != additionalLog.Operator { + continue + } - for _, tl := range operator.TiledLogs { - if tl.MonitoringURL == additionalLog.URL { - // Log already exists, skip it. - logFound = true - break - } - } + operatorFound = true + logFound := false - if !logFound { - // This works, since allLogs.Operators is a slice of pointers. - operator.TiledLogs = append(operator.TiledLogs, &customLog) + for _, tl := range operator.TiledLogs { + if tl.MonitoringURL == additionalLog.URL { + // Log already exists, skip it. + logFound = true + break } + } - break + if !logFound { + // This works, since allLogs.Operators is a slice of pointers. + operator.TiledLogs = append(operator.TiledLogs, &customLog) } + + break } if !operatorFound { @@ -162,6 +178,54 @@ func getAllLogs(logListFetcher LogListFetcher) (loglist3.LogList, error) { } } + // Remove excluded operators and logs after we've added custom logs. + if len(excludedOperators) > 0 || len(excludedLogs) > 0 { + filteredOperators := []*loglist3.Operator{} + + for _, operator := range allLogs.Operators { + // Skip whole operator if it's excluded + if slices.Contains(excludedOperators, operator.Name) { + continue + } + + // Filter normal logs + if len(operator.Logs) > 0 { + keepLogs := []*loglist3.Log{} + for _, l := range operator.Logs { + if slices.Contains(excludedLogs, normalizeCtlogURL(l.URL)) { + // excluded, skip + log.Println("Excluding log based on the config: ", operator.Name, normalizeCtlogURL(l.URL)) + continue + } + + keepLogs = append(keepLogs, l) + } + + operator.Logs = keepLogs + } + + // Filter tiled logs + if len(operator.TiledLogs) > 0 { + keepTiled := []*loglist3.TiledLog{} + for _, tl := range operator.TiledLogs { + if slices.Contains(excludedLogs, normalizeCtlogURL(tl.MonitoringURL)) { + // excluded, skip + log.Println("Excluding tiled log based on the config: ", operator.Name, normalizeCtlogURL(tl.MonitoringURL)) + continue + } + + keepTiled = append(keepTiled, tl) + } + + operator.TiledLogs = keepTiled + } + + filteredOperators = append(filteredOperators, operator) + } + + allLogs.Operators = filteredOperators + } + return allLogs, nil } diff --git a/internal/certificatetransparency/ct-loglist_test.go b/internal/certificatetransparency/ct-loglist_test.go index b4e8bfd..702c7e4 100644 --- a/internal/certificatetransparency/ct-loglist_test.go +++ b/internal/certificatetransparency/ct-loglist_test.go @@ -482,3 +482,110 @@ func TestGetAllLogs_MixedAdditionalLogs(t *testing.T) { t.Errorf("MixedOp: expected 1 classic + 1 tiled, got %d classic, %d tiled", len(op.Logs), len(op.TiledLogs)) } } + +func TestGetAllLogs_ExcludedOperator(t *testing.T) { + t.Cleanup(func() { + config.AppConfig = config.Config{} + }) + + mockList := buildLogList([]struct { + name string + logs []string + tiledLogs []string + }{ + {"Google", []string{"ct.googleapis.com/logs/xenon2024/"}, []string{"tiled.googleapis.com/logs/xenon2024/"}}, + {"Cloudflare", []string{"ct.cloudflare.com/logs/nimbus2024/"}, nil}, + }) + + config.AppConfig.General.DisableDefaultLogs = false + config.AppConfig.General.ExcludedLogs = []config.LogConfig{{Operator: "Google"}} + + result, err := getAllLogs(newMockListFetcher(t, mockList, nil)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if findOperator(result, "Google") != nil { + t.Fatal("expected operator 'Google' to be excluded") + } + + if countLogs(result) != 1 { + t.Errorf("expected 1 classic log after exclusion, got %d", countLogs(result)) + } +} + +func TestGetAllLogs_ExcludedLogByURLWithNormalization(t *testing.T) { + t.Cleanup(func() { + config.AppConfig = config.Config{} + }) + + mockList := buildLogList([]struct { + name string + logs []string + tiledLogs []string + }{ + {"Google", []string{"https://ct.googleapis.com/logs/xenon2024/"}, nil}, + }) + + config.AppConfig.General.DisableDefaultLogs = false + config.AppConfig.General.ExcludedLogs = []config.LogConfig{{URL: "ct.googleapis.com/logs/xenon2024"}} + + result, err := getAllLogs(newMockListFetcher(t, mockList, nil)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if countLogs(result) != 0 { + t.Errorf("expected 0 classic logs after URL exclusion, got %d", countLogs(result)) + } +} + +func TestGetAllLogs_ExcludedTiledLogByURL(t *testing.T) { + t.Cleanup(func() { + config.AppConfig = config.Config{} + }) + + mockList := buildLogList([]struct { + name string + logs []string + tiledLogs []string + }{ + {"Cloudflare", nil, []string{"https://ct.cloudflare.com/logs/raio2025h2b/"}}, + }) + + config.AppConfig.General.DisableDefaultLogs = false + config.AppConfig.General.ExcludedLogs = []config.LogConfig{{URL: "https://ct.cloudflare.com/logs/raio2025h2b"}} + + result, err := getAllLogs(newMockListFetcher(t, mockList, nil)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if countTiledLogs(result) != 0 { + t.Errorf("expected 0 tiled logs after URL exclusion, got %d", countTiledLogs(result)) + } +} + +func TestGetAllLogs_ExcludedRulesApplyToAdditionalLogs(t *testing.T) { + t.Cleanup(func() { + config.AppConfig = config.Config{} + }) + + config.AppConfig.General.DisableDefaultLogs = true + config.AppConfig.General.AdditionalLogs = []config.LogConfig{ + {Operator: "Custom", URL: "https://custom.example.com/classic", Description: "Classic"}, + } + config.AppConfig.General.AdditionalTiledLogs = []config.LogConfig{ + {Operator: "Custom", URL: "https://custom.example.com/tiled", Description: "Tiled"}, + } + config.AppConfig.General.ExcludedLogs = []config.LogConfig{{Operator: "Custom"}} + + result, err := getAllLogs(newMockListFetcher(t, emptyLogList, nil)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(result.Operators) != 0 { + t.Fatalf("expected no operators after exclusions, got %d", len(result.Operators)) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 58245a3..02eefea 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -62,6 +62,7 @@ type Config struct { // AdditionalLogs contains additional logs provided by the user that can be used in addition to the default logs. AdditionalLogs []LogConfig `mapstructure:"additional_logs"` AdditionalTiledLogs []LogConfig `mapstructure:"additional_tiled_logs"` + ExcludedLogs []LogConfig `mapstructure:"excluded_logs"` BufferSizes BufferSizes `mapstructure:"buffer_sizes"` DropOldLogs *bool `mapstructure:"drop_old_logs"` Recovery struct { @@ -267,7 +268,7 @@ func validateConfig(config *Config) bool { } } - var validLogs, validTiledLogs []LogConfig + var validLogs, validTiledLogs, validExcludedLogs []LogConfig if len(config.General.AdditionalLogs) > 0 { for _, ctLog := range config.General.AdditionalLogs { @@ -291,8 +292,28 @@ func validateConfig(config *Config) bool { } } + if len(config.General.ExcludedLogs) > 0 { + for _, excludedLog := range config.General.ExcludedLogs { + excludedLog.Operator = strings.TrimSpace(excludedLog.Operator) + excludedLog.URL = strings.TrimSpace(excludedLog.URL) + + if excludedLog.Operator == "" && excludedLog.URL == "" { + log.Println("Ignoring empty excluded_logs entry. Set operator and/or url.") + continue + } + + if excludedLog.URL != "" && !URLRegex.MatchString(excludedLog.URL) { + log.Println("Ignoring invalid excluded log URL: ", excludedLog.URL) + continue + } + + validExcludedLogs = append(validExcludedLogs, excludedLog) + } + } + config.General.AdditionalLogs = validLogs config.General.AdditionalTiledLogs = validTiledLogs + config.General.ExcludedLogs = validExcludedLogs if len(config.General.AdditionalLogs) == 0 && len(config.General.AdditionalTiledLogs) == 0 && config.General.DisableDefaultLogs { log.Fatalln("Default logs are disabled, but no additional logs are configured. Please add at least one log to the config or enable default logs.") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f9c9107..7d09de4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -435,6 +435,57 @@ general: } } +func TestReadConfigViper_ExcludedLogs(t *testing.T) { + yaml := minimalValidYAML + ` +general: + excluded_logs: + - operator: "Google" + - url: "https://ct.example.com/log" +` + configPath := writeConfigFile(t, yaml) + + cfg, err := ReadConfig(configPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(cfg.General.ExcludedLogs) != 2 { + t.Fatalf("ExcludedLogs: want 2 entries, got %d", len(cfg.General.ExcludedLogs)) + } + + if cfg.General.ExcludedLogs[0].Operator != "Google" { + t.Errorf("ExcludedLogs[0].Operator: want 'Google', got %q", cfg.General.ExcludedLogs[0].Operator) + } + + if cfg.General.ExcludedLogs[1].URL != "https://ct.example.com/log" { + t.Errorf("ExcludedLogs[1].URL: want 'https://ct.example.com/log', got %q", cfg.General.ExcludedLogs[1].URL) + } +} + +func TestReadConfigViper_InvalidExcludedLogURLIgnored(t *testing.T) { + yaml := minimalValidYAML + ` +general: + excluded_logs: + - operator: "Google" + url: "not-a-valid-url" + - operator: "Cloudflare" +` + configPath := writeConfigFile(t, yaml) + + cfg, err := ReadConfig(configPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(cfg.General.ExcludedLogs) != 1 { + t.Fatalf("ExcludedLogs: want 1 valid entry, got %d", len(cfg.General.ExcludedLogs)) + } + + if cfg.General.ExcludedLogs[0].Operator != "Cloudflare" { + t.Errorf("ExcludedLogs[0].Operator: want 'Cloudflare', got %q", cfg.General.ExcludedLogs[0].Operator) + } +} + func TestReadConfigViper_RecoveryConfig(t *testing.T) { yaml := minimalValidYAML + ` general: From 37c32a998e77cd806897f2db73c5760c868867a0 Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 5 Jul 2026 22:28:09 +0200 Subject: [PATCH 35/75] refactor: reformat ct-tiled.go --- internal/certificatetransparency/ct-tiled.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/certificatetransparency/ct-tiled.go b/internal/certificatetransparency/ct-tiled.go index 15e972c..8a3cc84 100644 --- a/internal/certificatetransparency/ct-tiled.go +++ b/internal/certificatetransparency/ct-tiled.go @@ -269,7 +269,7 @@ func NewStaticCTClient(url string, httpClient *http.Client, userAgent string, st } // Monitor continuously monitors the tiled CT log for new entries, starting from the current ctIndex. -func (s *StaticCTClient) Monitor(ctx context.Context, foundCert func(*ct.RawLogEntry), foundPrecert func(*ct.RawLogEntry)) error { +func (s *StaticCTClient) Monitor(ctx context.Context, foundCert, foundPrecert func(*ct.RawLogEntry)) error { for { hadNewEntries, err := s.fetchAndProcessTiles(ctx, foundCert, foundPrecert) if err != nil { @@ -298,7 +298,7 @@ func (s *StaticCTClient) Monitor(ctx context.Context, foundCert func(*ct.RawLogE // fetchAndProcessTiles checks for new entries in the tiled log and processes them. // It returns true if at least one full tile was fetched. -func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert func(*ct.RawLogEntry), foundPrecert func(*ct.RawLogEntry)) (bool, error) { +func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert, foundPrecert func(*ct.RawLogEntry)) (bool, error) { // Fetch current checkpoint checkpoint, fetchErr := s.FetchCheckpoint(ctx) if fetchErr != nil { @@ -346,8 +346,6 @@ func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert fun case time.Since(s.partialTileFirstSeen) >= s.maxPartialWait: // The partial tile has been pending too long – fetch it now to prevent // extreme processing delays on slow-growing logs. - // log.Println("Forcefully fetching partial tile", endTile, "with size", partialSize) - if err := s.processTile(ctx, endTile, partialSize, foundCert, foundPrecert); err != nil { log.Printf("Warning: error processing partial tile %d: %s\n", endTile, err) } @@ -369,7 +367,7 @@ func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert fun // processTile processes a single tile from the tiled log. // partialWidth of 0 means full tile, otherwise fetch partial tile with that width. -func (s *StaticCTClient) processTile(ctx context.Context, tileIndex, partialWidth uint64, foundCert func(*ct.RawLogEntry), foundPrecert func(*ct.RawLogEntry)) error { +func (s *StaticCTClient) processTile(ctx context.Context, tileIndex, partialWidth uint64, foundCert, foundPrecert func(*ct.RawLogEntry)) error { leaves, err := s.fetchTile(ctx, tileIndex, partialWidth) if err != nil { return fmt.Errorf("fetching tile: %w", err) From 69b24b3a6110a967e3386e80ca81807c1ae1be02 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 6 Jul 2026 00:21:56 +0200 Subject: [PATCH 36/75] build: update setup-go in release_build workflow to v6 --- .github/workflows/release_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml index b4cc7bb..d11b918 100644 --- a/.github/workflows/release_build.yml +++ b/.github/workflows/release_build.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Set up Go 1.22 - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ^1.22 id: go From bcfc0af0777d8b9b7ed48d05f251d7a9e8d3ce8e Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 3 Aug 2025 19:12:26 +0200 Subject: [PATCH 37/75] docs: fix typo in docker-compose config --- docker/docker-compose.metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.metrics.yml b/docker/docker-compose.metrics.yml index 316a8ec..301a165 100644 --- a/docker/docker-compose.metrics.yml +++ b/docker/docker-compose.metrics.yml @@ -1,6 +1,6 @@ version: '2' -# Make sure to create the sub directories "prometheus", "prometheus_data", "grafana", "grafana_data" and "certstream" +# Make sure to create the subdirectories "prometheus", "prometheus_data", "grafana", "grafana_data" and "certstream" # and create the config files for all three services. For further details please refer to https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics networks: From 6865b6b20445a0ab48f7923abbe5d0d658678182 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 4 Aug 2025 01:57:24 +0200 Subject: [PATCH 38/75] feat: first implementation for streaming proessors such as kafka Relates to #23 and #35 --- config.sample.yaml | 13 +- go.mod | 3 + go.sum | 10 ++ internal/broadcast/baseclient.go | 48 ++++++ internal/broadcast/broadcastmanager.go | 150 ++++++++++++++++++ internal/broadcast/kafkaclient.go | 122 ++++++++++++++ .../websocketclient.go} | 49 +++--- .../certificatetransparency/ct-watcher.go | 5 +- internal/certstream/certstream.go | 15 +- internal/config/config.go | 18 ++- internal/metrics/logmetrics.go | 2 + internal/metrics/prometheus.go | 2 + internal/web/broadcastmanager.go | 133 ---------------- internal/web/examplecert.go | 2 +- internal/web/server.go | 21 +-- 15 files changed, 410 insertions(+), 183 deletions(-) create mode 100644 internal/broadcast/baseclient.go create mode 100644 internal/broadcast/broadcastmanager.go create mode 100644 internal/broadcast/kafkaclient.go rename internal/{web/client.go => broadcast/websocketclient.go} (84%) delete mode 100644 internal/web/broadcastmanager.go diff --git a/config.sample.yaml b/config.sample.yaml index 74fe9a6..d22dbeb 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -37,6 +37,14 @@ prometheus: whitelist: - "127.0.0.1/8" +# Configuration related to external stream processing tools go here. +streamprocessing: + kafka: + enabled: true + server_address: "127.0.0.1" + server_port: 9092 + topic: "certstream" + general: # DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. disable_default_logs: false @@ -66,9 +74,8 @@ general: websocket: 300 # Buffer for each CT log connection ctlog: 1000 - # Combined buffer for the broadcast manager - broadcastmanager: 10000 - + # Combined buffer for the cert dispatcher + dispatcher: 10000 # Google regularly updates the log list. If this option is set to true, the server will remove all logs no longer listed in the Google log list. # This option defaults to true. See https://github.com/d-Rickyy-b/certstream-server-go/issues/51 drop_old_logs: true diff --git a/go.mod b/go.mod index 162d32c..8aadea7 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/google/certificate-transparency-go v1.3.3 github.com/google/trillian v1.7.3 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 + github.com/segmentio/kafka-go v0.4.51 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 golang.org/x/crypto v0.49.0 @@ -18,7 +19,9 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/pelletier/go-toml/v2 v2.3.0 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect diff --git a/go.sum b/go.sum index 6fb7886..105cf71 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -33,6 +35,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM= github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= @@ -61,6 +65,12 @@ github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= diff --git a/internal/broadcast/baseclient.go b/internal/broadcast/baseclient.go new file mode 100644 index 0000000..9b69e3e --- /dev/null +++ b/internal/broadcast/baseclient.go @@ -0,0 +1,48 @@ +package broadcast + +import "log" + +// BaseClient defines the basic structure for a client that can receive broadcast messages. +// Other client types can embed this struct to inherit its functionality. +type BaseClient struct { + broadcastChan chan []byte + stopChan chan struct{} + name string + subType SubscriptionType + skippedCerts uint64 +} + +// Close cleans up the client's resources by closing the stop and broadcast channels. +func (c *BaseClient) Close() { + close(c.stopChan) + close(c.broadcastChan) +} + +// Name returns the name of the client. +func (c *BaseClient) Name() string { + return c.name +} + +// SubType returns the subscription type of the client. +func (c *BaseClient) SubType() SubscriptionType { + return c.subType +} + +// SkippedCerts returns the number of certificates that were skipped due to the client's broadcast channel being full. +func (c *BaseClient) SkippedCerts() uint64 { + return c.skippedCerts +} + +// Write sends a message to the client's broadcast channel. +// If the channel is full, it increments the skippedCerts counter and logs a message. +func (c *BaseClient) Write(data []byte) { + select { + case c.broadcastChan <- data: + default: + // Default case is executed if the client's broadcast channel is full. + c.skippedCerts++ + if c.skippedCerts%1000 == 1 { + log.Printf("Not providing client '%s' with cert because client's buffer is full. The client can't keep up. Skipped certs: %d\n", c.name, c.skippedCerts) + } + } +} diff --git a/internal/broadcast/broadcastmanager.go b/internal/broadcast/broadcastmanager.go new file mode 100644 index 0000000..2803231 --- /dev/null +++ b/internal/broadcast/broadcastmanager.go @@ -0,0 +1,150 @@ +package broadcast + +import ( + "log" + "sync" + + "github.com/d-Rickyy-b/certstream-server-go/internal/config" + "github.com/d-Rickyy-b/certstream-server-go/internal/metrics" + "github.com/d-Rickyy-b/certstream-server-go/internal/models" +) + +// ClientHandler dispatches certificate entries to registered clients. +var ClientHandler *Dispatcher + +type Dispatcher struct { + MessageQueue chan models.Entry + clients []CertProcessor + clientLock sync.RWMutex +} + +// NewDispatcher creates a new Dispatcher instance and assigns it to the ClientHandler variable. +func NewDispatcher() *Dispatcher { + d := &Dispatcher{} + d.MessageQueue = make(chan models.Entry, config.AppConfig.General.BufferSizes.Dispatcher) + ClientHandler = d + + metrics.Prometheus.RegisterGaugeMetricInt("certstreamservergo_clients_total{type=\"full\"}", d.ClientFullCount) + metrics.Prometheus.RegisterGaugeMetricInt("certstreamservergo_clients_total{type=\"lite\"}", d.ClientLiteCount) + metrics.Prometheus.RegisterGaugeMetricInt("certstreamservergo_clients_total{type=\"domain\"}", d.ClientDomainsCount) + return d +} + +// RegisterClient adds a client to the list of clients of the Dispatcher. +// The client will receive certificate broadcasts right after registration. +func (bm *Dispatcher) RegisterClient(c CertProcessor) { + // TODO: check if the client is already registered + bm.clientLock.Lock() + bm.clients = append(bm.clients, c) + log.Printf("Added new client. Clients: %d, Capacity: %d\n", len(bm.clients), cap(bm.clients)) + metrics.Prometheus.RegisterClient(c.Name(), func() float64 { return float64(c.SkippedCerts()) }) + bm.clientLock.Unlock() +} + +// UnregisterClient removes a client from the list of clients of the Dispatcher. +// The client will no longer receive certificate broadcasts right after unregistering. +func (bm *Dispatcher) UnregisterClient(clientName string) { + bm.clientLock.Lock() + + for i, client := range bm.clients { + if clientName == client.Name() { + // Copy the last element of the slice to the position of the removed element + // Then remove the last element by re-slicing + bm.clients[i] = bm.clients[len(bm.clients)-1] + bm.clients[len(bm.clients)-1] = nil + bm.clients = bm.clients[:len(bm.clients)-1] + + // Close the broadcast channel of the client, otherwise this leads to a memory leak + client.Close() + + metrics.Prometheus.UnregisterClient(c.name) + + break + } + } + + bm.clientLock.Unlock() +} + +// ClientFullCount returns the current number of clients connected to the service on the `full` endpoint. +func (bm *Dispatcher) ClientFullCount() (count int64) { + return bm.clientCountByType(SubTypeFull) +} + +// ClientLiteCount returns the current number of clients connected to the service on the `lite` endpoint. +func (bm *Dispatcher) ClientLiteCount() (count int64) { + return bm.clientCountByType(SubTypeLite) +} + +// ClientDomainsCount returns the current number of clients connected to the service on the `domains-only` endpoint. +func (bm *Dispatcher) ClientDomainsCount() (count int64) { + return bm.clientCountByType(SubTypeDomain) +} + +// clientCountByType returns the current number of clients connected to the service on the endpoint matching +// the specified SubscriptionType. +func (bm *Dispatcher) clientCountByType(subType SubscriptionType) (count int64) { + bm.clientLock.RLock() + defer bm.clientLock.RUnlock() + + for _, c := range bm.clients { + if c.SubType() == subType { + count++ + } + } + + return count +} + +// GetSkippedCerts returns a map of client names to the number of skipped certificates for each client. +func (bm *Dispatcher) GetSkippedCerts() map[string]uint64 { + bm.clientLock.RLock() + defer bm.clientLock.RUnlock() + + skippedCerts := make(map[string]uint64, len(bm.clients)) + for _, c := range bm.clients { + skippedCerts[c.Name()] = c.SkippedCerts() + } + + return skippedCerts +} + +// broadcaster is run in a goroutine and handles the dispatching of certs to clients. +func (bm *Dispatcher) broadcaster() { + for { + var data []byte + + // Take entry out of broadcast channel and generate JSON representations for the entry. + entry := <-bm.Broadcast + dataLite := entry.JSONLite() + dataFull := entry.JSON() + dataDomain := entry.JSONDomains() + + bm.clientLock.RLock() + + for _, c := range bm.clients { + switch c.SubType() { + case SubTypeLite: + data = dataLite + case SubTypeFull: + data = dataFull + case SubTypeDomain: + data = dataDomain + default: + // This should never happen, but if it does, we log it and skip the client. + log.Printf("Unknown subscription type '%d' for client '%s'. Skipping this client!\n", c.SubType(), c.Name()) + continue + } + + c.Write(data) + } + + bm.clientLock.RUnlock() + } +} + +// Start starts the broadcaster goroutine. +func (bm *Dispatcher) Start() { + go bm.broadcaster() + log.Println("Dispatcher started. Listening for certificate entries...") +} diff --git a/internal/broadcast/kafkaclient.go b/internal/broadcast/kafkaclient.go new file mode 100644 index 0000000..b89b845 --- /dev/null +++ b/internal/broadcast/kafkaclient.go @@ -0,0 +1,122 @@ +package broadcast + +import ( + "context" + "log" + "net" + "strconv" + "time" + + "github.com/d-Rickyy-b/certstream-server-go/internal/config" + + "github.com/segmentio/kafka-go" +) + +const ( + TOPIC = "certstream" +) + +// KafkaClient connects to a Kafka server in order to provide it with certificates. +type KafkaClient struct { + conn *kafka.Conn // Kafka connection + addr string + isConnected bool + BaseClient +} + +// NewKafkaClient creates a new Kafka client that immediately connects to the configured Kafka server. +func NewKafkaClient(subType SubscriptionType, name string, certBufferSize int) *KafkaClient { + addr := net.JoinHostPort(config.AppConfig.StreamProcessing.Kafka.ServerAddr, strconv.Itoa(config.AppConfig.StreamProcessing.Kafka.ServerPort)) + + // Connect to the Kafka server + // TODO make topic configurable + conn, err := kafka.DialLeader(context.Background(), "tcp", addr, TOPIC, 0) + if err != nil { + log.Println("failed to connect to kafka:", err) + } + + kc := &KafkaClient{ + conn: conn, + addr: addr, + BaseClient: BaseClient{ + broadcastChan: make(chan []byte, certBufferSize), + stopChan: make(chan struct{}), + name: name, + subType: subType, + }, + } + + go kc.broadcastHandler() + go kc.reconnectHandler() + + return kc +} + +// reconnectHandler is a background job that attempts to reconnect to the Kafka server if the connection is lost. +func (c *KafkaClient) reconnectHandler() { + for { + select { + case <-c.stopChan: + log.Println("Stopping reconnectHandler for kafka producer:", c.addr) + c.conn.Close() + + return + default: + if c.isConnected { + // If already connected or no connection exists, skip reconnection + time.Sleep(5 * time.Second) + continue + } + conn, err := kafka.DialLeader(context.Background(), "tcp", c.addr, TOPIC, 0) + if err != nil { + log.Printf("Reconnect failed: %v. Retrying in 5s...", err) + time.Sleep(5 * time.Second) + + continue + } + // Close old connection if exists + if c.conn != nil { + _ = c.conn.Close() + } + c.conn = conn + c.isConnected = true + log.Println("Reconnected to Kafka at", c.addr) + } + } +} + +// Each client has a broadcastHandler that runs in the background and sends out the broadcast messages to the client. +func (c *KafkaClient) broadcastHandler() { + writeWait := 60 * time.Second + + defer func() { + log.Println("Closing broadcast handler for kafka producer:", c.addr) + if err := c.conn.Close(); err != nil { + log.Println("failed to close writer:", err) + } + + ClientHandler.UnregisterClient(c.name) + }() + + for { + select { + case <-c.stopChan: + return + case message := <-c.broadcastChan: + if !c.isConnected { + continue + } + + _ = c.conn.SetWriteDeadline(time.Now().Add(writeWait)) + + c.conn.Broker() + _, err := c.conn.WriteMessages( + kafka.Message{Value: message}, + ) + if err != nil { + c.isConnected = false + log.Println("Failed to write messages to kafka:", err) + } + } + } +} diff --git a/internal/web/client.go b/internal/broadcast/websocketclient.go similarity index 84% rename from internal/web/client.go rename to internal/broadcast/websocketclient.go index 6626c2a..642600a 100644 --- a/internal/web/client.go +++ b/internal/broadcast/websocketclient.go @@ -1,4 +1,4 @@ -package web +package broadcast import ( "fmt" @@ -21,33 +21,26 @@ const ( type SubscriptionType int -// client represents a single client's connection to the server. -type client struct { - clientData - - id string - conn *websocket.Conn - broadcastChan chan []byte - subType SubscriptionType - skippedCerts uint64 -} - -type clientData struct { - userAgent string - connectionIP string - connectionPort string - realIPFromHeader string +// WebsocketClient represents a single WebSocket client's connection to the server. +type WebsocketClient struct { + conn *websocket.Conn + *BaseClient } -// newClient creates a new client struct that holds information about a connected client. -func newClient(conn *websocket.Conn, subType SubscriptionType, data clientData, certBufferSize int) *client { - return &client{ - clientData: data, - id: generateClientID(), - conn: conn, - broadcastChan: make(chan []byte, certBufferSize), - subType: subType, +// NewWebsocketClient creates a new WebSocket client from the given connection. +func NewWebsocketClient(conn *websocket.Conn, subType SubscriptionType, name string, certBufferSize int) *WebsocketClient { + c := &WebsocketClient{ + conn: conn, + BaseClient: &BaseClient{ + broadcastChan: make(chan []byte, certBufferSize), + name: name, + subType: subType, + }, } + go c.broadcastHandler() + go c.listenWebsocket() + + return c } // generateClientID generates a random 8-char identifier for the client. @@ -62,7 +55,7 @@ func generateClientID() string { } // Each client has a broadcastHandler that runs in the background and sends out the broadcast messages to the client. -func (c *client) broadcastHandler() { +func (c *WebsocketClient) broadcastHandler() { writeWait := 60 * time.Second pingTicker := time.NewTicker(30 * time.Second) @@ -109,10 +102,10 @@ func (c *client) broadcastHandler() { // listenWebsocket is running in the background on a goroutine and listens for messages from the client. // It responds to ping messages with a pong message. It closes the connection if the client sends // a close message or no ping is received within 65 seconds. -func (c *client) listenWebsocket() { +func (c *WebsocketClient) listenWebsocket() { defer func() { _ = c.conn.Close() - ClientHandler.unregisterClient(c) + ClientHandler.UnregisterClient(c.name) }() readWait := 65 * time.Second diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index b4eb458..6db1174 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -13,6 +13,9 @@ import ( "sync/atomic" "time" + "github.com/google/trillian/client/backoff" + + "github.com/d-Rickyy-b/certstream-server-go/internal/broadcast" "github.com/d-Rickyy-b/certstream-server-go/internal/config" "github.com/d-Rickyy-b/certstream-server-go/internal/metrics" "github.com/d-Rickyy-b/certstream-server-go/internal/models" @@ -553,7 +556,7 @@ func certHandler(entryChan chan models.Entry) { } // Run JSON encoding in the background and send the result to the clients. - web.ClientHandler.Broadcast <- entry + broadcast.ClientHandler.MessageQueue <- entry // Update metrics url := entry.Data.Source.NormalizedURL diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index 3bf4065..af00b44 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -11,6 +11,7 @@ import ( "os/signal" "syscall" + "github.com/d-Rickyy-b/certstream-server-go/internal/broadcast" "github.com/d-Rickyy-b/certstream-server-go/internal/certificatetransparency" "github.com/d-Rickyy-b/certstream-server-go/internal/config" "github.com/d-Rickyy-b/certstream-server-go/internal/metrics" @@ -35,6 +36,11 @@ func NewRawCertstream(config config.Config) *Certstream { func NewCertstreamServer(config config.Config) (*Certstream, error) { cs := NewRawCertstream(config) + // Start the broadcast dispatcher + broadcast.NewDispatcher() + broadcast.ClientHandler.Start() + + // TODO: add support do disable websocket Server // Initialize the webserver used for the websocket server webserver := web.NewWebsocketServer( config.Webserver.ListenAddr, @@ -48,7 +54,14 @@ func NewCertstreamServer(config config.Config) (*Certstream, error) { // Setup metrics server cs.setupMetrics(webserver) - return cs, nil + if config.StreamProcessing.Kafka.Enabled { + log.Println("Initializing Kafka client...") + + kc := broadcast.NewKafkaClient(broadcast.SubTypeFull, "kafka-producer", config.General.BufferSizes.Websocket) + broadcast.ClientHandler.RegisterClient(kc) + } + + return &cs, nil } // NewCertstreamFromConfigFile creates a new Certstream server from a config file. diff --git a/internal/config/config.go b/internal/config/config.go index 02eefea..7f3a167 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -38,6 +38,7 @@ type BufferSizes struct { Websocket int `mapstructure:"websocket"` CTLog int `mapstructure:"ctlog"` BroadcastManager int `mapstructure:"broadcastmanager"` + Dispatcher int `mapstructure:"dispatcher"` } type Config struct { @@ -56,6 +57,14 @@ type Config struct { MetricsURL string `mapstructure:"metrics_url"` ExposeSystemMetrics bool `mapstructure:"expose_system_metrics"` } + StreamProcessing struct { + Kafka struct { + Enabled bool `yaml:"enabled"` + ServerAddr string `yaml:"server_addr"` + ServerPort int `yaml:"server_port"` + Topic string `yaml:"topic"` + } + } General struct { // DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. DisableDefaultLogs bool `mapstructure:"disable_default_logs"` @@ -327,8 +336,13 @@ func validateConfig(config *Config) bool { config.General.BufferSizes.CTLog = 1000 } - if config.General.BufferSizes.BroadcastManager <= 0 { - config.General.BufferSizes.BroadcastManager = 10000 + // For backward compatibility, copy value from deprecated BroadcastManager field + if config.General.BufferSizes.BroadcastManager != 0 { + config.General.BufferSizes.Dispatcher = config.General.BufferSizes.BroadcastManager + } + + if config.General.BufferSizes.Dispatcher <= 0 { + config.General.BufferSizes.Dispatcher = 10000 } // If the cleanup flag is not set, default to true diff --git a/internal/metrics/logmetrics.go b/internal/metrics/logmetrics.go index 4f45102..fb10415 100644 --- a/internal/metrics/logmetrics.go +++ b/internal/metrics/logmetrics.go @@ -327,10 +327,12 @@ func GetProcessedPrecerts() int64 { return ProcessedPrecerts } +// GetCertMetrics returns a copy of the internal metrics map. func GetCertMetrics() CTMetrics { return Metrics.GetCTMetrics() } +// GetLogOperators returns a map of operator names to a list of CT logs. func GetLogOperators() map[string][]string { return Metrics.OperatorLogMapping() } diff --git a/internal/metrics/prometheus.go b/internal/metrics/prometheus.go index caca426..2f72bb7 100644 --- a/internal/metrics/prometheus.go +++ b/internal/metrics/prometheus.go @@ -10,6 +10,8 @@ import ( "sync" "time" + "github.com/d-Rickyy-b/certstream-server-go/internal/certificatetransparency" + "github.com/VictoriaMetrics/metrics" ) diff --git a/internal/web/broadcastmanager.go b/internal/web/broadcastmanager.go deleted file mode 100644 index e2e4350..0000000 --- a/internal/web/broadcastmanager.go +++ /dev/null @@ -1,133 +0,0 @@ -package web - -import ( - "log" - "sync" - - "github.com/d-Rickyy-b/certstream-server-go/internal/metrics" - "github.com/d-Rickyy-b/certstream-server-go/internal/models" -) - -type BroadcastManager struct { - Broadcast chan models.Entry - clients []*client - clientLock sync.RWMutex -} - -func NewBroadcastManager() *BroadcastManager { - bm := &BroadcastManager{} - metrics.Prometheus.RegisterGaugeMetricInt("certstreamservergo_clients_total{type=\"full\"}", bm.ClientFullCount) - metrics.Prometheus.RegisterGaugeMetricInt("certstreamservergo_clients_total{type=\"lite\"}", bm.ClientLiteCount) - metrics.Prometheus.RegisterGaugeMetricInt("certstreamservergo_clients_total{type=\"domain\"}", bm.ClientDomainsCount) - - return bm -} - -// registerClient adds a client to the list of clients of the BroadcastManager. -// The client will receive certificate broadcasts right after registration. -func (bm *BroadcastManager) registerClient(c *client) { - bm.clientLock.Lock() - bm.clients = append(bm.clients, c) - log.Printf("Clients: %d, Capacity: %d\n", len(bm.clients), cap(bm.clients)) - metrics.Prometheus.RegisterClient(c.id, c.connectionIP, c.connectionPort, c.realIPFromHeader, c.userAgent, func() float64 { return float64(c.skippedCerts) }) - bm.clientLock.Unlock() -} - -// unregisterClient removes a client from the list of clients of the BroadcastManager. -// The client will no longer receive certificate broadcasts right after unregistering. -func (bm *BroadcastManager) unregisterClient(c *client) { - bm.clientLock.Lock() - log.Println("Unregistering client:", c.conn.RemoteAddr()) - - // Close the broadcast channel of the client, otherwise this leads to a memory leak - close(c.broadcastChan) - metrics.Prometheus.UnregisterClient(c.id, c.connectionIP, c.connectionPort, c.realIPFromHeader, c.userAgent) - - // Remove client from internal client list - for i, storedClient := range bm.clients { - if c != storedClient { - continue - } - - // Copy the last element of the slice to the position of the removed element - // Then remove the last element by re-slicing - bm.clients[i] = bm.clients[len(bm.clients)-1] - bm.clients[len(bm.clients)-1] = nil - bm.clients = bm.clients[:len(bm.clients)-1] - - break - } - - bm.clientLock.Unlock() -} - -// ClientFullCount returns the current number of clients connected to the service on the `full` endpoint. -func (bm *BroadcastManager) ClientFullCount() (count int64) { - return bm.clientCountByType(SubTypeFull) -} - -// ClientLiteCount returns the current number of clients connected to the service on the `lite` endpoint. -func (bm *BroadcastManager) ClientLiteCount() (count int64) { - return bm.clientCountByType(SubTypeLite) -} - -// ClientDomainsCount returns the current number of clients connected to the service on the `domains-only` endpoint. -func (bm *BroadcastManager) ClientDomainsCount() (count int64) { - return bm.clientCountByType(SubTypeDomain) -} - -// clientCountByType returns the current number of clients connected to the service on the endpoint matching -// the specified SubscriptionType. -func (bm *BroadcastManager) clientCountByType(subType SubscriptionType) (count int64) { - bm.clientLock.RLock() - defer bm.clientLock.RUnlock() - - for _, c := range bm.clients { - if c.subType == subType { - count++ - } - } - - return count -} - -// broadcaster is run in a goroutine and handles the dispatching of entries to clients. -func (bm *BroadcastManager) broadcaster() { - for { - var data []byte - - // Take entry out of broadcast channel and generate JSON representations for the entry. - entry := <-bm.Broadcast - dataLite := entry.JSONLite() - dataFull := entry.JSON() - dataDomain := entry.JSONDomains() - - bm.clientLock.RLock() - - for _, c := range bm.clients { - switch c.subType { - case SubTypeLite: - data = dataLite - case SubTypeFull: - data = dataFull - case SubTypeDomain: - data = dataDomain - default: - log.Printf("Unknown subscription type '%d'. Skipping client %s\n", c.subType, c.Name()) - continue - } - - select { - case c.broadcastChan <- data: - default: - // Default case is executed if the client's broadcast channel is full. - c.skippedCerts++ - if c.skippedCerts%1000 == 1 { - log.Printf("Client can't keep up. Skipped certs: %d | Client: %s\n", c.skippedCerts, c.Name()) - } - } - } - - bm.clientLock.RUnlock() - } -} diff --git a/internal/web/examplecert.go b/internal/web/examplecert.go index e841323..05576f3 100644 --- a/internal/web/examplecert.go +++ b/internal/web/examplecert.go @@ -29,7 +29,7 @@ func exampleDomains(w http.ResponseWriter, _ *http.Request) { w.Write(exampleCert.JSONDomains()) //nolint:errcheck } -// SetExampleCert sets one certificate as the example Cert that is returned by the example endpoints. +// SetExampleCert sets the example certificate to be used in the example endpoints. func SetExampleCert(cert models.Entry) { exampleCert = cert } diff --git a/internal/web/server.go b/internal/web/server.go index f3e1d80..4d391fd 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -228,7 +228,7 @@ func initFullWebsocket(w http.ResponseWriter, r *http.Request) { return } - setupClient(connection, SubTypeFull, r) + setupClient(connection, broadcast.SubTypeFull, r.RemoteAddr) } // initLiteWebsocket is called when a client connects to the / endpoint. @@ -240,7 +240,7 @@ func initLiteWebsocket(w http.ResponseWriter, r *http.Request) { return } - setupClient(connection, SubTypeLite, r) + setupClient(connection, broadcast.SubTypeLite, r.RemoteAddr) } // initDomainWebsocket is called when a client connects to the /domains-only endpoint. @@ -252,7 +252,7 @@ func initDomainWebsocket(w http.ResponseWriter, r *http.Request) { return } - setupClient(connection, SubTypeDomain, r) + setupClient(connection, broadcast.SubTypeDomain, r.RemoteAddr) } // upgradeConnection upgrades the connection to a websocket and returns the connection. @@ -283,7 +283,7 @@ func upgradeConnection(w http.ResponseWriter, r *http.Request) (*websocket.Conn, } // setupClient initializes a client struct and starts the broadcastHandler and websocket listener. -func setupClient(connection *websocket.Conn, subType SubscriptionType, r *http.Request) { +func setupClient(connection *websocket.Conn, subscriptionType broadcast.SubscriptionType, name string) { // Extract data from request origConnAddr, _ := r.Context().Value(origConnAddrKey).(string) @@ -305,11 +305,8 @@ func setupClient(connection *websocket.Conn, subType SubscriptionType, r *http.R realIPFromHeader: realIPFromHeader, } - c := newClient(connection, subType, data, config.AppConfig.General.BufferSizes.Websocket) - go c.broadcastHandler() - go c.listenWebsocket() - - ClientHandler.registerClient(c) + c := broadcast.NewWebsocketClient(connection, subscriptionType, name, config.AppConfig.General.BufferSizes.Websocket) + broadcast.ClientHandler.RegisterClient(c) } // setupWebsocketRoutes configures all the routes necessary for the websocket webserver. @@ -385,8 +382,7 @@ func NewMetricsServer(networkIf string, port int, certPath, keyPath string) *Ser } // NewWebsocketServer starts a new webserver and initialized it with the necessary routes. -// It also starts the broadcaster in ClientHandler as a background job and takes care of -// setting up websocket.Upgrader. +// It also takes care of setting up websocket.Upgrader. func NewWebsocketServer(networkIf string, port int, certPath, keyPath string) *Server { websocketServer := &Server{ networkIf: networkIf, @@ -417,9 +413,6 @@ func NewWebsocketServer(networkIf string, port int, certPath, keyPath string) *S setupWebsocketRoutes(websocketServer.routes) websocketServer.initServer() - ClientHandler.Broadcast = make(chan models.Entry, config.AppConfig.General.BufferSizes.BroadcastManager) - go ClientHandler.broadcaster() - return websocketServer } From 8e7e43091bafb1a86f62d610b035ae7b8c7c50f2 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 4 Aug 2025 01:57:53 +0200 Subject: [PATCH 39/75] chore: surround yaml strings by quotes --- docker/docker-compose.metrics.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose.metrics.yml b/docker/docker-compose.metrics.yml index 301a165..9c17a7f 100644 --- a/docker/docker-compose.metrics.yml +++ b/docker/docker-compose.metrics.yml @@ -30,7 +30,7 @@ services: ports: # Exposing Prometheus is NOT required, if you don't want to access it from outside the Docker network. # Using localhost enables you to use a reverse proxy (e.g. with basic auth) to access Prometheus in a more secure way. - - 127.0.0.1:9090:9090 + - "127.0.0.1:9090:9090" networks: - monitoring extra_hosts: @@ -44,7 +44,7 @@ services: depends_on: - prometheus ports: - - 127.0.0.1:8082:3000 + - "127.0.0.1:8082:3000" volumes: - ./grafana_data:/var/lib/grafana - ./grafana/provisioning/:/etc/grafana/provisioning/ @@ -60,7 +60,7 @@ services: # Configure the service to run as specific user. # user: "1000:1000" ports: - - 127.0.0.1:8080:80 + - "127.0.0.1:8080:80" # Don't forget to open the other port in case you run the Prometheus endpoint on another port than the websocket server. # - 127.0.0.1:8081:81 volumes: From 0fdcb45423b5e99f83c299fe330987031b84e0e3 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 4 Aug 2025 01:58:13 +0200 Subject: [PATCH 40/75] docs: add missing options to sample config --- config.sample.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config.sample.yaml b/config.sample.yaml index d22dbeb..cc8fb60 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -23,6 +23,11 @@ webserver: cert_key_path: "" # specify if the server should attempt to negotiate per message compression (RFC 7692) compression_enabled: false + # Use True-Client-IP, X-Real-IP or the X-Forwarded-For headers (in that order) to determine the real IP address of the client. + # If you are using a reverse proxy, you should set this to true. + real_ip: false + whitelist: + - "127.0.0.1/8" prometheus: enabled: true From 6703a32633352f44dde58727a7ee06a63f4603c5 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 4 Aug 2025 01:59:30 +0200 Subject: [PATCH 41/75] fix: add missing imports and sort them --- internal/web/server.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/web/server.go b/internal/web/server.go index 4d391fd..0e05542 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -12,12 +12,11 @@ import ( "strings" "time" - "github.com/go-chi/chi/v5" - "github.com/go-chi/chi/v5/middleware" - + "github.com/d-Rickyy-b/certstream-server-go/internal/broadcast" "github.com/d-Rickyy-b/certstream-server-go/internal/config" - "github.com/d-Rickyy-b/certstream-server-go/internal/models" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" "github.com/gorilla/websocket" ) From 28bfc2ae1a4b6564187a518e01c261d3e879fa98 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 4 Aug 2025 02:04:35 +0200 Subject: [PATCH 42/75] docs: add comments for config --- internal/config/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/config/config.go b/internal/config/config.go index 7f3a167..025b7fe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -72,7 +72,9 @@ type Config struct { AdditionalLogs []LogConfig `mapstructure:"additional_logs"` AdditionalTiledLogs []LogConfig `mapstructure:"additional_tiled_logs"` ExcludedLogs []LogConfig `mapstructure:"excluded_logs"` + // BufferSizes contains the buffer sizes for the different components of the server. They usually don't need any adjustments. BufferSizes BufferSizes `mapstructure:"buffer_sizes"` + // DropOldLogs indicates whether downloading CT-Logs should start at the latest index (true) or should from the beginning (false). DropOldLogs *bool `mapstructure:"drop_old_logs"` Recovery struct { Enabled bool `mapstructure:"enabled"` From 92d983a9b7e8a0a3bd62a3e42585716a1ae07847 Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 5 Aug 2025 17:37:09 +0200 Subject: [PATCH 43/75] feat: add certprocessor interface --- internal/broadcast/certprocessor.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 internal/broadcast/certprocessor.go diff --git a/internal/broadcast/certprocessor.go b/internal/broadcast/certprocessor.go new file mode 100644 index 0000000..d10d47a --- /dev/null +++ b/internal/broadcast/certprocessor.go @@ -0,0 +1,26 @@ +package broadcast + +// CertProcessor defines the interface for processing certificate messages. +// Different implementations can be used for different types of clients, such as WebSocket clients, kafka, or other message queues for example. +type CertProcessor interface { + // Write sends a message to the client's broadcast channel. + Write(message []byte) + // Close closes the client's connection and cleans up resources. + Close() + + // To obtain client details, there are getter methods for several values + Name() string + SubType() SubscriptionType + SkippedCerts() uint64 +} + +const ( + // SubTypeFull represents full certificate updates. + SubTypeFull SubscriptionType = iota + // SubTypeLite represents certificate updates with less details. + SubTypeLite + // SubTypeDomain represents updates that only include domain information. + SubTypeDomain +) + +type SubscriptionType int From e422401892792c539f42c77f57693aebb0382f54 Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 5 Aug 2025 18:08:46 +0200 Subject: [PATCH 44/75] feat: implement NSQ client refers to #23 and #35 I needed to change the config to an arrary in order to have proper support for multiple kinds of queues and stream processors. Currently NSQ and kafka are supported. Future releases could support amazon SNS/SQS by creating a new client in the broadcast package and by extending the config. --- config.sample.yaml | 12 ++- go.mod | 2 + go.sum | 4 + internal/broadcast/kafkaclient.go | 21 +++--- internal/broadcast/nsqclient.go | 119 ++++++++++++++++++++++++++++++ internal/certstream/certstream.go | 39 +++++++++- internal/config/config.go | 16 ++-- 7 files changed, 186 insertions(+), 27 deletions(-) create mode 100644 internal/broadcast/nsqclient.go diff --git a/config.sample.yaml b/config.sample.yaml index cc8fb60..2a52eee 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -43,10 +43,16 @@ prometheus: - "127.0.0.1/8" # Configuration related to external stream processing tools go here. -streamprocessing: - kafka: +stream_processing: + - name: "kafka" + enabled: false + server_addr: "127.0.0.1" + server_port: 9092 + topic: "certstream" + + - name: "nqs" enabled: true - server_address: "127.0.0.1" + server_addr: "127.0.0.1" server_port: 9092 topic: "certstream" diff --git a/go.mod b/go.mod index 8aadea7..1757d6f 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/google/certificate-transparency-go v1.3.3 github.com/google/trillian v1.7.3 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 + github.com/nsqio/go-nsq v1.1.0 github.com/segmentio/kafka-go v0.4.51 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 @@ -18,6 +19,7 @@ require ( github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/golang/snappy v0.0.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/pelletier/go-toml/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 105cf71..609977f 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/certificate-transparency-go v1.3.3 h1:hq/rSxztSkXN2tx/3jQqF6Xc0O565UQPdHrOWvZwybo= github.com/google/certificate-transparency-go v1.3.3/go.mod h1:iR17ZgSaXRzSa5qvjFl8TnVD5h8ky2JMVio+dzoKMgA= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -33,6 +35,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= +github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM= github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= diff --git a/internal/broadcast/kafkaclient.go b/internal/broadcast/kafkaclient.go index b89b845..a10a938 100644 --- a/internal/broadcast/kafkaclient.go +++ b/internal/broadcast/kafkaclient.go @@ -3,12 +3,8 @@ package broadcast import ( "context" "log" - "net" - "strconv" "time" - "github.com/d-Rickyy-b/certstream-server-go/internal/config" - "github.com/segmentio/kafka-go" ) @@ -20,24 +16,23 @@ const ( type KafkaClient struct { conn *kafka.Conn // Kafka connection addr string + topic string isConnected bool BaseClient } // NewKafkaClient creates a new Kafka client that immediately connects to the configured Kafka server. -func NewKafkaClient(subType SubscriptionType, name string, certBufferSize int) *KafkaClient { - addr := net.JoinHostPort(config.AppConfig.StreamProcessing.Kafka.ServerAddr, strconv.Itoa(config.AppConfig.StreamProcessing.Kafka.ServerPort)) - +func NewKafkaClient(subType SubscriptionType, addr, name, topic string, certBufferSize int) *KafkaClient { // Connect to the Kafka server - // TODO make topic configurable - conn, err := kafka.DialLeader(context.Background(), "tcp", addr, TOPIC, 0) + conn, err := kafka.DialLeader(context.Background(), "tcp", addr, topic, 0) if err != nil { log.Println("failed to connect to kafka:", err) } kc := &KafkaClient{ - conn: conn, - addr: addr, + conn: conn, + addr: addr, + topic: topic, BaseClient: BaseClient{ broadcastChan: make(chan []byte, certBufferSize), stopChan: make(chan struct{}), @@ -67,7 +62,9 @@ func (c *KafkaClient) reconnectHandler() { time.Sleep(5 * time.Second) continue } - conn, err := kafka.DialLeader(context.Background(), "tcp", c.addr, TOPIC, 0) + + // Attempt to connect to the Kafka server + conn, err := kafka.DialLeader(context.Background(), "tcp", c.addr, c.topic, 0) if err != nil { log.Printf("Reconnect failed: %v. Retrying in 5s...", err) time.Sleep(5 * time.Second) diff --git a/internal/broadcast/nsqclient.go b/internal/broadcast/nsqclient.go new file mode 100644 index 0000000..56091ec --- /dev/null +++ b/internal/broadcast/nsqclient.go @@ -0,0 +1,119 @@ +package broadcast + +import ( + "github.com/nsqio/go-nsq" + + "log" + "time" +) + +// NSQClient connects to a NSQ server in order to provide it with certificates. +type NSQClient struct { + conn *nsq.Producer // Kafka connection + addr string + topic string + isConnected bool + BaseClient +} + +// nullLogger is a logger that does not output anything. +// It is used to silence the NSQ logger. +type nullLogger struct{} + +func (l nullLogger) Output(calldepth int, s string) error { + // Do nothing, effectively silencing the logger + return nil +} + +// NewNSQClient creates a new NSQ client that immediately connects to the configured NSQ server +func NewNSQClient(subType SubscriptionType, addr, name, topic string, certBufferSize int) *NSQClient { + log.Println("Initializing NSQ client...") + + // Instantiate a producer. + conf := nsq.NewConfig() + conn, err := nsq.NewProducer(addr, conf) + if err != nil { + log.Println(err) + } + + log.Println("Connected to NSQ server at", addr) + + // Silence log output from NSQ + conn.SetLogger(nullLogger{}, nsq.LogLevelError) + + nsqc := &NSQClient{ + conn: conn, + addr: addr, + topic: topic, + BaseClient: BaseClient{ + broadcastChan: make(chan []byte, certBufferSize), + stopChan: make(chan struct{}), + name: name, + subType: subType, + }, + } + nsqc.isConnected = true + + go nsqc.broadcastHandler() + go nsqc.reconnectHandler() + + return nsqc +} + +// reconnectHandler is a background job that attempts to reconnect to the Kafka server if the connection is lost. +func (c *NSQClient) reconnectHandler() { + for { + select { + case <-c.stopChan: + log.Println("Stopping reconnectHandler for kafka producer:", c.addr) + return + default: + if c.isConnected { + // If already connected or no connection exists, skip reconnection + time.Sleep(5 * time.Second) + continue + } + // Attempt to connect to the NSQ server + err := c.conn.Ping() + if err != nil { + log.Printf("Reconnect to NSQ server failed: '%v'. Retrying in 5s...", err) + time.Sleep(5 * time.Second) + + continue + } + + c.isConnected = true + log.Println("Reconnected to NSQ server at", c.addr) + } + } +} + +// Each client has a broadcastHandler that runs in the background and sends out the broadcast messages to the client. +func (c *NSQClient) broadcastHandler() { + // writeWait := 60 * time.Second + + defer func() { + log.Println("Closing broadcast handler for nsq producer:", c.addr) + // Gracefully stop the producer when appropriate (e.g. before shutting down the service) + c.conn.Stop() + }() + + for { + select { + case <-c.stopChan: + return + case message := <-c.broadcastChan: + if !c.isConnected { + continue + } + + // Synchronously publish a single message to the specified topic. + // Messages can also be sent asynchronously and/or in batches. + err := c.conn.Publish(c.topic, message) + if err != nil { + log.Println("Error writing to NSQ topic:", err) + c.isConnected = false + } + } + } +} diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index af00b44..49c7edf 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -7,8 +7,10 @@ package certstream import ( "fmt" "log" + "net" "os" "os/signal" + "strconv" "syscall" "github.com/d-Rickyy-b/certstream-server-go/internal/broadcast" @@ -54,11 +56,40 @@ func NewCertstreamServer(config config.Config) (*Certstream, error) { // Setup metrics server cs.setupMetrics(webserver) - if config.StreamProcessing.Kafka.Enabled { - log.Println("Initializing Kafka client...") + log.Println(config.StreamProcessing) + // Initialize the stream processors if configured and enabled. + for _, streamProcessor := range config.StreamProcessing { + if !streamProcessor.Enabled { + continue + } - kc := broadcast.NewKafkaClient(broadcast.SubTypeFull, "kafka-producer", config.General.BufferSizes.Websocket) - broadcast.ClientHandler.RegisterClient(kc) + addr := net.JoinHostPort(streamProcessor.ServerAddr, strconv.Itoa(streamProcessor.ServerPort)) + log.Printf("Initializing stream processor: %s at %s\n", streamProcessor.Name, addr) + + switch streamProcessor.Type { + case "nsq": + log.Println("Initializing NSQ client...") + nc := broadcast.NewNSQClient( + broadcast.SubTypeFull, + addr, + streamProcessor.Name, + streamProcessor.Topic, + config.General.BufferSizes.Websocket, + ) + broadcast.ClientHandler.RegisterClient(nc) + case "kafka": + log.Println("Initializing Kafka client...") + kc := broadcast.NewKafkaClient( + broadcast.SubTypeFull, + addr, + streamProcessor.Name, + streamProcessor.Topic, + config.General.BufferSizes.Websocket, + ) + broadcast.ClientHandler.RegisterClient(kc) + default: + log.Printf("Unknown stream processor type '%s' for %s. Skipping...\n", streamProcessor.Type, streamProcessor.Name) + } } return &cs, nil diff --git a/internal/config/config.go b/internal/config/config.go index 025b7fe..f3a938f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -57,14 +57,14 @@ type Config struct { MetricsURL string `mapstructure:"metrics_url"` ExposeSystemMetrics bool `mapstructure:"expose_system_metrics"` } - StreamProcessing struct { - Kafka struct { - Enabled bool `yaml:"enabled"` - ServerAddr string `yaml:"server_addr"` - ServerPort int `yaml:"server_port"` - Topic string `yaml:"topic"` - } - } + StreamProcessing []struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + Enabled bool `yaml:"enabled"` + ServerAddr string `yaml:"server_addr"` + ServerPort int `yaml:"server_port"` + Topic string `yaml:"topic"` + } `yaml:"stream_processing"` General struct { // DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. DisableDefaultLogs bool `mapstructure:"disable_default_logs"` From 66ba556ccbacc8a4990b6d57cedd841586db66d7 Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 5 Aug 2025 22:29:01 +0200 Subject: [PATCH 45/75] refactor: remove unnecessary log --- internal/certstream/certstream.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index 49c7edf..d51ba71 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -56,7 +56,6 @@ func NewCertstreamServer(config config.Config) (*Certstream, error) { // Setup metrics server cs.setupMetrics(webserver) - log.Println(config.StreamProcessing) // Initialize the stream processors if configured and enabled. for _, streamProcessor := range config.StreamProcessing { if !streamProcessor.Enabled { From 38c53a182768f61b15424b559157488f68779acf Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 6 Aug 2025 01:01:37 +0200 Subject: [PATCH 46/75] refactor: change "kafka" in comments to "nsq" --- internal/broadcast/nsqclient.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/broadcast/nsqclient.go b/internal/broadcast/nsqclient.go index 56091ec..9aa6b4e 100644 --- a/internal/broadcast/nsqclient.go +++ b/internal/broadcast/nsqclient.go @@ -9,7 +9,7 @@ import ( // NSQClient connects to a NSQ server in order to provide it with certificates. type NSQClient struct { - conn *nsq.Producer // Kafka connection + conn *nsq.Producer // nsq connection addr string topic string isConnected bool @@ -60,12 +60,12 @@ func NewNSQClient(subType SubscriptionType, addr, name, topic string, certBuffer return nsqc } -// reconnectHandler is a background job that attempts to reconnect to the Kafka server if the connection is lost. +// reconnectHandler is a background job that attempts to reconnect to the NSQ server if the connection is lost. func (c *NSQClient) reconnectHandler() { for { select { case <-c.stopChan: - log.Println("Stopping reconnectHandler for kafka producer:", c.addr) + log.Println("Stopping reconnectHandler for nsq producer:", c.addr) return default: if c.isConnected { From 543b98b4f5e62eb1657a7bc07a83c1a13da967d6 Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 23 Nov 2025 18:51:31 +0100 Subject: [PATCH 47/75] fix: broken code due to rebase --- config.sample.yaml | 3 --- go.sum | 2 ++ internal/broadcast/broadcastmanager.go | 8 ++++---- internal/certificatetransparency/ct-watcher.go | 2 -- internal/certstream/certstream.go | 2 +- internal/metrics/prometheus.go | 2 -- internal/web/server.go | 5 +---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/config.sample.yaml b/config.sample.yaml index 2a52eee..aeb3bcd 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -23,9 +23,6 @@ webserver: cert_key_path: "" # specify if the server should attempt to negotiate per message compression (RFC 7692) compression_enabled: false - # Use True-Client-IP, X-Real-IP or the X-Forwarded-For headers (in that order) to determine the real IP address of the client. - # If you are using a reverse proxy, you should set this to true. - real_ip: false whitelist: - "127.0.0.1/8" diff --git a/go.sum b/go.sum index 609977f..7b2c8cd 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= +github.com/segmentio/kafka-go v0.4.51 h1:JgDPPG75tC1rWIS2Me6MwcvXJ6f49UQ4HjAOef71Hno= +github.com/segmentio/kafka-go v0.4.51/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= diff --git a/internal/broadcast/broadcastmanager.go b/internal/broadcast/broadcastmanager.go index 2803231..b1a7fb6 100644 --- a/internal/broadcast/broadcastmanager.go +++ b/internal/broadcast/broadcastmanager.go @@ -57,10 +57,10 @@ func (bm *Dispatcher) UnregisterClient(clientName string) { // Close the broadcast channel of the client, otherwise this leads to a memory leak client.Close() - metrics.Prometheus.UnregisterClient(c.name) + metrics.Prometheus.UnregisterClient(client.Name()) - break - } + break + } } bm.clientLock.Unlock() @@ -115,7 +115,7 @@ func (bm *Dispatcher) broadcaster() { var data []byte // Take entry out of broadcast channel and generate JSON representations for the entry. - entry := <-bm.Broadcast + entry := <-bm.MessageQueue dataLite := entry.JSONLite() dataFull := entry.JSON() dataDomain := entry.JSONDomains() diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index 6db1174..6a271eb 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -13,8 +13,6 @@ import ( "sync/atomic" "time" - "github.com/google/trillian/client/backoff" - "github.com/d-Rickyy-b/certstream-server-go/internal/broadcast" "github.com/d-Rickyy-b/certstream-server-go/internal/config" "github.com/d-Rickyy-b/certstream-server-go/internal/metrics" diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index d51ba71..b2bd095 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -91,7 +91,7 @@ func NewCertstreamServer(config config.Config) (*Certstream, error) { } } - return &cs, nil + return cs, nil } // NewCertstreamFromConfigFile creates a new Certstream server from a config file. diff --git a/internal/metrics/prometheus.go b/internal/metrics/prometheus.go index 2f72bb7..caca426 100644 --- a/internal/metrics/prometheus.go +++ b/internal/metrics/prometheus.go @@ -10,8 +10,6 @@ import ( "sync" "time" - "github.com/d-Rickyy-b/certstream-server-go/internal/certificatetransparency" - "github.com/VictoriaMetrics/metrics" ) diff --git a/internal/web/server.go b/internal/web/server.go index 0e05542..bc6809f 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -20,10 +20,7 @@ import ( "github.com/gorilla/websocket" ) -var ( - ClientHandler = NewBroadcastManager() - upgrader websocket.Upgrader -) +var upgrader websocket.Upgrader type contextKey int From dea5349df3b46d222a396169f1ce74bb16fe26c7 Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 7 Jul 2026 01:50:46 +0200 Subject: [PATCH 48/75] refactor: multiple code fixes after rebase --- config.sample.yaml | 2 -- internal/broadcast/nsqclient.go | 4 +-- internal/broadcast/websocketclient.go | 36 +++++++++---------- .../websocketclient_test.go} | 18 +++++----- internal/metrics/prometheus.go | 17 ++------- internal/web/server.go | 17 +++------ 6 files changed, 33 insertions(+), 61 deletions(-) rename internal/{web/client_test.go => broadcast/websocketclient_test.go} (92%) diff --git a/config.sample.yaml b/config.sample.yaml index aeb3bcd..224a555 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -23,8 +23,6 @@ webserver: cert_key_path: "" # specify if the server should attempt to negotiate per message compression (RFC 7692) compression_enabled: false - whitelist: - - "127.0.0.1/8" prometheus: enabled: true diff --git a/internal/broadcast/nsqclient.go b/internal/broadcast/nsqclient.go index 9aa6b4e..ff9c029 100644 --- a/internal/broadcast/nsqclient.go +++ b/internal/broadcast/nsqclient.go @@ -1,10 +1,10 @@ package broadcast import ( - "github.com/nsqio/go-nsq" - "log" "time" + + "github.com/nsqio/go-nsq" ) // NSQClient connects to a NSQ server in order to provide it with certificates. diff --git a/internal/broadcast/websocketclient.go b/internal/broadcast/websocketclient.go index 642600a..29b913e 100644 --- a/internal/broadcast/websocketclient.go +++ b/internal/broadcast/websocketclient.go @@ -13,24 +13,25 @@ import ( const idChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" -const ( - SubTypeFull SubscriptionType = iota - SubTypeLite - SubTypeDomain -) - -type SubscriptionType int - // WebsocketClient represents a single WebSocket client's connection to the server. type WebsocketClient struct { - conn *websocket.Conn + conn *websocket.Conn + userAgent string + hostIP string + hostPort string + realIPFromHeader string + *BaseClient } // NewWebsocketClient creates a new WebSocket client from the given connection. -func NewWebsocketClient(conn *websocket.Conn, subType SubscriptionType, name string, certBufferSize int) *WebsocketClient { +func NewWebsocketClient(conn *websocket.Conn, subType SubscriptionType, name, userAgent, hostIP, hostPort, realIPFromHeader string, certBufferSize int) *WebsocketClient { c := &WebsocketClient{ - conn: conn, + conn: conn, + userAgent: userAgent, + hostIP: hostIP, + hostPort: hostPort, + realIPFromHeader: realIPFromHeader, BaseClient: &BaseClient{ broadcastChan: make(chan []byte, certBufferSize), name: name, @@ -177,19 +178,14 @@ func sanitizeInput(s string) string { } // Name returns the name/identifier for this client. -func (c *client) Name() string { - var clientName string - - clientName = fmt.Sprintf("[%s] - ", c.id) +func (c *WebsocketClient) Name() string { + clientName := fmt.Sprintf("[%s] - ", c.name) - connIP := sanitizeInput(c.connectionIP) - connPort := sanitizeInput(c.connectionPort) realIP := sanitizeInput(c.realIPFromHeader) - - socket := net.JoinHostPort(connIP, connPort) + socket := net.JoinHostPort(c.hostIP, c.hostPort) // If the realIP is set and if it differs from the connection IP, return both the connection IP and the real IP. - if realIP != "" && realIP != connIP { + if realIP != "" && realIP != c.hostIP { clientName += fmt.Sprintf("%s (via %s)", socket, realIP) } else { clientName += socket diff --git a/internal/web/client_test.go b/internal/broadcast/websocketclient_test.go similarity index 92% rename from internal/web/client_test.go rename to internal/broadcast/websocketclient_test.go index b1e2b30..cc39dd3 100644 --- a/internal/web/client_test.go +++ b/internal/broadcast/websocketclient_test.go @@ -1,19 +1,17 @@ -package web +package broadcast import ( "fmt" "testing" ) -func newTestClient(id, connectionIP, connectionPort, realIP, userAgent string) *client { - c := &client{ - clientData: clientData{ - connectionIP: connectionIP, - connectionPort: connectionPort, - realIPFromHeader: realIP, - userAgent: userAgent, - }, - id: id, +func newTestClient(name, connectionIP, connectionPort, realIP, userAgent string) *WebsocketClient { + c := &WebsocketClient{ + hostIP: connectionIP, + hostPort: connectionPort, + realIPFromHeader: realIP, + userAgent: userAgent, + BaseClient: &BaseClient{name: name}, } return c } diff --git a/internal/metrics/prometheus.go b/internal/metrics/prometheus.go index caca426..a6a9ac4 100644 --- a/internal/metrics/prometheus.go +++ b/internal/metrics/prometheus.go @@ -64,35 +64,22 @@ func (pm *PrometheusExporter) RegisterGaugeMetricInt(label string, callback func } // RegisterClient registers a new gauge metric for the client with the given name. -func (pm *PrometheusExporter) RegisterClient(id, connIP, connPort, realIP, useragent string, skippedCertsCallback func() float64) { +func (pm *PrometheusExporter) RegisterClient(id string, skippedCertsCallback func() float64) { argMap := make(map[string]string) argMap["id"] = id - argMap["conn_ip"] = connIP - argMap["conn_port"] = connPort - argMap["real_ip"] = realIP - argMap["useragent"] = useragent label := createMetric("certstreamservergo_skipped_certs", argMap) - // label := fmt.Sprintf("certstreamservergo_skipped_certs{id=\"%s\",conn_ip=\"%s\",conn_port=\"%s\",real_ip=\"%s\",useragent=\"%s\"}", - // id, connIP, connPort, realIP, useragent) metrics.GetOrCreateGauge(label, skippedCertsCallback) } // UnregisterClient unregisters the metric for the client with the given name. -func (pm *PrometheusExporter) UnregisterClient(id, connIP, connPort, realIP, useragent string) { +func (pm *PrometheusExporter) UnregisterClient(id string) { argMap := make(map[string]string) argMap["id"] = id - argMap["conn_ip"] = connIP - argMap["conn_port"] = connPort - argMap["real_ip"] = realIP - argMap["useragent"] = useragent label := createMetric("certstreamservergo_skipped_certs", argMap) - // label := fmt.Sprintf("certstreamservergo_skipped_certs{id=\"%s\",conn_ip=\"%s\",conn_port=\"%s\",real_ip=\"%s\",useragent=\"%s\"}", - // id, connIP, connPort, realIP, useragent) - ok := metrics.UnregisterMetric(label) if !ok { log.Printf("failed to unregister metric '%s'", label) diff --git a/internal/web/server.go b/internal/web/server.go index bc6809f..72d0c8e 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -224,7 +224,7 @@ func initFullWebsocket(w http.ResponseWriter, r *http.Request) { return } - setupClient(connection, broadcast.SubTypeFull, r.RemoteAddr) + setupClient(connection, broadcast.SubTypeFull, r.RemoteAddr, r) } // initLiteWebsocket is called when a client connects to the / endpoint. @@ -236,7 +236,7 @@ func initLiteWebsocket(w http.ResponseWriter, r *http.Request) { return } - setupClient(connection, broadcast.SubTypeLite, r.RemoteAddr) + setupClient(connection, broadcast.SubTypeLite, r.RemoteAddr, r) } // initDomainWebsocket is called when a client connects to the /domains-only endpoint. @@ -248,7 +248,7 @@ func initDomainWebsocket(w http.ResponseWriter, r *http.Request) { return } - setupClient(connection, broadcast.SubTypeDomain, r.RemoteAddr) + setupClient(connection, broadcast.SubTypeDomain, r.RemoteAddr, r) } // upgradeConnection upgrades the connection to a websocket and returns the connection. @@ -279,7 +279,7 @@ func upgradeConnection(w http.ResponseWriter, r *http.Request) (*websocket.Conn, } // setupClient initializes a client struct and starts the broadcastHandler and websocket listener. -func setupClient(connection *websocket.Conn, subscriptionType broadcast.SubscriptionType, name string) { +func setupClient(connection *websocket.Conn, subscriptionType broadcast.SubscriptionType, name string, r *http.Request) { // Extract data from request origConnAddr, _ := r.Context().Value(origConnAddrKey).(string) @@ -294,14 +294,7 @@ func setupClient(connection *websocket.Conn, subscriptionType broadcast.Subscrip realIPFromHeader = r.RemoteAddr } - data := clientData{ - userAgent: r.Header.Get("User-Agent"), - connectionIP: hostIP, - connectionPort: hostPort, - realIPFromHeader: realIPFromHeader, - } - - c := broadcast.NewWebsocketClient(connection, subscriptionType, name, config.AppConfig.General.BufferSizes.Websocket) + c := broadcast.NewWebsocketClient(connection, subscriptionType, name, r.Header.Get("User-Agent"), hostIP, hostPort, realIPFromHeader, config.AppConfig.General.BufferSizes.Websocket) broadcast.ClientHandler.RegisterClient(c) } From d214a0c87a4268a7d556cb39581cfe8cec8380c6 Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 7 Jul 2026 01:50:59 +0200 Subject: [PATCH 49/75] refactor: fix typo --- cmd/certpicker/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/certpicker/main.go b/cmd/certpicker/main.go index cc82bf5..7323fad 100644 --- a/cmd/certpicker/main.go +++ b/cmd/certpicker/main.go @@ -65,7 +65,7 @@ func main() { log.Fatalln("Error getting entry from CT log: ", getEntryErr) } - // Loop over entries and pars each one. + // Loop over entries and parse each one. for _, leafEntry := range entries.Entries { rawLogEntry, err := ct.RawLogEntryFromLeaf(certID, &leafEntry) if err != nil { From 52a5c91da5000eb641929d0287207bf2087c5d3d Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 7 Jul 2026 01:59:18 +0200 Subject: [PATCH 50/75] refactor: fix config to use mapstructure --- internal/config/config.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index f3a938f..2a930fd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -38,7 +38,7 @@ type BufferSizes struct { Websocket int `mapstructure:"websocket"` CTLog int `mapstructure:"ctlog"` BroadcastManager int `mapstructure:"broadcastmanager"` - Dispatcher int `mapstructure:"dispatcher"` + Dispatcher int `mapstructure:"dispatcher"` } type Config struct { @@ -58,13 +58,13 @@ type Config struct { ExposeSystemMetrics bool `mapstructure:"expose_system_metrics"` } StreamProcessing []struct { - Name string `yaml:"name"` - Type string `yaml:"type"` - Enabled bool `yaml:"enabled"` - ServerAddr string `yaml:"server_addr"` - ServerPort int `yaml:"server_port"` - Topic string `yaml:"topic"` - } `yaml:"stream_processing"` + Name string `mapstructure:"name"` + Type string `mapstructure:"type"` + Enabled bool `mapstructure:"enabled"` + ServerAddr string `mapstructure:"server_addr"` + ServerPort int `mapstructure:"server_port"` + Topic string `mapstructure:"topic"` + } `mapstructure:"stream_processing"` General struct { // DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. DisableDefaultLogs bool `mapstructure:"disable_default_logs"` @@ -73,10 +73,10 @@ type Config struct { AdditionalTiledLogs []LogConfig `mapstructure:"additional_tiled_logs"` ExcludedLogs []LogConfig `mapstructure:"excluded_logs"` // BufferSizes contains the buffer sizes for the different components of the server. They usually don't need any adjustments. - BufferSizes BufferSizes `mapstructure:"buffer_sizes"` + BufferSizes BufferSizes `mapstructure:"buffer_sizes"` // DropOldLogs indicates whether downloading CT-Logs should start at the latest index (true) or should from the beginning (false). - DropOldLogs *bool `mapstructure:"drop_old_logs"` - Recovery struct { + DropOldLogs *bool `mapstructure:"drop_old_logs"` + Recovery struct { Enabled bool `mapstructure:"enabled"` CTIndexFile string `mapstructure:"ct_index_file"` } `mapstructure:"recovery"` From c1781281be78b47aa65f3abe73bf1de441553e62 Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 7 Jul 2026 02:50:31 +0200 Subject: [PATCH 51/75] feat: add compression options for kafka --- internal/broadcast/kafkaclient.go | 15 ++++++++++++++- internal/certstream/certstream.go | 1 + internal/config/config.go | 20 ++++++++++++++------ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/internal/broadcast/kafkaclient.go b/internal/broadcast/kafkaclient.go index a10a938..bebbb35 100644 --- a/internal/broadcast/kafkaclient.go +++ b/internal/broadcast/kafkaclient.go @@ -17,12 +17,13 @@ type KafkaClient struct { conn *kafka.Conn // Kafka connection addr string topic string + compression kafka.Compression isConnected bool BaseClient } // NewKafkaClient creates a new Kafka client that immediately connects to the configured Kafka server. -func NewKafkaClient(subType SubscriptionType, addr, name, topic string, certBufferSize int) *KafkaClient { +func NewKafkaClient(subType SubscriptionType, addr, name, topic, compression string, certBufferSize int) *KafkaClient { // Connect to the Kafka server conn, err := kafka.DialLeader(context.Background(), "tcp", addr, topic, 0) if err != nil { @@ -41,6 +42,18 @@ func NewKafkaClient(subType SubscriptionType, addr, name, topic string, certBuff }, } + switch compression { + case "gzip": + kc.compression = kafka.Gzip + case "snappy": + kc.compression = kafka.Snappy + case "lz4": + kc.compression = kafka.Lz4 + case "none": + default: + log.Println("invalid compression type:", compression) + } + go kc.broadcastHandler() go kc.reconnectHandler() diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index b2bd095..438e541 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -83,6 +83,7 @@ func NewCertstreamServer(config config.Config) (*Certstream, error) { addr, streamProcessor.Name, streamProcessor.Topic, + streamProcessor.Compression, config.General.BufferSizes.Websocket, ) broadcast.ClientHandler.RegisterClient(kc) diff --git a/internal/config/config.go b/internal/config/config.go index 2a930fd..a2f24d0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -58,12 +58,13 @@ type Config struct { ExposeSystemMetrics bool `mapstructure:"expose_system_metrics"` } StreamProcessing []struct { - Name string `mapstructure:"name"` - Type string `mapstructure:"type"` - Enabled bool `mapstructure:"enabled"` - ServerAddr string `mapstructure:"server_addr"` - ServerPort int `mapstructure:"server_port"` - Topic string `mapstructure:"topic"` + Name string `mapstructure:"name"` + Type string `mapstructure:"type"` + Enabled bool `mapstructure:"enabled"` + ServerAddr string `mapstructure:"server_addr"` + ServerPort int `mapstructure:"server_port"` + Topic string `mapstructure:"topic"` + Compression string `mapstructure:"compression"` } `mapstructure:"stream_processing"` General struct { // DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. @@ -303,6 +304,13 @@ func validateConfig(config *Config) bool { } } + if len(config.StreamProcessing) > 0 { + for _, streamProcessing := range config.StreamProcessing { + streamProcessing.Type = strings.ToLower(streamProcessing.Type) + streamProcessing.Compression = strings.ToLower(streamProcessing.Compression) + } + } + if len(config.General.ExcludedLogs) > 0 { for _, excludedLog := range config.General.ExcludedLogs { excludedLog.Operator = strings.TrimSpace(excludedLog.Operator) From 0f96e5995c4d9d1665ba923e3bf5db26c5409609 Mon Sep 17 00:00:00 2001 From: Rico Date: Tue, 7 Jul 2026 02:51:52 +0200 Subject: [PATCH 52/75] feat: batch kafka messages for efficiency Sends messages to the broker in batches to improve throughput. Messages are collected and flushed when a maximum batch size is reached or after a configured wait time. This significantly reduces individual write calls. Messages are also dropped if the client isn't connected. --- internal/broadcast/kafkaclient.go | 46 +++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/internal/broadcast/kafkaclient.go b/internal/broadcast/kafkaclient.go index bebbb35..763462e 100644 --- a/internal/broadcast/kafkaclient.go +++ b/internal/broadcast/kafkaclient.go @@ -9,7 +9,9 @@ import ( ) const ( - TOPIC = "certstream" + maxBatchSize = 50 + maxBatchWait = 60 * time.Second + writeWait = 60 * time.Second ) // KafkaClient connects to a Kafka server in order to provide it with certificates. @@ -97,8 +99,6 @@ func (c *KafkaClient) reconnectHandler() { // Each client has a broadcastHandler that runs in the background and sends out the broadcast messages to the client. func (c *KafkaClient) broadcastHandler() { - writeWait := 60 * time.Second - defer func() { log.Println("Closing broadcast handler for kafka producer:", c.addr) if err := c.conn.Close(); err != nil { @@ -108,25 +108,49 @@ func (c *KafkaClient) broadcastHandler() { ClientHandler.UnregisterClient(c.name) }() + batch := make([]kafka.Message, 0, maxBatchSize) + t := time.NewTimer(maxBatchWait) + for { select { case <-c.stopChan: return case message := <-c.broadcastChan: + // Drop messages if not connected if !c.isConnected { continue } - _ = c.conn.SetWriteDeadline(time.Now().Add(writeWait)) + msg := kafka.Message{Value: message} + batch = append(batch, msg) - c.conn.Broker() - _, err := c.conn.WriteMessages( - kafka.Message{Value: message}, - ) - if err != nil { - c.isConnected = false - log.Println("Failed to write messages to kafka:", err) + // Write batch if it reaches max size + if len(batch) >= maxBatchSize { + c.writeBatch(batch) + batch = batch[:0] + t.Reset(maxBatchWait) } + case <-t.C: + if len(batch) == 0 { + continue + } + + // Write any remaining batch after maxBatchWait + c.writeBatch(batch) + batch = batch[:0] } } } + +func (c *KafkaClient) writeBatch(batch []kafka.Message) { + if len(batch) == 0 { + return + } + + _ = c.conn.SetWriteDeadline(time.Now().Add(writeWait)) + _, err := c.conn.WriteMessages(batch...) + if err != nil { + c.isConnected = false + log.Println("Failed to write messages to kafka:", err) + } +} From d66529c1c2316b3bde4f54f834abf775257f84a9 Mon Sep 17 00:00:00 2001 From: Rico Date: Fri, 10 Jul 2026 21:47:16 +0200 Subject: [PATCH 53/75] refactor: split config into multiple individual files The code for reading and validating the config grew quite big, so it made sense to split it up into multiple files and have a method called Valid() that checks for config validity on each individual struct. In a further iteration we should split the config_test.go into multiple test files each testing only the respective individual files. --- internal/config/buffersizes.go | 16 ++ internal/config/config.go | 247 ++-------------------------- internal/config/general.go | 115 +++++++++++++ internal/config/prometheus.go | 62 +++++++ internal/config/serverconfig.go | 11 ++ internal/config/streamprocessing.go | 72 ++++++++ internal/config/webserver.go | 69 ++++++++ 7 files changed, 358 insertions(+), 234 deletions(-) create mode 100644 internal/config/buffersizes.go create mode 100644 internal/config/general.go create mode 100644 internal/config/prometheus.go create mode 100644 internal/config/serverconfig.go create mode 100644 internal/config/streamprocessing.go create mode 100644 internal/config/webserver.go diff --git a/internal/config/buffersizes.go b/internal/config/buffersizes.go new file mode 100644 index 0000000..d826394 --- /dev/null +++ b/internal/config/buffersizes.go @@ -0,0 +1,16 @@ +package config + +type BufferSizes struct { + Websocket int `mapstructure:"websocket"` + CTLog int `mapstructure:"ctlog"` + BroadcastManager int `mapstructure:"broadcastmanager"` + Dispatcher int `mapstructure:"dispatcher"` +} + +func (b *BufferSizes) Valid() bool { + if b.Websocket <= 0 || b.CTLog <= 0 || b.BroadcastManager <= 0 || b.Dispatcher <= 0 { + return false + } + + return true +} diff --git a/internal/config/config.go b/internal/config/config.go index a2f24d0..30a3505 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "log" - "net" "regexp" "strings" @@ -12,76 +11,20 @@ import ( ) var ( + // AppConfig holds the parsed configuration. AppConfig Config Version = "1.9.0" ErrInvalidConfig = errors.New("invalid configuration") + URLPathRegex = regexp.MustCompile(`^(/[a-zA-Z0-9\-._]+)+$`) + URLRegex = regexp.MustCompile(`^https?://[a-zA-Z0-9\-._]+(:[0-9]+)?(/[a-zA-Z0-9\-._]+)*/?$`) ) -type ServerConfig struct { - ListenAddr string `mapstructure:"listen_addr"` - ListenPort int `mapstructure:"listen_port"` - CertPath string `mapstructure:"cert_path"` - CertKeyPath string `mapstructure:"cert_key_path"` - RealIP bool `mapstructure:"real_ip"` - TrustedProxies []string `mapstructure:"trusted_proxies"` - Whitelist []string `mapstructure:"whitelist"` -} - -type LogConfig struct { - Operator string `mapstructure:"operator"` - URL string `mapstructure:"url"` - Description string `mapstructure:"description"` -} - -type BufferSizes struct { - Websocket int `mapstructure:"websocket"` - CTLog int `mapstructure:"ctlog"` - BroadcastManager int `mapstructure:"broadcastmanager"` - Dispatcher int `mapstructure:"dispatcher"` -} - type Config struct { - Webserver struct { - ServerConfig `mapstructure:",squash"` - - FullURL string `mapstructure:"full_url"` - LiteURL string `mapstructure:"lite_url"` - DomainsOnlyURL string `mapstructure:"domains_only_url"` - CompressionEnabled bool `mapstructure:"compression_enabled"` - } - Prometheus struct { - ServerConfig `mapstructure:",squash"` - - Enabled bool `mapstructure:"enabled"` - MetricsURL string `mapstructure:"metrics_url"` - ExposeSystemMetrics bool `mapstructure:"expose_system_metrics"` - } - StreamProcessing []struct { - Name string `mapstructure:"name"` - Type string `mapstructure:"type"` - Enabled bool `mapstructure:"enabled"` - ServerAddr string `mapstructure:"server_addr"` - ServerPort int `mapstructure:"server_port"` - Topic string `mapstructure:"topic"` - Compression string `mapstructure:"compression"` - } `mapstructure:"stream_processing"` - General struct { - // DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. - DisableDefaultLogs bool `mapstructure:"disable_default_logs"` - // AdditionalLogs contains additional logs provided by the user that can be used in addition to the default logs. - AdditionalLogs []LogConfig `mapstructure:"additional_logs"` - AdditionalTiledLogs []LogConfig `mapstructure:"additional_tiled_logs"` - ExcludedLogs []LogConfig `mapstructure:"excluded_logs"` - // BufferSizes contains the buffer sizes for the different components of the server. They usually don't need any adjustments. - BufferSizes BufferSizes `mapstructure:"buffer_sizes"` - // DropOldLogs indicates whether downloading CT-Logs should start at the latest index (true) or should from the beginning (false). - DropOldLogs *bool `mapstructure:"drop_old_logs"` - Recovery struct { - Enabled bool `mapstructure:"enabled"` - CTIndexFile string `mapstructure:"ct_index_file"` - } `mapstructure:"recovery"` - } + Webserver Webserver + Prometheus Prometheus + StreamProcessing []StreamProcessor `mapstructure:"stream_processing"` + General General } // ReadConfig reads the configuration using Viper and returns a filled Config struct. @@ -185,188 +128,24 @@ func loadConfigFromViper(v *viper.Viper) (Config, error) { // validateConfig validates the config values and sets defaults for missing values. func validateConfig(config *Config) bool { // Still matches invalid IP addresses but good enough for detecting completely wrong formats - URLPathRegex := regexp.MustCompile(`^(/[a-zA-Z0-9\-._]+)+$`) - URLRegex := regexp.MustCompile(`^https?://[a-zA-Z0-9\-._]+(:[0-9]+)?(/[a-zA-Z0-9\-._]+)*/?$`) // Check webserver config - if config.Webserver.ListenAddr == "" || net.ParseIP(config.Webserver.ListenAddr) == nil { - log.Fatalln("Webhook listen IP is not a valid IP: ", config.Webserver.ListenAddr) + if !config.Webserver.Valid() { return false } - if config.Webserver.ListenPort == 0 { - log.Fatalln("Webhook listen port is not set") + if !config.Prometheus.Valid() { return false } - if config.Webserver.FullURL == "" || !URLPathRegex.MatchString(config.Webserver.FullURL) { - log.Println("Webhook full URL is not set or does not match pattern '/...'") - - config.Webserver.FullURL = "/full-stream" - } - - if config.Webserver.LiteURL == "" || !URLPathRegex.MatchString(config.Webserver.FullURL) { - log.Println("Webhook lite URL is not set or does not match pattern '/...'") - - config.Webserver.LiteURL = "/" - } - - if config.Webserver.DomainsOnlyURL == "" || !URLPathRegex.MatchString(config.Webserver.DomainsOnlyURL) { - log.Println("Webhook domains only URL is not set or does not match pattern '/...'") - - config.Webserver.FullURL = "/domains-only" - } - - if config.Webserver.FullURL == config.Webserver.LiteURL { - log.Fatalln("Webhook full URL is the same as lite URL - please fix the config!") - } - - if config.Webserver.DomainsOnlyURL == "" { - config.Webserver.FullURL = "/domains-only" - } - - for _, ip := range config.Webserver.TrustedProxies { - if net.ParseIP(ip) != nil { - continue - } - - _, _, err := net.ParseCIDR(ip) - if err != nil { - log.Fatalln("Invalid IP/CIDR in webserver trusted_proxies: ", ip) + for _, processor := range config.StreamProcessing { + if !processor.Valid() { return false } } - //nolint:nestif - if config.Prometheus.Enabled { - if config.Prometheus.ListenAddr == "" || net.ParseIP(config.Prometheus.ListenAddr) == nil { - log.Fatalln("Metrics export IP is not a valid IP") - return false - } - - if config.Prometheus.ListenPort == 0 { - log.Fatalln("Metrics export port is not set") - return false - } - - if config.Prometheus.Whitelist == nil { - config.Prometheus.Whitelist = []string{} - } - - // Check if IPs in whitelist match pattern - for _, ip := range config.Prometheus.Whitelist { - if net.ParseIP(ip) != nil { - continue - } - - // Provided entry is not an IP, check if it's a CIDR range - _, _, err := net.ParseCIDR(ip) - if err != nil { - log.Fatalln("Invalid IP in metrics whitelist: ", ip) - return false - } - } - - for _, ip := range config.Prometheus.TrustedProxies { - if net.ParseIP(ip) != nil { - continue - } - - _, _, err := net.ParseCIDR(ip) - if err != nil { - log.Fatalln("Invalid IP/CIDR in prometheus trusted_proxies: ", ip) - return false - } - } - } - - var validLogs, validTiledLogs, validExcludedLogs []LogConfig - - if len(config.General.AdditionalLogs) > 0 { - for _, ctLog := range config.General.AdditionalLogs { - if !URLRegex.MatchString(ctLog.URL) { - log.Println("Ignoring invalid additional log URL: ", ctLog.URL) - continue - } - - validLogs = append(validLogs, ctLog) - } - } - - if len(config.General.AdditionalTiledLogs) > 0 { - for _, ctLog := range config.General.AdditionalTiledLogs { - if !URLRegex.MatchString(ctLog.URL) { - log.Println("Ignoring invalid additional log URL: ", ctLog.URL) - continue - } - - validTiledLogs = append(validTiledLogs, ctLog) - } - } - - if len(config.StreamProcessing) > 0 { - for _, streamProcessing := range config.StreamProcessing { - streamProcessing.Type = strings.ToLower(streamProcessing.Type) - streamProcessing.Compression = strings.ToLower(streamProcessing.Compression) - } - } - - if len(config.General.ExcludedLogs) > 0 { - for _, excludedLog := range config.General.ExcludedLogs { - excludedLog.Operator = strings.TrimSpace(excludedLog.Operator) - excludedLog.URL = strings.TrimSpace(excludedLog.URL) - - if excludedLog.Operator == "" && excludedLog.URL == "" { - log.Println("Ignoring empty excluded_logs entry. Set operator and/or url.") - continue - } - - if excludedLog.URL != "" && !URLRegex.MatchString(excludedLog.URL) { - log.Println("Ignoring invalid excluded log URL: ", excludedLog.URL) - continue - } - - validExcludedLogs = append(validExcludedLogs, excludedLog) - } - } - - config.General.AdditionalLogs = validLogs - config.General.AdditionalTiledLogs = validTiledLogs - config.General.ExcludedLogs = validExcludedLogs - - if len(config.General.AdditionalLogs) == 0 && len(config.General.AdditionalTiledLogs) == 0 && config.General.DisableDefaultLogs { - log.Fatalln("Default logs are disabled, but no additional logs are configured. Please add at least one log to the config or enable default logs.") - } - - if config.General.BufferSizes.Websocket <= 0 { - config.General.BufferSizes.Websocket = 300 - } - - if config.General.BufferSizes.CTLog <= 0 { - config.General.BufferSizes.CTLog = 1000 - } - - // For backward compatibility, copy value from deprecated BroadcastManager field - if config.General.BufferSizes.BroadcastManager != 0 { - config.General.BufferSizes.Dispatcher = config.General.BufferSizes.BroadcastManager - } - - if config.General.BufferSizes.Dispatcher <= 0 { - config.General.BufferSizes.Dispatcher = 10000 - } - - // If the cleanup flag is not set, default to true - if config.General.DropOldLogs == nil { - log.Println("drop_old_logs is not set, defaulting to true") - - defaultCleanup := true - config.General.DropOldLogs = &defaultCleanup - } - - if config.General.Recovery.Enabled && config.General.Recovery.CTIndexFile == "" { - log.Println("Recovery enabled but no index file specified. Defaulting to ./ct_index.json") - - config.General.Recovery.CTIndexFile = "./ct_index.json" + if !config.General.Valid() { + return false } return true diff --git a/internal/config/general.go b/internal/config/general.go new file mode 100644 index 0000000..4a244e9 --- /dev/null +++ b/internal/config/general.go @@ -0,0 +1,115 @@ +package config + +import ( + "log" + "strings" +) + +type LogConfig struct { + Operator string `mapstructure:"operator"` + URL string `mapstructure:"url"` + Description string `mapstructure:"description"` +} + +type General struct { + // DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. + DisableDefaultLogs bool `mapstructure:"disable_default_logs"` + // AdditionalLogs contains additional logs provided by the user that can be used in addition to the default logs. + AdditionalLogs []LogConfig `mapstructure:"additional_logs"` + AdditionalTiledLogs []LogConfig `mapstructure:"additional_tiled_logs"` + ExcludedLogs []LogConfig `mapstructure:"excluded_logs"` + // BufferSizes contains the buffer sizes for the different components of the server. They usually don't need any adjustments. + BufferSizes BufferSizes `mapstructure:"buffer_sizes"` + // DropOldLogs indicates whether downloading CT-Logs should start at the latest index (true) or should from the beginning (false). + DropOldLogs *bool `mapstructure:"drop_old_logs"` + Recovery struct { + Enabled bool `mapstructure:"enabled"` + CTIndexFile string `mapstructure:"ct_index_file"` + } `mapstructure:"recovery"` +} + +func (g *General) Valid() bool { + var validLogs, validTiledLogs, validExcludedLogs []LogConfig + + if len(g.AdditionalLogs) > 0 { + for _, ctLog := range g.AdditionalLogs { + if !URLRegex.MatchString(ctLog.URL) { + log.Println("Ignoring invalid additional log URL: ", ctLog.URL) + continue + } + + validLogs = append(validLogs, ctLog) + } + } + + if len(g.AdditionalTiledLogs) > 0 { + for _, ctLog := range g.AdditionalTiledLogs { + if !URLRegex.MatchString(ctLog.URL) { + log.Println("Ignoring invalid additional log URL: ", ctLog.URL) + continue + } + + validTiledLogs = append(validTiledLogs, ctLog) + } + } + + if len(g.ExcludedLogs) > 0 { + for _, excludedLog := range g.ExcludedLogs { + excludedLog.Operator = strings.TrimSpace(excludedLog.Operator) + excludedLog.URL = strings.TrimSpace(excludedLog.URL) + + if excludedLog.Operator == "" && excludedLog.URL == "" { + log.Println("Ignoring empty excluded_logs entry. Set operator and/or url.") + continue + } + + if excludedLog.URL != "" && !URLRegex.MatchString(excludedLog.URL) { + log.Println("Ignoring invalid excluded log URL: ", excludedLog.URL) + continue + } + + validExcludedLogs = append(validExcludedLogs, excludedLog) + } + } + + g.AdditionalLogs = validLogs + g.AdditionalTiledLogs = validTiledLogs + g.ExcludedLogs = validExcludedLogs + + if len(g.AdditionalLogs) == 0 && len(g.AdditionalTiledLogs) == 0 && g.DisableDefaultLogs { + log.Fatalln("Default logs are disabled, but no additional logs are configured. Please add at least one log to the config or enable default logs.") + } + + if g.BufferSizes.Websocket <= 0 { + g.BufferSizes.Websocket = 300 + } + + if g.BufferSizes.CTLog <= 0 { + g.BufferSizes.CTLog = 1000 + } + + // For backward compatibility, copy value from deprecated BroadcastManager field + if g.BufferSizes.BroadcastManager != 0 { + g.BufferSizes.Dispatcher = g.BufferSizes.BroadcastManager + } + + if g.BufferSizes.Dispatcher <= 0 { + g.BufferSizes.Dispatcher = 10000 + } + + // If the cleanup flag is not set, default to true + if g.DropOldLogs == nil { + log.Println("drop_old_logs is not set, defaulting to true") + + defaultCleanup := true + g.DropOldLogs = &defaultCleanup + } + + if g.Recovery.Enabled && g.Recovery.CTIndexFile == "" { + log.Println("Recovery enabled but no index file specified. Defaulting to ./ct_index.json") + + g.Recovery.CTIndexFile = "./ct_index.json" + } + + return true +} diff --git a/internal/config/prometheus.go b/internal/config/prometheus.go new file mode 100644 index 0000000..cfbcf58 --- /dev/null +++ b/internal/config/prometheus.go @@ -0,0 +1,62 @@ +package config + +import ( + "log" + "net" +) + +type Prometheus struct { + ServerConfig `mapstructure:",squash"` + + Enabled bool `mapstructure:"enabled"` + MetricsURL string `mapstructure:"metrics_url"` + ExposeSystemMetrics bool `mapstructure:"expose_system_metrics"` +} + +func (p *Prometheus) Valid() bool { + if !p.Enabled { + return true + } + + if p.ListenAddr == "" || net.ParseIP(p.ListenAddr) == nil { + log.Fatalln("Metrics export IP is not a valid IP") + return false + } + + if p.ListenPort == 0 { + log.Fatalln("Metrics export port is not set") + return false + } + + if p.Whitelist == nil { + p.Whitelist = []string{} + } + + // Check if IPs in whitelist match pattern + for _, ip := range p.Whitelist { + if net.ParseIP(ip) != nil { + continue + } + + // Provided entry is not an IP, check if it's a CIDR range + _, _, err := net.ParseCIDR(ip) + if err != nil { + log.Fatalln("Invalid IP in metrics whitelist: ", ip) + return false + } + } + + for _, ip := range p.TrustedProxies { + if net.ParseIP(ip) != nil { + continue + } + + _, _, err := net.ParseCIDR(ip) + if err != nil { + log.Fatalln("Invalid IP/CIDR in prometheus trusted_proxies: ", ip) + return false + } + } + + return true +} diff --git a/internal/config/serverconfig.go b/internal/config/serverconfig.go new file mode 100644 index 0000000..a246d4a --- /dev/null +++ b/internal/config/serverconfig.go @@ -0,0 +1,11 @@ +package config + +type ServerConfig struct { + ListenAddr string `mapstructure:"listen_addr"` + ListenPort int `mapstructure:"listen_port"` + CertPath string `mapstructure:"cert_path"` + CertKeyPath string `mapstructure:"cert_key_path"` + RealIP bool `mapstructure:"real_ip"` + TrustedProxies []string `mapstructure:"trusted_proxies"` + Whitelist []string `mapstructure:"whitelist"` +} diff --git a/internal/config/streamprocessing.go b/internal/config/streamprocessing.go new file mode 100644 index 0000000..4c47ab1 --- /dev/null +++ b/internal/config/streamprocessing.go @@ -0,0 +1,72 @@ +package config + +import ( + "log" + "net" +) + +type StreamProcessorType string + +const ( + StreamProcessorTypeKafka StreamProcessorType = "kafka" + StreamProcessorTypeNQS StreamProcessorType = "nqs" +) + +type StreamProcessorCompression string + +const ( + StreamProcessorCompressionNone StreamProcessorCompression = "none" + StreamProcessorCompressionGzip StreamProcessorCompression = "gzip" + StreamProcessorCompressionSnappy StreamProcessorCompression = "snappy" +) + +func (c StreamProcessorCompression) Valid() bool { + switch c { + case StreamProcessorCompressionNone, StreamProcessorCompressionGzip, StreamProcessorCompressionSnappy, "": + return true + default: + return false + } +} + +type StreamProcessor struct { + Name string `mapstructure:"name"` + Type StreamProcessorType `mapstructure:"type"` + Enabled bool `mapstructure:"enabled"` + ServerAddr string `mapstructure:"server_addr"` + ServerPort int `mapstructure:"server_port"` + Topic string `mapstructure:"topic"` + Compression StreamProcessorCompression `mapstructure:"compression"` +} + +func (s *StreamProcessor) Valid() bool { + ip := net.ParseIP(s.ServerAddr) + if ip == nil { + log.Fatalln("Invalid IP for stream processor:", s.ServerAddr) + return false + } + + if s.ServerPort <= 0 { + log.Fatalln("Invalid server port for stream processor:", s.ServerPort) + return false + } + + switch s.Type { + case StreamProcessorTypeKafka, StreamProcessorTypeNQS: + default: + log.Fatalf("Invalid stream processor type '%s' for name '%s'\n", s.Type, s.Name) + return false + } + + if s.Topic == "" { + log.Println("Found stream processing config with empty topic - using \"certstream\" as topic for name", s.Name) + s.Topic = "certstream" + } + + if !s.Compression.Valid() { + log.Fatalf("Invalid compression '%s' for stream processor '%s'\n", s.Compression, s.Name) + return false + } + + return true +} diff --git a/internal/config/webserver.go b/internal/config/webserver.go new file mode 100644 index 0000000..fb846d9 --- /dev/null +++ b/internal/config/webserver.go @@ -0,0 +1,69 @@ +package config + +import ( + "log" + "net" +) + +type Webserver struct { + ServerConfig `mapstructure:",squash"` + + FullURL string `mapstructure:"full_url"` + LiteURL string `mapstructure:"lite_url"` + DomainsOnlyURL string `mapstructure:"domains_only_url"` + CompressionEnabled bool `mapstructure:"compression_enabled"` +} + +func (w *Webserver) Valid() bool { + // Still matches invalid IP addresses but good enough for detecting completely wrong formats + + if w.ListenAddr == "" || net.ParseIP(w.ListenAddr) == nil { + log.Fatalln("Webhook listen IP is not a valid IP: ", w.ListenAddr) + return false + } + + if w.ListenPort == 0 { + log.Fatalln("Webhook listen port is not set") + return false + } + + if w.FullURL == "" || !URLPathRegex.MatchString(w.FullURL) { + log.Println("Webhook full URL is not set or does not match pattern '/...'") + + w.FullURL = "/full-stream" + } + + if w.LiteURL == "" || !URLPathRegex.MatchString(w.FullURL) { + log.Println("Webhook lite URL is not set or does not match pattern '/...'") + + w.LiteURL = "/" + } + + if w.DomainsOnlyURL == "" || !URLPathRegex.MatchString(w.DomainsOnlyURL) { + log.Println("Webhook domains only URL is not set or does not match pattern '/...'") + + w.FullURL = "/domains-only" + } + + if w.FullURL == w.LiteURL { + log.Fatalln("Webhook full URL is the same as lite URL - please fix the config!") + } + + if w.DomainsOnlyURL == "" { + w.FullURL = "/domains-only" + } + + for _, ip := range w.TrustedProxies { + if net.ParseIP(ip) != nil { + continue + } + + _, _, err := net.ParseCIDR(ip) + if err != nil { + log.Fatalln("Invalid IP/CIDR in webserver trusted_proxies: ", ip) + return false + } + } + + return true +} From 8fc04dec542bcd68621a9863b5d63f2b3ebabfec Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 01:32:18 +0200 Subject: [PATCH 54/75] refactor: rename config variable to cfg --- internal/certstream/certstream.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index 438e541..d03c759 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -35,8 +35,8 @@ func NewRawCertstream(config config.Config) *Certstream { } // NewCertstreamServer creates a new Certstream server from a config struct. -func NewCertstreamServer(config config.Config) (*Certstream, error) { - cs := NewRawCertstream(config) +func NewCertstreamServer(cfg config.Config) (*Certstream, error) { + cs := NewRawCertstream(cfg) // Start the broadcast dispatcher broadcast.NewDispatcher() @@ -45,10 +45,10 @@ func NewCertstreamServer(config config.Config) (*Certstream, error) { // TODO: add support do disable websocket Server // Initialize the webserver used for the websocket server webserver := web.NewWebsocketServer( - config.Webserver.ListenAddr, - config.Webserver.ListenPort, - config.Webserver.CertPath, - config.Webserver.CertKeyPath, + cfg.Webserver.ListenAddr, + cfg.Webserver.ListenPort, + cfg.Webserver.CertPath, + cfg.Webserver.CertKeyPath, ) cs.webserver = webserver cs.watcher = certificatetransparency.NewWatcher() @@ -57,7 +57,7 @@ func NewCertstreamServer(config config.Config) (*Certstream, error) { cs.setupMetrics(webserver) // Initialize the stream processors if configured and enabled. - for _, streamProcessor := range config.StreamProcessing { + for _, streamProcessor := range cfg.StreamProcessing { if !streamProcessor.Enabled { continue } @@ -73,7 +73,7 @@ func NewCertstreamServer(config config.Config) (*Certstream, error) { addr, streamProcessor.Name, streamProcessor.Topic, - config.General.BufferSizes.Websocket, + cfg.General.BufferSizes.Websocket, ) broadcast.ClientHandler.RegisterClient(nc) case "kafka": @@ -83,9 +83,10 @@ func NewCertstreamServer(config config.Config) (*Certstream, error) { addr, streamProcessor.Name, streamProcessor.Topic, - streamProcessor.Compression, - config.General.BufferSizes.Websocket, + string(streamProcessor.Compression), + cfg.General.BufferSizes.Websocket, ) + broadcast.ClientHandler.RegisterClient(kc) default: log.Printf("Unknown stream processor type '%s' for %s. Skipping...\n", streamProcessor.Type, streamProcessor.Name) From 4d005db3f6d8029bbef3d592b1f26ed061edf86f Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 01:32:57 +0200 Subject: [PATCH 55/75] refactor: make stream type configurable for stream processing --- internal/certstream/certstream.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index d03c759..b3b3815 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -65,11 +65,21 @@ func NewCertstreamServer(cfg config.Config) (*Certstream, error) { addr := net.JoinHostPort(streamProcessor.ServerAddr, strconv.Itoa(streamProcessor.ServerPort)) log.Printf("Initializing stream processor: %s at %s\n", streamProcessor.Name, addr) + var subscriptionType broadcast.SubscriptionType + switch streamProcessor.Stream { + case config.StreamTypeFull: + subscriptionType = broadcast.SubTypeFull + case config.StreamTypeLite: + subscriptionType = broadcast.SubTypeLite + case config.StreamTypeDomainsOnly: + subscriptionType = broadcast.SubTypeDomain + } + switch streamProcessor.Type { case "nsq": log.Println("Initializing NSQ client...") nc := broadcast.NewNSQClient( - broadcast.SubTypeFull, + subscriptionType, addr, streamProcessor.Name, streamProcessor.Topic, @@ -79,7 +89,7 @@ func NewCertstreamServer(cfg config.Config) (*Certstream, error) { case "kafka": log.Println("Initializing Kafka client...") kc := broadcast.NewKafkaClient( - broadcast.SubTypeFull, + subscriptionType, addr, streamProcessor.Name, streamProcessor.Topic, From 258a6c5fef1f871578da65d1a02ce87ff45c6bbf Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 01:34:07 +0200 Subject: [PATCH 56/75] refactor(nsqclient): improve formatting and error handling --- internal/broadcast/nsqclient.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/internal/broadcast/nsqclient.go b/internal/broadcast/nsqclient.go index ff9c029..9b99bad 100644 --- a/internal/broadcast/nsqclient.go +++ b/internal/broadcast/nsqclient.go @@ -31,6 +31,7 @@ func NewNSQClient(subType SubscriptionType, addr, name, topic string, certBuffer // Instantiate a producer. conf := nsq.NewConfig() + conn, err := nsq.NewProducer(addr, conf) if err != nil { log.Println(err) @@ -66,6 +67,7 @@ func (c *NSQClient) reconnectHandler() { select { case <-c.stopChan: log.Println("Stopping reconnectHandler for nsq producer:", c.addr) + return default: if c.isConnected { @@ -73,6 +75,7 @@ func (c *NSQClient) reconnectHandler() { time.Sleep(5 * time.Second) continue } + // Attempt to connect to the NSQ server err := c.conn.Ping() if err != nil { @@ -90,19 +93,25 @@ func (c *NSQClient) reconnectHandler() { // Each client has a broadcastHandler that runs in the background and sends out the broadcast messages to the client. func (c *NSQClient) broadcastHandler() { - // writeWait := 60 * time.Second - defer func() { log.Println("Closing broadcast handler for nsq producer:", c.addr) - // Gracefully stop the producer when appropriate (e.g. before shutting down the service) - c.conn.Stop() + if c.conn != nil { + // Gracefully stop the producer when appropriate (e.g. before shutting down the service) + c.conn.Stop() + } + }() for { select { case <-c.stopChan: return - case message := <-c.broadcastChan: + case message, ok := <-c.broadcastChan: + if !ok { + log.Println("broadcastChan closed for nsqClient:", c.addr) + return + } + if !c.isConnected { continue } @@ -111,7 +120,7 @@ func (c *NSQClient) broadcastHandler() { // Messages can also be sent asynchronously and/or in batches. err := c.conn.Publish(c.topic, message) if err != nil { - log.Println("Error writing to NSQ topic:", err) + log.Println("Failed to write messages to NSQ:", err) c.isConnected = false } } From 4cd6c16c76543db987555023fecb8d287f7a2dbe Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 01:36:06 +0200 Subject: [PATCH 57/75] refactor(nsqclient): minor logic fixes --- internal/broadcast/nsqclient.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/broadcast/nsqclient.go b/internal/broadcast/nsqclient.go index 9b99bad..de4d4c0 100644 --- a/internal/broadcast/nsqclient.go +++ b/internal/broadcast/nsqclient.go @@ -67,6 +67,7 @@ func (c *NSQClient) reconnectHandler() { select { case <-c.stopChan: log.Println("Stopping reconnectHandler for nsq producer:", c.addr) + c.conn.Stop() return default: @@ -100,6 +101,7 @@ func (c *NSQClient) broadcastHandler() { c.conn.Stop() } + ClientHandler.UnregisterClient(c.name) }() for { @@ -112,7 +114,9 @@ func (c *NSQClient) broadcastHandler() { return } + // Drop messages if not connected if !c.isConnected { + time.Sleep(5 * time.Second) continue } From fec2d29240f8584acb14e160e0fe0e5a11808ef5 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 01:36:29 +0200 Subject: [PATCH 58/75] feat(nsqclient): add batch processing feature --- internal/broadcast/nsqclient.go | 47 ++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/internal/broadcast/nsqclient.go b/internal/broadcast/nsqclient.go index de4d4c0..f8e042e 100644 --- a/internal/broadcast/nsqclient.go +++ b/internal/broadcast/nsqclient.go @@ -7,6 +7,11 @@ import ( "github.com/nsqio/go-nsq" ) +const ( + nsqMaxBatchSize = 50 + nsqMaxBatchWait = 1 * time.Second +) + // NSQClient connects to a NSQ server in order to provide it with certificates. type NSQClient struct { conn *nsq.Producer // nsq connection @@ -104,6 +109,9 @@ func (c *NSQClient) broadcastHandler() { ClientHandler.UnregisterClient(c.name) }() + batch := make([][]byte, 0, nsqMaxBatchSize) + t := time.NewTicker(nsqMaxBatchWait) + for { select { case <-c.stopChan: @@ -120,13 +128,46 @@ func (c *NSQClient) broadcastHandler() { continue } - // Synchronously publish a single message to the specified topic. - // Messages can also be sent asynchronously and/or in batches. - err := c.conn.Publish(c.topic, message) + batch = append(batch, message) + + // Write batch if it reaches max size + if len(batch) >= nsqMaxBatchSize { + err := c.writeBatch(batch) + if err != nil { + log.Println("Failed to write messages to NSQ:", err) + c.isConnected = false + } + batch = batch[:0] + t.Reset(kafkaMaxBatchWait) + } + case <-t.C: + // If batch size has not reached nsqMaxBatchSize, write the batch after nsqMaxBatchWait + if len(batch) == 0 { + continue + } + + err := c.writeBatch(batch) if err != nil { log.Println("Failed to write messages to NSQ:", err) c.isConnected = false } + batch = batch[:0] } } } + +func (c *NSQClient) writeBatch(batch [][]byte) error { + if len(batch) == 0 { + return nil + } + + // Synchronously publish a batch of messages to the specified topic. + err := c.conn.MultiPublish(c.topic, batch) + if err != nil { + log.Println("Failed to write messages to NSQ:", err) + c.isConnected = false + return err + } + + return nil +} From f8ca66e9715f420e20a0a512f08e71328ef3b85f Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 01:37:30 +0200 Subject: [PATCH 59/75] refactor(config.sample.yml): add new options to sample config --- config.sample.yaml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/config.sample.yaml b/config.sample.yaml index 224a555..bed04c4 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -39,14 +39,29 @@ prometheus: # Configuration related to external stream processing tools go here. stream_processing: - - name: "kafka" - enabled: false + # List of stream processing configurations. + # Each configuration specifies a stream processing tool to use and its settings. + - name: "kafka-prod" + # Type describes the type of stream processing tool to use. + # Supported types: "kafka", "nqs" + type: "kafka" + # The stream type can be any of "full", "lite", "domains-only" - same as the certstream websocket urls + # Default is "full" + stream: "full" + # Enable or disable this stream processing configuration. + enabled: true + # The server address and port to connect to the stream processing tool server_addr: "127.0.0.1" server_port: 9092 topic: "certstream" + # Available compression types depend on the stream processing tool. + # E.g. for Kafka, supported types are "none" / "", "gzip", "snappy", "lz4", "zstd" + compression: "" - - name: "nqs" - enabled: true + - name: "nqs-prod" + type: "nqs" + stream: "lite" + enabled: false server_addr: "127.0.0.1" server_port: 9092 topic: "certstream" From 0c52b420f486dc44dcec0d45cd65ccd76ba94218 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 01:54:39 +0200 Subject: [PATCH 60/75] refactor: improve streamprocessing struct and methods Made stream type configurable so that users can decide which stream they want to feed into kafka. --- internal/certstream/certstream.go | 2 +- internal/config/streamprocessing.go | 87 ++++++++++++++++++++++++----- 2 files changed, 75 insertions(+), 14 deletions(-) diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index b3b3815..01cda11 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -58,7 +58,7 @@ func NewCertstreamServer(cfg config.Config) (*Certstream, error) { // Initialize the stream processors if configured and enabled. for _, streamProcessor := range cfg.StreamProcessing { - if !streamProcessor.Enabled { + if !*streamProcessor.Enabled { continue } diff --git a/internal/config/streamprocessing.go b/internal/config/streamprocessing.go index 4c47ab1..7dd8478 100644 --- a/internal/config/streamprocessing.go +++ b/internal/config/streamprocessing.go @@ -5,6 +5,8 @@ import ( "net" ) +// StreamProcessorType represents the type of stream processing tool to use. +// Supported types are "kafka" and "nqs". type StreamProcessorType string const ( @@ -12,34 +14,86 @@ const ( StreamProcessorTypeNQS StreamProcessorType = "nqs" ) -type StreamProcessorCompression string +// StreamType represents the type of stream to process. +// Supported types are "full", "lite", and "domains-only". +type StreamType string const ( - StreamProcessorCompressionNone StreamProcessorCompression = "none" - StreamProcessorCompressionGzip StreamProcessorCompression = "gzip" - StreamProcessorCompressionSnappy StreamProcessorCompression = "snappy" + StreamTypeFull StreamType = "full" + StreamTypeLite StreamType = "lite" + StreamTypeDomainsOnly StreamType = "domains-only" ) -func (c StreamProcessorCompression) Valid() bool { +// Compression represents the compression type for stream processing. +type Compression string + +const ( + CompressionNone Compression = "none" + CompressionGzip Compression = "gzip" + CompressionSnappy Compression = "snappy" + CompressionZstd Compression = "zstd" + CompressionLz4 Compression = "lz4" + CompressionDeflate Compression = "deflate" +) + +func (c Compression) setDefaults() { + if c == "" { + c = CompressionNone + } +} + +// Valid returns true if the compression type is valid. +func (c Compression) Valid() bool { + c.setDefaults() + switch c { - case StreamProcessorCompressionNone, StreamProcessorCompressionGzip, StreamProcessorCompressionSnappy, "": + case CompressionNone, CompressionGzip, CompressionSnappy, CompressionZstd, CompressionLz4: return true default: return false } } +// SupportedBy returns true if the compression type is supported by the stream processing tool. +func (c Compression) SupportedBy(t StreamProcessorType) bool { + if c == CompressionNone { + return true + } + + switch t { + case StreamProcessorTypeKafka: + return c == CompressionNone || c == CompressionGzip || c == CompressionSnappy || c == CompressionZstd || c == CompressionLz4 + case StreamProcessorTypeNQS: + return c == CompressionNone || c == CompressionDeflate || c == CompressionSnappy + default: + return false + } +} + type StreamProcessor struct { - Name string `mapstructure:"name"` - Type StreamProcessorType `mapstructure:"type"` - Enabled bool `mapstructure:"enabled"` - ServerAddr string `mapstructure:"server_addr"` - ServerPort int `mapstructure:"server_port"` - Topic string `mapstructure:"topic"` - Compression StreamProcessorCompression `mapstructure:"compression"` + Name string `mapstructure:"name"` + Type StreamProcessorType `mapstructure:"type"` + Stream StreamType `mapstructure:"stream"` + Enabled *bool `mapstructure:"enabled"` + ServerAddr string `mapstructure:"server_addr"` + ServerPort int `mapstructure:"server_port"` + Topic string `mapstructure:"topic"` + Compression Compression `mapstructure:"compression"` +} + +func (s *StreamProcessor) setDefaults() { + if s.Enabled == nil { + enabled := true + s.Enabled = &enabled + } + if s.Compression == "" { + s.Compression = CompressionNone + } } func (s *StreamProcessor) Valid() bool { + s.setDefaults() + ip := net.ParseIP(s.ServerAddr) if ip == nil { log.Fatalln("Invalid IP for stream processor:", s.ServerAddr) @@ -63,10 +117,17 @@ func (s *StreamProcessor) Valid() bool { s.Topic = "certstream" } + // Check that the compression type is generally valid if !s.Compression.Valid() { log.Fatalf("Invalid compression '%s' for stream processor '%s'\n", s.Compression, s.Name) return false } + // Check that the compression type is supported by the stream processing tool + if !s.Compression.SupportedBy(s.Type) { + log.Fatalf("Compression '%s' is not supported by stream processor type '%s' for name '%s'\n", s.Compression, s.Type, s.Name) + return false + } + return true } From 4cf6d25756f3c2382d19990686082f3c16151b60 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 02:27:12 +0200 Subject: [PATCH 61/75] feat: implement custom exponential backoff handler --- internal/backoff/backoff.go | 32 ++++++++++++++++++++++++++++++++ internal/backoff/backoff_test.go | 18 ++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 internal/backoff/backoff.go create mode 100644 internal/backoff/backoff_test.go diff --git a/internal/backoff/backoff.go b/internal/backoff/backoff.go new file mode 100644 index 0000000..afbcb26 --- /dev/null +++ b/internal/backoff/backoff.go @@ -0,0 +1,32 @@ +package backoff + +import ( + "sync" + "time" +) + +// NewBackoff returns a function that implements exponential backoff for the given reset window. +func NewBackoff(resetWindow time.Duration) func(target func()) { + mu := sync.Mutex{} + var errorCount int + var errorLastTime time.Time + errorLogResetWindow := resetWindow + + return func(target func()) { + mu.Lock() + defer mu.Unlock() + now := time.Now() + + // Reset counter if enough time has passed since the last error + if now.Sub(errorLastTime) > errorLogResetWindow { + errorCount = 0 + } + + errorCount++ + errorLastTime = now + + if errorCount == 1 || (errorCount&(errorCount-1)) == 0 { + target() + } + } +} diff --git a/internal/backoff/backoff_test.go b/internal/backoff/backoff_test.go new file mode 100644 index 0000000..1db85e7 --- /dev/null +++ b/internal/backoff/backoff_test.go @@ -0,0 +1,18 @@ +package backoff + +import ( + "testing" + "time" +) + +func TestErrorBackoff(t *testing.T) { + backoffHandler := NewBackoff(1 * time.Second) + + for i := range 20 { + backoffHandler(func() { + test := i + 1 + t.Log("Hello World", test) + time.Sleep(1100 * time.Millisecond) + }) + } +} From 4cbf903f4c7159efa655228f36d789731ca16e5c Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 02:28:07 +0200 Subject: [PATCH 62/75] feat(kafkaclient): improve error handling and logging --- CHANGELOG.md | 1 + internal/broadcast/kafkaclient.go | 117 +++++++++++++++++++++++------- 2 files changed, 92 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5232c3e..db51105 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Support for stream processing tools like kafka or nsq - Option to configure trusted_proxies for which the X-Forwarded-For and X-Real-IP headers are used as client IP (b062edca) - Added multiple unit tests for ensuring correct functionality - Add `excluded_logs` config option to skip specific logs and operators. Works both for classic and tiled logs. diff --git a/internal/broadcast/kafkaclient.go b/internal/broadcast/kafkaclient.go index 763462e..2c965a5 100644 --- a/internal/broadcast/kafkaclient.go +++ b/internal/broadcast/kafkaclient.go @@ -2,16 +2,19 @@ package broadcast import ( "context" + "errors" "log" + "net" "time" + "github.com/d-Rickyy-b/certstream-server-go/internal/backoff" "github.com/segmentio/kafka-go" ) -const ( - maxBatchSize = 50 - maxBatchWait = 60 * time.Second - writeWait = 60 * time.Second +var ( + kafkaMaxBatchSize = 100 + kafkaMaxBatchWait = 1 * time.Second + kafkaConnTimeout = 5 * time.Second ) // KafkaClient connects to a Kafka server in order to provide it with certificates. @@ -27,10 +30,13 @@ type KafkaClient struct { // NewKafkaClient creates a new Kafka client that immediately connects to the configured Kafka server. func NewKafkaClient(subType SubscriptionType, addr, name, topic, compression string, certBufferSize int) *KafkaClient { // Connect to the Kafka server - conn, err := kafka.DialLeader(context.Background(), "tcp", addr, topic, 0) + ctx, cancel := context.WithTimeout(context.Background(), kafkaConnTimeout) + defer cancel() + conn, err := kafka.DialLeader(ctx, "tcp", addr, topic, 0) if err != nil { log.Println("failed to connect to kafka:", err) } + // TODO implement explicit topic creation kc := &KafkaClient{ conn: conn, @@ -44,18 +50,21 @@ func NewKafkaClient(subType SubscriptionType, addr, name, topic, compression str }, } + var kafkaCompression kafka.Compression switch compression { case "gzip": - kc.compression = kafka.Gzip + kafkaCompression = kafka.Gzip case "snappy": - kc.compression = kafka.Snappy + kafkaCompression = kafka.Snappy case "lz4": - kc.compression = kafka.Lz4 - case "none": + kafkaCompression = kafka.Lz4 + case "none", "": default: log.Println("invalid compression type:", compression) } + kc.compression = kafkaCompression + go kc.broadcastHandler() go kc.reconnectHandler() @@ -79,7 +88,9 @@ func (c *KafkaClient) reconnectHandler() { } // Attempt to connect to the Kafka server - conn, err := kafka.DialLeader(context.Background(), "tcp", c.addr, c.topic, 0) + ctx, cancel := context.WithTimeout(context.Background(), kafkaConnTimeout) + defer cancel() + conn, err := kafka.DialLeader(ctx, "tcp", c.addr, c.topic, 0) if err != nil { log.Printf("Reconnect failed: %v. Retrying in 5s...", err) time.Sleep(5 * time.Second) @@ -90,6 +101,7 @@ func (c *KafkaClient) reconnectHandler() { if c.conn != nil { _ = c.conn.Close() } + c.conn = conn c.isConnected = true log.Println("Reconnected to Kafka at", c.addr) @@ -101,23 +113,33 @@ func (c *KafkaClient) reconnectHandler() { func (c *KafkaClient) broadcastHandler() { defer func() { log.Println("Closing broadcast handler for kafka producer:", c.addr) - if err := c.conn.Close(); err != nil { - log.Println("failed to close writer:", err) + if c.conn != nil { + if err := c.conn.Close(); err != nil { + log.Println("failed to close conn:", err) + return + } } ClientHandler.UnregisterClient(c.name) }() - batch := make([]kafka.Message, 0, maxBatchSize) - t := time.NewTimer(maxBatchWait) + backoffHandler := backoff.NewBackoff(60 * time.Second) + batch := make([]kafka.Message, 0, kafkaMaxBatchSize) + t := time.NewTimer(kafkaMaxBatchWait) for { select { case <-c.stopChan: return - case message := <-c.broadcastChan: + case message, ok := <-c.broadcastChan: + if !ok { + log.Println("broadcastChan closed for kafkaClient:", c.addr) + return + } + // Drop messages if not connected if !c.isConnected { + time.Sleep(5 * time.Second) continue } @@ -125,32 +147,75 @@ func (c *KafkaClient) broadcastHandler() { batch = append(batch, msg) // Write batch if it reaches max size - if len(batch) >= maxBatchSize { - c.writeBatch(batch) + if len(batch) >= kafkaMaxBatchSize { + err := c.writeBatch(batch) + if err != nil { + // Without using a backoff strategy, the errors would massively spam the log + backoffHandler(func() { + var netErr *net.OpError + if errors.As(err, &netErr) { + c.isConnected = false + } + + log.Printf("Error writing messages to kafka: %v", err) + }) + } + batch = batch[:0] - t.Reset(maxBatchWait) + t.Reset(kafkaMaxBatchWait) } case <-t.C: + // If batch size has not reached kafkaMaxBatchSize, write the batch after kafkaMaxBatchWait if len(batch) == 0 { continue } - // Write any remaining batch after maxBatchWait - c.writeBatch(batch) + err := c.writeBatch(batch) + if err != nil { + // Without using a backoff strategy, the errors would massively spam the log + backoffHandler(func() { + var netErr *net.OpError + if errors.As(err, &netErr) { + c.isConnected = false + } + + log.Printf("Error writing messages to kafka: %v", err) + }) + } + batch = batch[:0] } } } -func (c *KafkaClient) writeBatch(batch []kafka.Message) { +// writeBatch writes a batch of messages to Kafka, handling exponential backoff and connection state +func (c *KafkaClient) writeBatch(batch []kafka.Message) error { if len(batch) == 0 { - return + return nil + } + + if c.conn == nil || !c.isConnected { + return errors.New("no connection to kafka") } - _ = c.conn.SetWriteDeadline(time.Now().Add(writeWait)) - _, err := c.conn.WriteMessages(batch...) + _ = c.conn.SetWriteDeadline(time.Now().Add(kafkaConnTimeout)) + + _, err := c.conn.WriteCompressedMessages( + c.compression.Codec(), + batch..., + ) if err != nil { - c.isConnected = false - log.Println("Failed to write messages to kafka:", err) + // Treat kafka errors specially + var kafkaErr kafka.Error + if errors.As(err, &kafkaErr) { + if errors.Is(kafkaErr, kafka.MessageSizeTooLarge) { + log.Printf("Message size is too large for kafka broker '%s' - reducing batch size to %d", c.addr, kafkaMaxBatchSize/2) + kafkaMaxBatchSize = kafkaMaxBatchSize / 2 + // TODO: currently there is no retry mechanism implemented. We should try to resend the current batch with the reduced batch size. + } + } + + return err } + return nil } From 748cf7e554edc094f45461060f0e31b3cd69a721 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 02:28:31 +0200 Subject: [PATCH 63/75] chore: add task to run the server --- taskfile.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/taskfile.yml b/taskfile.yml index 3fb3fc3..9140c8f 100644 --- a/taskfile.yml +++ b/taskfile.yml @@ -15,6 +15,11 @@ tasks: cmds: - go test -v ./... + run: + desc: Run the server binary. + cmds: + - go run ./cmd/certstream-server-go + build: desc: Build the server binary for your current OS and architecture. cmds: From 8120312925ebb11cc4736f4d3ddc39267d61c523 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 02:33:08 +0200 Subject: [PATCH 64/75] refactor: set default kafka batch size to 50 This should work well with a default kafka configuration. In the future this could become a config option. --- internal/broadcast/kafkaclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/broadcast/kafkaclient.go b/internal/broadcast/kafkaclient.go index 2c965a5..e0e7ada 100644 --- a/internal/broadcast/kafkaclient.go +++ b/internal/broadcast/kafkaclient.go @@ -12,7 +12,7 @@ import ( ) var ( - kafkaMaxBatchSize = 100 + kafkaMaxBatchSize = 50 kafkaMaxBatchWait = 1 * time.Second kafkaConnTimeout = 5 * time.Second ) From 0ddf935ce8bc40366187d627708dfea1f5fb0afa Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 13:17:30 +0200 Subject: [PATCH 65/75] docs: reformat and update changelog for 1.10.0 --- CHANGELOG.md | 99 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 81 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db51105..7c7ffd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,48 +6,63 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + ### Added + +### Changed + +### Removed + +### Fixed + +## [1.10.0] - 2026-07-xx + +### Added + - Support for stream processing tools like kafka or nsq -- Option to configure trusted_proxies for which the X-Forwarded-For and X-Real-IP headers are used as client IP (b062edca) +- Option to configure trusted_proxies for which the X-Forwarded-For and X-Real-IP headers are used as client IP (b062edc) +- Add `excluded_logs` config option to skip specific logs and operators. Works both for classic and tiled logs. (170fbeb) - Added multiple unit tests for ensuring correct functionality -- Add `excluded_logs` config option to skip specific logs and operators. Works both for classic and tiled logs. ### Changed -- Updated weak cipher suites to stronger ones (939517cd) + +- Updated weak cipher suites to stronger ones (939517c) - Updated http client settings to prevent timeouts and other connectivity issues - Updated http server settings to allow for higher delays -- Minor code improvements and refactoring, mostly style related - Updated batch size from 100 to 256 (#97) -### Removed - ### Fixed -- Calculate correct starting tile index for static ct logs -- Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client -- Respect ct_index path in config file for the `create-index` command -- Ensure all additionalLog entries are processed correctly -### Docs +- Prevent excessive tile fetching for static CT logs (#104) +- Calculate correct starting tile index for static ct logs (a2b4a9f) +- Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client (76ff240) +- Respect ct_index path in config file for the `create-index` command (67838a9) +- Ensure all additionalLog entries are processed correctly (c8bd735) ## [1.9.0] - 2026-04-03 + ### Added + - Ability to store and resume processing of certs from where it left off after a restart - see sample config "recovery" (#49) - New CLI switch for creating an index file from a CT log (#49) -- Support for [Static CT](https://github.com/C2SP/C2SP/blob/main/static-ct-api.md) logs +- Support for [Static CT](https://github.com/C2SP/C2SP/blob/main/static-ct-api.md) logs - Check for retired CT logs and prevent them from being watched / stop watching them (#77) - Accept websocket connections from all origins - Option to disable the default logs provided by Google - see sample config "disable_default_logs" - Use of cobra for CLI argument parsing. New commands for displaying version and creating an index file -- Override confiuration options via environment variables (e.g. `CERTSTREAM_WEBSERVER_LISTEN_PORT=1234` to change the listen port) +- Override configuration options via environment variables (e.g. `CERTSTREAM_WEBSERVER_LISTEN_PORT=1234` to change the listen port) - New field in json `source.timestamp` representing the timestamp of when the cert was processed by the CT log (#98) ### Changed + - **Breaking:** The configuration file for the docker container is now read from the /app/config/ directory (b9e5e6) ### Removed + - Deleted non-functional Dodo log from sample config (#78) ### Fixed + - Properly remove stopped ct log workers (#74) - Added missing fields certificatePolicies and ctlPoisonByte (#85) - Prevent race condition caused by simultaneous rw access to logmetrics (#91) @@ -55,140 +70,188 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Properly add new metrics for all newly found logs (#96) ## [1.8.2] - 2025-11-22 + ### Fixed + - Added missing fields certificatePolicies and ctlPoisonByte (#85) ## [1.8.1] - 2025-05-04 + ### Fixed + - No longer reject URLs with trailing slashes defined in the `additional_logs` config (#62) - When using `drop_old_logs` in the config, the server won't remove logs defined in `additional_logs` anymore (#64) ## [1.8.0] - 2025-05-03 + ### Security + - Close several CVEs in x/crypto and x/net dependencies (#59) ### Added + - New CLI tool for fetching certificates from a CT log (#47) - Ability to add custom CT logs to the config (#56) - Remove old CT logs as soon as they are removed from the Google CT Loglist (#60) - New configuration for buffer sizes (#58) ### Fixed + - Properly handle IPv6 addresses in config (#61) ## [1.7.1] - 2025-05-03 + ### Fixed + - Properly handle IPv6 addresses in config (#61) ## [1.7.0] - 2024-08-20 + ### Added + - Support for websocket compression - disabled by default (#40) - Support for non-browsers by implementing server initiated heartbeats (#39) - Start new ct-watchers as new ct logs become available (#42) - More logging to document currently watched logs (03d878e) - + ### Changed + - Changed log output to be better grepable (5c055cc) - Update ct log update interval to once per hour instead of once per 6 hours as previously (9b6e77d) ### Fixed + - Fixed a possible race condition when accessing metrics ## [1.6.0] - 2024-03-05 + ### Added + - New metric for skipped certs per client (#34) ## [1.5.2] - 2024-02-17 + ### Fixed + - Fixed an issue with ip whitelists for the websocket server (#33) ## [1.5.1] - 2024-01-18 + ### Fixed -- Fixed a rare issue where it was possible for the all_domains json property (or data property in case of the domains-only endpoint) to be null + +- Fixed a rare issue where it was possible for the all_domains json property (or data property in case of the domains-only endpoint) to be null ## [1.5.0] - 2023-12-21 + ### Added + - New `-version` switch to print version and exit afterwards - Print version on every run of the tool - Count and log number of skipped certificates per client ### Changed + - Update to chi/v5 - Update ct-watcher timeout from 5 to 30 seconds ### Fixed + - Prevent invalid subscription types to be used - Kill connection after broadcasthandler was stopped ## [1.4.0] - 2023-11-29 + ### Added + - Config option to use X-Forwarded-For or X-Real-IP header as client IP - Config option to whitelist client IPs for both websocket and metrics endpoints - Config option to enable system metrics (cpu, memory, etc.) ## [1.3.2] - 2023-11-28 + ### Fixed + - Memory leak related to clients disconnecting from the websocket not being handled properly ## [1.3.1] - 2023-09-18 + ### Changed + - Updated config.sample.yaml to run both certstream and prometheus metrics on same socket ### Docs + - Fixed wrong docker command in readme ## [1.3.0] - 2023-04-11 + ### Added + - Calculate and display Sha256 sum of certificate ### Changed + - Update dependencies - Better logging for CT log errors ### Fixed + - End execution after all workers stopped - Implement timeout for the http client - Keep ct watcher from crashing upon a connection reset from server ## [1.2.2] - 2023-01-10 + ### Added + - Two docker-compose files - Check for presence of .yml or .yaml files in the current directory ### Fixed + - Handle sudden disconnects of CT logs ### Docs -- Added [wiki entry for docker-compose](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics) + +- Added [wiki entry for docker-compose](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics) ## [1.2.1] - 2022-12-16 + ### Changed + - Updated ci pipeline to use new setup-go and checkout actions - Use correct package name `github.com/d-Rickyy-b/certstream-server-go` ## [1.2.0] - 2022-12-15 + ### Added + - Log x-Forwarded-For header for requests - More logging for certain error situations - Add operator to ct log cert count metrics ### Changed + - Updated certificate-transparency-go dependency to v1.1.4 - Code improvements, adhering to styleguide - Rename module to certstream-server-go - Use log_list.json instead of all_logs_list.json ## [1.1.0] - 2022-10-19 + Fix for missing loglist urls. ### Fixed + Fixed the connection issue due to the offline Google loglist urls. ## [1.0.0] - 2022-08-08 + Initial release! First stable version of certstream-server-go is published as v1.0.0 -[unreleased]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.9.0...HEAD -[1.8.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.2...v1.9.0 +[unreleased]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.10.0...HEAD +[1.10.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.9.0...v1.10.0 +[1.9.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.2...v1.9.0 [1.8.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.1...v1.8.2 [1.8.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.0...v1.8.1 [1.8.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.7.1...v1.8.0 From b5b5d11fd439badf54defa229673de811a93e8a3 Mon Sep 17 00:00:00 2001 From: Rico Date: Mon, 13 Jul 2026 13:40:39 +0200 Subject: [PATCH 66/75] refactor: rename broadcastmanager to dispatcher --- internal/broadcast/{broadcastmanager.go => dispatcher.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/broadcast/{broadcastmanager.go => dispatcher.go} (100%) diff --git a/internal/broadcast/broadcastmanager.go b/internal/broadcast/dispatcher.go similarity index 100% rename from internal/broadcast/broadcastmanager.go rename to internal/broadcast/dispatcher.go From 0423f616e3c3813a19b6037cb39b71771c242a3a Mon Sep 17 00:00:00 2001 From: Filippo's Agent Date: Wed, 9 Sep 2026 19:14:03 +0200 Subject: [PATCH 67/75] fix(static-ct): make ctIndex consistently the next entry to process StaticCTClient.ctIndex was initialized to the checkpoint tree size (the number of entries already seen) but then updated to the index of the last processed entry, and the skip check treated it as the latter. The two conventions are off by one, with three visible effects: - Once caught up, currentTreeSize is always ctIndex+1, so the "no new entries" early return never fires. Every poll falls through to the partial tile logic, re-fetching a tile whose entries were all already processed. Before the deferral logic this happened on every poll, keeping the backoff reset; with the deferral logic it still happens once per maxPartialWait, forever, on idle logs. - The first entry after a startup at the checkpoint size (index N when ctIndex was set to N) is skipped, since N <= N. - Likewise, starting from a saved index of 0 skips entry 0. Make ctIndex mean "index of the next entry to process" throughout: skip entries with index < ctIndex, advance to entryIndex+1 after processing, and start the full tile loop at ctIndex/TileSize. With that, the tree size comparison is exact, a caught-up client does only the checkpoint fetch per poll, and no entry is dropped at startup. A saved recovery index written by the metrics package holds the last processed entry index, so after this change resuming from it re-emits that one entry, matching the behavior of the RFC 6962 scanner path (which also uses the saved index as StartIndex). Add tests that drive the client against an in-process static log and check that entries are emitted exactly once, that an idle log causes no tile fetches, and that the partial tile deferral still works. Updates #104 --- internal/certificatetransparency/ct-tiled.go | 6 +- .../certificatetransparency/ct-tiled_test.go | 265 ++++++++++++++++++ 2 files changed, 268 insertions(+), 3 deletions(-) diff --git a/internal/certificatetransparency/ct-tiled.go b/internal/certificatetransparency/ct-tiled.go index 8a3cc84..872e912 100644 --- a/internal/certificatetransparency/ct-tiled.go +++ b/internal/certificatetransparency/ct-tiled.go @@ -312,7 +312,7 @@ func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert, fo } // Process entries from current index to new tree size - startTile := (s.ctIndex + 1) / TileSize + startTile := s.ctIndex / TileSize endTile := currentTreeSize / TileSize // Process full tiles @@ -380,7 +380,7 @@ func (s *StaticCTClient) processTile(ctx context.Context, tileIndex, partialWidt entryIndex := baseIndex + uint64(i) // Skip entries we've already processed - if entryIndex <= s.ctIndex { + if entryIndex < s.ctIndex { continue } @@ -398,7 +398,7 @@ func (s *StaticCTClient) processTile(ctx context.Context, tileIndex, partialWidt } // Update the index - s.ctIndex = entryIndex + s.ctIndex = entryIndex + 1 } return nil diff --git a/internal/certificatetransparency/ct-tiled_test.go b/internal/certificatetransparency/ct-tiled_test.go index 4c246d1..4d78758 100644 --- a/internal/certificatetransparency/ct-tiled_test.go +++ b/internal/certificatetransparency/ct-tiled_test.go @@ -1,7 +1,17 @@ package certificatetransparency import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" "testing" + + ct "github.com/google/certificate-transparency-go" + "golang.org/x/crypto/cryptobyte" ) func Test_encodeTilePath(t *testing.T) { @@ -66,3 +76,258 @@ func Test_encodeTilePath(t *testing.T) { }) } } + +// fakeTiledLog serves a minimal static CT log (checkpoint and data tiles) and +// counts the requests it receives per path. +type fakeTiledLog struct { + t *testing.T + mu sync.Mutex + size uint64 + requests map[string]int +} + +func newFakeTiledLog(t *testing.T, size uint64) (*fakeTiledLog, *httptest.Server) { + t.Helper() + + l := &fakeTiledLog{t: t, size: size, requests: make(map[string]int)} + srv := httptest.NewServer(l) + t.Cleanup(srv.Close) + + return l, srv +} + +func (l *fakeTiledLog) grow(n uint64) { + l.mu.Lock() + defer l.mu.Unlock() + l.size += n +} + +func (l *fakeTiledLog) tileRequests() int { + l.mu.Lock() + defer l.mu.Unlock() + + n := 0 + + for path, count := range l.requests { + if strings.HasPrefix(path, "/tile/") { + n += count + } + } + + return n +} + +func (l *fakeTiledLog) ServeHTTP(w http.ResponseWriter, r *http.Request) { + l.mu.Lock() + defer l.mu.Unlock() + + l.requests[r.URL.Path]++ + + if r.URL.Path == "/checkpoint" { + fmt.Fprintf(w, "example.com/log\n%d\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\n", l.size) + return + } + + tilePath, ok := strings.CutPrefix(r.URL.Path, "/tile/data/") + if !ok { + http.NotFound(w, r) + return + } + + width := uint64(TileSize) + + if base, p, found := strings.Cut(tilePath, ".p/"); found { + tilePath = base + + var err error + + width, err = strconv.ParseUint(p, 10, 64) + if err != nil { + http.NotFound(w, r) + return + } + } + + var tileIndex uint64 + + for tileIndex = 0; encodeTilePath(tileIndex) != tilePath; tileIndex++ { + if tileIndex > l.size/TileSize { + http.NotFound(w, r) + return + } + } + + first := tileIndex * TileSize + if first+width > l.size || (width < TileSize && first+width != l.size) { + http.NotFound(w, r) + return + } + + var b cryptobyte.Builder + + for i := first; i < first+width; i++ { + b.AddUint64(i) // timestamp, set to the index for easy checking + b.AddUint16(EntryTypeCert) // entry type + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { b.AddBytes([]byte{0x30, 0x00}) }) + b.AddUint16LengthPrefixed(func(*cryptobyte.Builder) {}) + b.AddUint16LengthPrefixed(func(*cryptobyte.Builder) {}) + } + + data, err := b.Bytes() + if err != nil { + l.t.Errorf("building tile: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + + _, _ = w.Write(data) +} + +// runPoll performs a single fetchAndProcessTiles call and returns the indexes +// of the entries it emitted. +func runPoll(t *testing.T, c *StaticCTClient) (indexes []uint64, hadNewEntries bool) { + t.Helper() + + found := func(e *ct.RawLogEntry) { + if uint64(e.Index) != e.Leaf.TimestampedEntry.Timestamp { + t.Errorf("entry index %d does not match tile content %d", e.Index, e.Leaf.TimestampedEntry.Timestamp) + } + + indexes = append(indexes, uint64(e.Index)) + } + + hadNewEntries, err := c.fetchAndProcessTiles(context.Background(), found, found) + if err != nil { + t.Fatalf("fetchAndProcessTiles: %v", err) + } + + return indexes, hadNewEntries +} + +func expectRange(t *testing.T, got []uint64, start, end uint64) { + t.Helper() + + if uint64(len(got)) != end-start { + t.Fatalf("got %d entries, want %d (indexes %v)", len(got), end-start, got) + } + + for i, idx := range got { + if idx != start+uint64(i) { + t.Fatalf("entry %d has index %d, want %d", i, idx, start+uint64(i)) + } + } +} + +func TestStaticCTClient_NoRefetchWhenIdle(t *testing.T) { + log, srv := newFakeTiledLog(t, 300) + c := NewStaticCTClient(srv.URL, srv.Client(), "test", 0) + // A partial tile is always deferred the first time it is observed, and + // fetched on the next poll once the deferral window has expired. + c.maxPartialWait = 0 + + indexes, hadNewEntries := runPoll(t, c) + expectRange(t, indexes, 0, 256) + + if !hadNewEntries { + t.Error("first poll should report new entries") + } + + indexes, _ = runPoll(t, c) + expectRange(t, indexes, 256, 300) + + if c.ctIndex != 300 { + t.Errorf("ctIndex = %d, want 300", c.ctIndex) + } + + tileRequests := log.tileRequests() + + // The log has not grown: no tile should be fetched again, and no entry + // should be emitted again. + for range 3 { + indexes, hadNewEntries = runPoll(t, c) + expectRange(t, indexes, 0, 0) + + if hadNewEntries { + t.Error("poll on an idle log should not report new entries") + } + } + + if got := log.tileRequests(); got != tileRequests { + t.Errorf("idle polls fetched %d tiles, want 0", got-tileRequests) + } +} + +func TestStaticCTClient_StartAtCheckpointSize(t *testing.T) { + log, srv := newFakeTiledLog(t, 300) + c := NewStaticCTClient(srv.URL, srv.Client(), "test", 0) + c.maxPartialWait = 0 + // Like runTiledWorker without recovery, start at the current tree size. + c.ctIndex = 300 + + indexes, _ := runPoll(t, c) + expectRange(t, indexes, 0, 0) + + // The first new entry after startup must not be skipped. The partial tile + // is deferred once, then fetched on the next poll. + log.grow(1) + runPoll(t, c) + indexes, _ = runPoll(t, c) + expectRange(t, indexes, 300, 301) + + log.grow(2) + runPoll(t, c) + indexes, _ = runPoll(t, c) + expectRange(t, indexes, 301, 303) +} + +func TestStaticCTClient_DeferPartialTiles(t *testing.T) { + log, srv := newFakeTiledLog(t, 256) + c := NewStaticCTClient(srv.URL, srv.Client(), "test", 256) + + // A new partial tile is observed but not fetched within the deferral window. + log.grow(10) + indexes, hadNewEntries := runPoll(t, c) + expectRange(t, indexes, 0, 0) + + if hadNewEntries { + t.Error("deferred partial should not report new entries") + } + + if got := log.tileRequests(); got != 0 { + t.Errorf("fetched %d tiles during deferral, want 0", got) + } + + // Once the tile fills up, the full tile is fetched and every entry is + // emitted exactly once. + log.grow(246) + indexes, hadNewEntries = runPoll(t, c) + expectRange(t, indexes, 256, 512) + + if !hadNewEntries { + t.Error("full tile fetch should report new entries") + } + + if got := log.tileRequests(); got != 1 { + t.Errorf("fetched %d tiles, want 1", got) + } + + // When the deferral window expires, the partial is fetched once, and then + // not again until the log grows. + log.grow(5) + runPoll(t, c) // starts the deferral clock + c.partialTileFirstSeen = c.partialTileFirstSeen.Add(-2 * DefaultMaxPartialWait) + indexes, _ = runPoll(t, c) + expectRange(t, indexes, 512, 517) + + tileRequests := log.tileRequests() + + for range 3 { + indexes, _ = runPoll(t, c) + expectRange(t, indexes, 0, 0) + } + + if got := log.tileRequests(); got != tileRequests { + t.Errorf("idle polls fetched %d tiles, want 0", got-tileRequests) + } +} From 783a3a3037c36cd3d368dcef0c1bab707b06b4c9 Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 10 Sep 2026 02:05:17 +0200 Subject: [PATCH 68/75] chore: bump version to 1.10.0 --- internal/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/config.go b/internal/config/config.go index 30a3505..ae59ee8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -13,7 +13,7 @@ import ( var ( // AppConfig holds the parsed configuration. AppConfig Config - Version = "1.9.0" + Version = "1.10.0" ErrInvalidConfig = errors.New("invalid configuration") URLPathRegex = regexp.MustCompile(`^(/[a-zA-Z0-9\-._]+)+$`) From ed0c4718b7d6aec27fa0ce50f8254db0f19523bc Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 10 Sep 2026 02:08:29 +0200 Subject: [PATCH 69/75] refactor: simplify code for easier extendability --- internal/certstream/certstream.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index 01cda11..aa4ab9a 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -75,20 +75,20 @@ func NewCertstreamServer(cfg config.Config) (*Certstream, error) { subscriptionType = broadcast.SubTypeDomain } + var c broadcast.CertProcessor switch streamProcessor.Type { case "nsq": log.Println("Initializing NSQ client...") - nc := broadcast.NewNSQClient( + c = broadcast.NewNSQClient( subscriptionType, addr, streamProcessor.Name, streamProcessor.Topic, cfg.General.BufferSizes.Websocket, ) - broadcast.ClientHandler.RegisterClient(nc) case "kafka": log.Println("Initializing Kafka client...") - kc := broadcast.NewKafkaClient( + c = broadcast.NewKafkaClient( subscriptionType, addr, streamProcessor.Name, @@ -101,6 +101,9 @@ func NewCertstreamServer(cfg config.Config) (*Certstream, error) { default: log.Printf("Unknown stream processor type '%s' for %s. Skipping...\n", streamProcessor.Type, streamProcessor.Name) } + + log.Println(c) + broadcast.ClientHandler.RegisterClient(c) } return cs, nil From 750ed24d5554d15894a5d4b11a449d39f06eebd3 Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 10 Sep 2026 03:06:15 +0200 Subject: [PATCH 70/75] fix: issue regarding disconnecting websocket clients Enhances client robustness by: - Making client resource cleanup safe from panics when internal channels are nil or closed repeatedly. - Updating the unregistration process to use client objects directly for more reliable removal. - Ensuring client internal channels are consistently initialized. --- internal/broadcast/baseclient.go | 17 ++++++++-- internal/broadcast/baseclient_test.go | 16 ++++++++++ internal/broadcast/dispatcher.go | 4 +-- internal/broadcast/dispatcher_test.go | 45 +++++++++++++++++++++++++++ internal/broadcast/kafkaclient.go | 2 +- internal/broadcast/nsqclient.go | 2 +- internal/broadcast/websocketclient.go | 3 +- 7 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 internal/broadcast/baseclient_test.go create mode 100644 internal/broadcast/dispatcher_test.go diff --git a/internal/broadcast/baseclient.go b/internal/broadcast/baseclient.go index 9b69e3e..c838ffc 100644 --- a/internal/broadcast/baseclient.go +++ b/internal/broadcast/baseclient.go @@ -1,6 +1,9 @@ package broadcast -import "log" +import ( + "log" + "sync" +) // BaseClient defines the basic structure for a client that can receive broadcast messages. // Other client types can embed this struct to inherit its functionality. @@ -10,12 +13,20 @@ type BaseClient struct { name string subType SubscriptionType skippedCerts uint64 + closeOnce sync.Once } // Close cleans up the client's resources by closing the stop and broadcast channels. func (c *BaseClient) Close() { - close(c.stopChan) - close(c.broadcastChan) + c.closeOnce.Do(func() { + if c.stopChan != nil { + close(c.stopChan) + } + + if c.broadcastChan != nil { + close(c.broadcastChan) + } + }) } // Name returns the name of the client. diff --git a/internal/broadcast/baseclient_test.go b/internal/broadcast/baseclient_test.go new file mode 100644 index 0000000..f1508b9 --- /dev/null +++ b/internal/broadcast/baseclient_test.go @@ -0,0 +1,16 @@ +package broadcast + +import "testing" + +func TestBaseClientCloseIsSafeForNilChannelsAndRepeatedCalls(t *testing.T) { + client := &BaseClient{} + + defer func() { + if r := recover(); r != nil { + t.Fatalf("BaseClient.Close() panicked: %v", r) + } + }() + + client.Close() + client.Close() +} diff --git a/internal/broadcast/dispatcher.go b/internal/broadcast/dispatcher.go index b1a7fb6..df8d1a7 100644 --- a/internal/broadcast/dispatcher.go +++ b/internal/broadcast/dispatcher.go @@ -43,11 +43,11 @@ func (bm *Dispatcher) RegisterClient(c CertProcessor) { // UnregisterClient removes a client from the list of clients of the Dispatcher. // The client will no longer receive certificate broadcasts right after unregistering. -func (bm *Dispatcher) UnregisterClient(clientName string) { +func (bm *Dispatcher) UnregisterClient(clientToRemove CertProcessor) { bm.clientLock.Lock() for i, client := range bm.clients { - if clientName == client.Name() { + if clientToRemove == client { // Copy the last element of the slice to the position of the removed element // Then remove the last element by re-slicing bm.clients[i] = bm.clients[len(bm.clients)-1] diff --git a/internal/broadcast/dispatcher_test.go b/internal/broadcast/dispatcher_test.go new file mode 100644 index 0000000..17bf2f6 --- /dev/null +++ b/internal/broadcast/dispatcher_test.go @@ -0,0 +1,45 @@ +package broadcast + +import "testing" + +func newDispatcherTestWebsocketClient(name string) *WebsocketClient { + return &WebsocketClient{ + hostIP: "127.0.0.1", + hostPort: "12345", + BaseClient: &BaseClient{ + broadcastChan: make(chan []byte, 1), + name: name, + subType: SubTypeFull, + }, + } +} + +func TestDispatcherUnregisterWebsocketClient(t *testing.T) { + d := &Dispatcher{} + client := newDispatcherTestWebsocketClient("client-1") + + d.RegisterClient(client) + if got := len(d.clients); got != 1 { + t.Fatalf("expected 1 registered client, got %d", got) + } + + d.UnregisterClient(client) + if got := len(d.clients); got != 0 { + t.Fatalf("expected 0 registered clients after unregister, got %d", got) + } +} + +func TestDispatcherUnregisterWebsocketClientDoesNotPanicOnNilStopChan(t *testing.T) { + d := &Dispatcher{} + client := newDispatcherTestWebsocketClient("client-2") + + d.RegisterClient(client) + + defer func() { + if r := recover(); r != nil { + t.Fatalf("unregistering websocket client panicked: %v", r) + } + }() + + d.UnregisterClient(client) +} diff --git a/internal/broadcast/kafkaclient.go b/internal/broadcast/kafkaclient.go index e0e7ada..ed711de 100644 --- a/internal/broadcast/kafkaclient.go +++ b/internal/broadcast/kafkaclient.go @@ -120,7 +120,7 @@ func (c *KafkaClient) broadcastHandler() { } } - ClientHandler.UnregisterClient(c.name) + ClientHandler.UnregisterClient(c) }() backoffHandler := backoff.NewBackoff(60 * time.Second) diff --git a/internal/broadcast/nsqclient.go b/internal/broadcast/nsqclient.go index f8e042e..5b809ea 100644 --- a/internal/broadcast/nsqclient.go +++ b/internal/broadcast/nsqclient.go @@ -106,7 +106,7 @@ func (c *NSQClient) broadcastHandler() { c.conn.Stop() } - ClientHandler.UnregisterClient(c.name) + ClientHandler.UnregisterClient(c) }() batch := make([][]byte, 0, nsqMaxBatchSize) diff --git a/internal/broadcast/websocketclient.go b/internal/broadcast/websocketclient.go index 29b913e..4e30063 100644 --- a/internal/broadcast/websocketclient.go +++ b/internal/broadcast/websocketclient.go @@ -34,6 +34,7 @@ func NewWebsocketClient(conn *websocket.Conn, subType SubscriptionType, name, us realIPFromHeader: realIPFromHeader, BaseClient: &BaseClient{ broadcastChan: make(chan []byte, certBufferSize), + stopChan: make(chan struct{}), name: name, subType: subType, }, @@ -106,7 +107,7 @@ func (c *WebsocketClient) broadcastHandler() { func (c *WebsocketClient) listenWebsocket() { defer func() { _ = c.conn.Close() - ClientHandler.UnregisterClient(c.name) + ClientHandler.UnregisterClient(c) }() readWait := 65 * time.Second From 2d4f132943f15d1ef2cf10854813a1bab72f7bb1 Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 10 Sep 2026 03:07:09 +0200 Subject: [PATCH 71/75] refactor: rename variable name to better describe its function --- internal/certificatetransparency/ct-watcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index 6a271eb..8128d6b 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -481,8 +481,8 @@ func (w *worker) runTiledWorker(ctx context.Context) error { staticCTClient := NewStaticCTClient(w.ctURL, httpClient, UserAgent, w.ctIndex) // If recovery is enabled and the CT index is set, we start at the saved index. Otherwise, we start at the latest checkpoint. - validSavedCTIndexExists := config.AppConfig.General.Recovery.Enabled - if !validSavedCTIndexExists { + recoveryEnabled := config.AppConfig.General.Recovery.Enabled + if !recoveryEnabled { checkpoint, err := staticCTClient.FetchCheckpoint(ctx) if err != nil { log.Printf("Could not get checkpoint for '%s': %s\n", w.ctURL, err) From 85e90bf0c5d17b5d8022ff84257c419a31a057b6 Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 10 Sep 2026 03:19:57 +0200 Subject: [PATCH 72/75] fix: reload last CTIndex for restarted workers Without this, the worker would start at the last known ctIndex from when the worker was initially created as the ctIndex value does not get updated at runtime, since we're using the scanner implementation and our callbacks don't increment the ctIndex. This fix prevents processing already processed certificates again, after a worker crashed. --- internal/certificatetransparency/ct-watcher.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index 8128d6b..bee9e08 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -373,6 +373,10 @@ func (w *worker) startDownloadingCerts(ctx context.Context) { w.mu.Unlock() for { + // reload the CT index from metrics before restarting the worker + lastCTIndex := metrics.Metrics.GetCTIndex(normalizeCtlogURL(w.ctURL)) + w.ctIndex = lastCTIndex + log.Printf("Starting worker for CT log: %s\n", w.ctURL) var workerErr error From 1ec776cf9f8a48e418ff2ebb5c6ab03b7abc06e8 Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 10 Sep 2026 03:20:51 +0200 Subject: [PATCH 73/75] docs: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c7ffd9..37fa399 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client (76ff240) - Respect ct_index path in config file for the `create-index` command (67838a9) - Ensure all additionalLog entries are processed correctly (c8bd735) +- Reload CT index from metrics before restarting the worker (85e90bf) ## [1.9.0] - 2026-04-03 From fc1b2638448a1c494c20ceca3be8e7df1d0d675b Mon Sep 17 00:00:00 2001 From: Rico Date: Thu, 10 Sep 2026 03:29:21 +0200 Subject: [PATCH 74/75] fix: forgot to rename this variable --- internal/certstream/certstream.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/certstream/certstream.go b/internal/certstream/certstream.go index aa4ab9a..fef83a2 100644 --- a/internal/certstream/certstream.go +++ b/internal/certstream/certstream.go @@ -97,7 +97,7 @@ func NewCertstreamServer(cfg config.Config) (*Certstream, error) { cfg.General.BufferSizes.Websocket, ) - broadcast.ClientHandler.RegisterClient(kc) + broadcast.ClientHandler.RegisterClient(c) default: log.Printf("Unknown stream processor type '%s' for %s. Skipping...\n", streamProcessor.Type, streamProcessor.Name) } From 38e1529213af0aff3c31b5c946752553f4cda2a5 Mon Sep 17 00:00:00 2001 From: Rico Date: Sun, 13 Sep 2026 01:45:46 +0200 Subject: [PATCH 75/75] docs(changelog): add latest 1.10 related changes --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37fa399..c153231 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -## [1.10.0] - 2026-07-xx +## [1.10.0] - 2026-09-13 ### Added @@ -34,11 +34,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Prevent excessive tile fetching for static CT logs (#104) +- Fix off-by-one is Static CT path causing refetches of non-growing logs by @FiloSottile in https://github.com/d-Rickyy-b/certstream-server-go/pull/114 - Calculate correct starting tile index for static ct logs (a2b4a9f) - Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client (76ff240) - Respect ct_index path in config file for the `create-index` command (67838a9) - Ensure all additionalLog entries are processed correctly (c8bd735) -- Reload CT index from metrics before restarting the worker (85e90bf) +- Reload CT index from metrics before restarting the worker (85e90bf, #113) ## [1.9.0] - 2026-04-03