Merge changes for 1.10 into master branch

.github/goreleaser.yml

@@ -15,23 +15,23 @@ builds:
       - darwin
       - windows
     goarch:
-      - 386
+      - "386"
       - amd64
       - arm
       - arm64
     ignore:
       - goos: darwin
-        goarch: 386
+        goarch: "386"
       - goos: darwin
         goarch: arm
       - goos: windows
         goarch: arm
       - goos: windows
         goarch: arm64
       - goos: windows
-        goarch: 386
+        goarch: "386"
 checksum:
-  name_template: '{{.ProjectName}}_{{.Version}}_checksums.txt'
+  name_template: "{{.ProjectName}}_{{.Version}}_checksums.txt"
 changelog:
   disable: true
 
@@ -41,8 +41,8 @@ dockers_v2:
       - "ghcr.io/d-rickyy-b/{{.ProjectName}}"
     dockerfile: docker/Dockerfile.goreleaser
     tags:
-        - "{{.Tag}}"
-        - "{{ if not .Prerelease }}latest{{ end }}"
+      - '{{ if eq .Branch "dev" }}dev{{ else }}{{ .Tag }}{{ end }}'
+      - '{{ if and (ne .Branch "dev") (not .Prerelease) }}latest{{ end }}'
     platforms:
       - linux/amd64
       - linux/arm64
@@ -58,7 +58,7 @@ dockers_v2:
       "org.opencontainers.image.source": "https://github.com/d-Rickyy-b/certstream-server-go"
 
 archives:
-  - formats: [ "binary" ]
+  - formats: ["binary"]
     name_template: >-
       {{- .ProjectName }}_
       {{- .Version}}_

.github/workflows/release_build.yml

@@ -4,6 +4,8 @@ on:
   push:
     tags:
       - "*"
+    branches:
+      - dev
 
 jobs:
   build:
@@ -15,7 +17,7 @@ jobs:
 
     steps:
       - name: Set up Go 1.22
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: ^1.22
         id: go
@@ -45,9 +47,19 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run GoReleaser
+        if: startsWith(github.ref, 'refs/tags/')
         uses: goreleaser/goreleaser-action@v7
         with:
           version: latest
           args: release --clean --config .github/goreleaser.yml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run GoReleaser (dev snapshot)
+        if: github.ref == 'refs/heads/dev'
+        uses: goreleaser/goreleaser-action@v7
+        with:
+          version: latest
+          args: release --snapshot --clean --config .github/goreleaser.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.golangci.yml

@@ -36,7 +36,7 @@ linters:
     - godot
     # - godox
     - gomoddirectives
-    - gomodguard
+    - gomodguard_v2
     - goprintffuncname
     - gosec
     - govet
@@ -109,6 +109,7 @@ linters:
         - err error
         - ctx context.Context
         - v *viper.Viper
+        - v uint64
         - c *client
         - cs *Certstream
         - k x509.KeyUsage
@@ -136,6 +137,8 @@ linters:
           - varnamelen
           - goconst
           - err113
+          - gosec
+          - dupl
         path: _test\.go
       - linters:
           - gochecknoinits

CHANGELOG.md

@@ -6,178 +6,254 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+### Added
+
+### Changed
+
+### Removed
+
+### Fixed
+
+## [1.10.0] - 2026-09-13
+
 ### Added
+
+- Support for stream processing tools like kafka or nsq
+- Option to configure trusted_proxies for which the X-Forwarded-For and X-Real-IP headers are used as client IP (b062edc)
+- Add `excluded_logs` config option to skip specific logs and operators. Works both for classic and tiled logs. (170fbeb)
 - Added multiple unit tests for ensuring correct functionality
+
 ### Changed
-- Updated weak cipher suites to stronger ones (939517cd)
+
+- Updated weak cipher suites to stronger ones (939517c)
 - Updated http client settings to prevent timeouts and other connectivity issues
 - Updated http server settings to allow for higher delays
-- Minor code improvements and refactoring, mostly style related
-### Removed
+- Updated batch size from 100 to 256 (#97)
+
 ### Fixed
-- Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client
-### Docs
+
+- Prevent excessive tile fetching for static CT logs (#104)
+- Fix off-by-one is Static CT path causing refetches of non-growing logs by @FiloSottile in https://github.com/d-Rickyy-b/certstream-server-go/pull/114
+- Calculate correct starting tile index for static ct logs (a2b4a9f)
+- Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client (76ff240)
+- Respect ct_index path in config file for the `create-index` command (67838a9)
+- Ensure all additionalLog entries are processed correctly (c8bd735)
+- Reload CT index from metrics before restarting the worker (85e90bf, #113)
 
 ## [1.9.0] - 2026-04-03
+
 ### Added
+
 - Ability to store and resume processing of certs from where it left off after a restart - see sample config "recovery" (#49)
 - New CLI switch for creating an index file from a CT log (#49)
-- Support for [Static CT](https://github.com/C2SP/C2SP/blob/main/static-ct-api.md) logs 
+- Support for [Static CT](https://github.com/C2SP/C2SP/blob/main/static-ct-api.md) logs
 - Check for retired CT logs and prevent them from being watched / stop watching them (#77)
 - Accept websocket connections from all origins
 - Option to disable the default logs provided by Google - see sample config "disable_default_logs"
 - Use of cobra for CLI argument parsing. New commands for displaying version and creating an index file
-- Override confiuration options via environment variables (e.g. `CERTSTREAM_WEBSERVER_LISTEN_PORT=1234` to change the listen port)
+- Override configuration options via environment variables (e.g. `CERTSTREAM_WEBSERVER_LISTEN_PORT=1234` to change the listen port)
 - New field in json `source.timestamp` representing the timestamp of when the cert was processed by the CT log (#98)
 
 ### Changed
+
 - **Breaking:** The configuration file for the docker container is now read from the /app/config/ directory (b9e5e6)
 
 ### Removed
+
 - Deleted non-functional Dodo log from sample config (#78)
 
 ### Fixed
+
 - Properly remove stopped ct log workers (#74)
 - Added missing fields certificatePolicies and ctlPoisonByte (#85)
 - Prevent race condition caused by simultaneous rw access to logmetrics (#91)
 - Properly display metrics for all initially watched logs (#95)
 - Properly add new metrics for all newly found logs (#96)
 
 ## [1.8.2] - 2025-11-22
+
 ### Fixed
+
 - Added missing fields certificatePolicies and ctlPoisonByte (#85)
 
 ## [1.8.1] - 2025-05-04
+
 ### Fixed
+
 - No longer reject URLs with trailing slashes defined in the `additional_logs` config (#62)
 - When using `drop_old_logs` in the config, the server won't remove logs defined in `additional_logs` anymore (#64)
 
 ## [1.8.0] - 2025-05-03
+
 ### Security
+
 - Close several CVEs in x/crypto and x/net dependencies (#59)
 
 ### Added
+
 - New CLI tool for fetching certificates from a CT log (#47)
 - Ability to add custom CT logs to the config (#56)
 - Remove old CT logs as soon as they are removed from the Google CT Loglist (#60)
 - New configuration for buffer sizes (#58)
 
 ### Fixed
+
 - Properly handle IPv6 addresses in config (#61)
 
 ## [1.7.1] - 2025-05-03
+
 ### Fixed
+
 - Properly handle IPv6 addresses in config (#61)
 
 ## [1.7.0] - 2024-08-20
+
 ### Added
+
 - Support for websocket compression - disabled by default (#40)
 - Support for non-browsers by implementing server initiated heartbeats (#39)
 - Start new ct-watchers as new ct logs become available (#42)
 - More logging to document currently watched logs (03d878e)
- 
+
 ### Changed
+
 - Changed log output to be better grepable (5c055cc)
 - Update ct log update interval to once per hour instead of once per 6 hours as previously (9b6e77d)
 
 ### Fixed
+
 - Fixed a possible race condition when accessing metrics
 
 ## [1.6.0] - 2024-03-05
+
 ### Added
+
 - New metric for skipped certs per client (#34)
 
 ## [1.5.2] - 2024-02-17
+
 ### Fixed
+
 - Fixed an issue with ip whitelists for the websocket server (#33)
 
 ## [1.5.1] - 2024-01-18
+
 ### Fixed
-- Fixed a rare issue where it was possible for the all_domains json property (or data property in case of the domains-only endpoint) to be null 
+
+- Fixed a rare issue where it was possible for the all_domains json property (or data property in case of the domains-only endpoint) to be null
 
 ## [1.5.0] - 2023-12-21
+
 ### Added
+
 - New `-version` switch to print version and exit afterwards
 - Print version on every run of the tool
 - Count and log number of skipped certificates per client
 
 ### Changed
+
 - Update to chi/v5
 - Update ct-watcher timeout from 5 to 30 seconds
 
 ### Fixed
+
 - Prevent invalid subscription types to be used
 - Kill connection after broadcasthandler was stopped
 
 ## [1.4.0] - 2023-11-29
+
 ### Added
+
 - Config option to use X-Forwarded-For or X-Real-IP header as client IP
 - Config option to whitelist client IPs for both websocket and metrics endpoints
 - Config option to enable system metrics (cpu, memory, etc.)
 
 ## [1.3.2] - 2023-11-28
+
 ### Fixed
+
 - Memory leak related to clients disconnecting from the websocket not being handled properly
 
 ## [1.3.1] - 2023-09-18
+
 ### Changed
+
 - Updated config.sample.yaml to run both certstream and prometheus metrics on same socket
 
 ### Docs
+
 - Fixed wrong docker command in readme
 
 ## [1.3.0] - 2023-04-11
+
 ### Added
+
 - Calculate and display Sha256 sum of certificate
 
 ### Changed
+
 - Update dependencies
 - Better logging for CT log errors
 
 ### Fixed
+
 - End execution after all workers stopped
 - Implement timeout for the http client
 - Keep ct watcher from crashing upon a connection reset from server
 
 ## [1.2.2] - 2023-01-10
+
 ### Added
+
 - Two docker-compose files
 - Check for presence of .yml or .yaml files in the current directory
 
 ### Fixed
+
 - Handle sudden disconnects of CT logs
 
 ### Docs
-- Added [wiki entry for docker-compose](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics) 
+
+- Added [wiki entry for docker-compose](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics)
 
 ## [1.2.1] - 2022-12-16
+
 ### Changed
+
 - Updated ci pipeline to use new setup-go and checkout actions
 - Use correct package name `github.com/d-Rickyy-b/certstream-server-go`
 
 ## [1.2.0] - 2022-12-15
+
 ### Added
+
 - Log x-Forwarded-For header for requests
 - More logging for certain error situations
 - Add operator to ct log cert count metrics
 
 ### Changed
+
 - Updated certificate-transparency-go dependency to v1.1.4
 - Code improvements, adhering to styleguide
 - Rename module to certstream-server-go
 - Use log_list.json instead of all_logs_list.json
 
 ## [1.1.0] - 2022-10-19
+
 Fix for missing loglist urls.
 
 ### Fixed
+
 Fixed the connection issue due to the offline Google loglist urls.
 
 ## [1.0.0] - 2022-08-08
+
 Initial release! First stable version of certstream-server-go is published as v1.0.0
 
-[unreleased]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.9.0...HEAD
-[1.8.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.2...v1.9.0
+[unreleased]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.10.0...HEAD
+[1.10.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.9.0...v1.10.0
+[1.9.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.2...v1.9.0
 [1.8.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.1...v1.8.2
 [1.8.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.0...v1.8.1
 [1.8.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.7.1...v1.8.0

cmd/certpicker/main.go

@@ -65,7 +65,7 @@ func main() {
 		log.Fatalln("Error getting entry from CT log: ", getEntryErr)
 	}
 
-	// Loop over entries and pars each one.
+	// Loop over entries and parse each one.
 	for _, leafEntry := range entries.Entries {
 		rawLogEntry, err := ct.RawLogEntryFromLeaf(certID, &leafEntry)
 		if err != nil {