feat(ct): parse names tiles

internal/certificatetransparency/ct-names-tile.go

@@ -0,0 +1,39 @@
+package certificatetransparency
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+)
+
+// NamesEntry is a single entry of a names tile.
+// See https://github.com/FiloSottile/sunlight/blob/main/names-tiles.md
+type NamesEntry struct {
+	// Timestamp in milliseconds.
+	Timestamp int64 `json:"Timestamp"`
+	Subject   struct {
+		CommonName string `json:"CommonName"`
+	} `json:"Subject"`
+	DNS []string `json:"DNS"`
+	IP  []string `json:"IP"`
+}
+
+// ParseNamesTile parses a names tile into NamesEntry values.
+func ParseNamesTile(data []byte) ([]NamesEntry, error) {
+	var entries []NamesEntry
+
+	for line := range bytes.Lines(data) {
+		if len(bytes.TrimSpace(line)) == 0 {
+			continue
+		}
+
+		var entry NamesEntry
+		if err := json.Unmarshal(line, &entry); err != nil {
+			return nil, fmt.Errorf("%w: %w", ErrInvalidNamesTile, err)
+		}
+
+		entries = append(entries, entry)
+	}
+
+	return entries, nil
+}

internal/certificatetransparency/ct-names-tile_test.go

@@ -0,0 +1,66 @@
+package certificatetransparency
+
+import (
+	"errors"
+	"slices"
+	"testing"
+)
+
+const namesTileJSONL = `{"Timestamp":1753375092043,"Subject":{"CommonName":"example.com"},"DNS":["example.com","www.example.com"]}
+{"Timestamp":1753375092044,"Subject":{"Organization":["Example Inc"]},"DNS":["mail.example.org"],"IP":["192.0.2.1"]}
+`
+
+func TestParseNamesTile(t *testing.T) {
+	entries, err := ParseNamesTile([]byte(namesTileJSONL))
+	if err != nil {
+		t.Fatalf("ParseNamesTile: %v", err)
+	}
+
+	if len(entries) != 2 {
+		t.Fatalf("got %d entries, want 2", len(entries))
+	}
+
+	if entries[0].Timestamp != 1753375092043 || entries[0].Subject.CommonName != "example.com" {
+		t.Errorf("unexpected first entry: %+v", entries[0])
+	}
+
+	if !slices.Equal(entries[0].DNS, []string{"example.com", "www.example.com"}) {
+		t.Errorf("DNS = %v", entries[0].DNS)
+	}
+
+	if !slices.Equal(entries[1].IP, []string{"192.0.2.1"}) {
+		t.Errorf("IP = %v", entries[1].IP)
+	}
+}
+
+func TestParseNamesTile_Invalid(t *testing.T) {
+	if _, err := ParseNamesTile([]byte("{not json}\n")); !errors.Is(err, ErrInvalidNamesTile) {
+		t.Errorf("want ErrInvalidNamesTile, got %v", err)
+	}
+}
+
+func TestParseNamesEntry(t *testing.T) {
+	var namesEntry NamesEntry
+	namesEntry.Timestamp = 1753375092043
+	namesEntry.Subject.CommonName = "example.com"
+	namesEntry.DNS = []string{"www.example.com", "example.com"}
+
+	entry := ParseNamesEntry(namesEntry, 42, "operator", "log", "https://ct.example.com/log/")
+
+	if !entry.DomainsOnly {
+		t.Error("entry should be marked as domains only")
+	}
+
+	if entry.Data.CertIndex != 42 {
+		t.Errorf("CertIndex = %d, want 42", entry.Data.CertIndex)
+	}
+
+	// The common name is already in the SANs and must not be added twice.
+	if !slices.Equal(entry.Data.LeafCert.AllDomains, []string{"www.example.com", "example.com"}) {
+		t.Errorf("AllDomains = %v", entry.Data.LeafCert.AllDomains)
+	}
+
+	if entry.Data.Source.Timestamp != 1753375092.043 {
+		t.Errorf("Source.Timestamp = %f", entry.Data.Source.Timestamp)
+	}
+}

internal/certificatetransparency/ct-parser.go

@@ -446,3 +446,33 @@ func ParseCertstreamEntry(rawEntry *ct.RawLogEntry, operatorName, logname, ctURL
 
 	return entry, nil
 }
+
+// ParseNamesEntry creates an Entry from a names tile entry. It only contains the domains, so
+// it is only useful for the domains-only stream. Names tiles don't say whether an entry is a
+// precertificate, so UpdateType is left empty.
+func ParseNamesEntry(namesEntry NamesEntry, index uint64, operatorName, logName, ctURL string) models.Entry {
+	domains := make([]string, 0, len(namesEntry.DNS)+1)
+	domains = append(domains, namesEntry.DNS...)
+
+	if cn := namesEntry.Subject.CommonName; cn != "" && !slices.Contains(domains, cn) {
+		domains = append(domains, cn)
+	}
+
+	return models.Entry{
+		Data: models.Data{
+			CertIndex: index,
+			Seen:      float64(time.Now().UnixMilli()) / 1_000,
+			Source: models.Source{
+				Name:          logName,
+				URL:           ctURL,
+				Timestamp:     float64(namesEntry.Timestamp) / 1_000,
+				Operator:      operatorName,
+				NormalizedURL: normalizeCtlogURL(ctURL),
+				Type:          models.SourceIsTiled,
+			},
+			LeafCert: models.LeafCert{AllDomains: domains},
+		},
+		MessageType: "certificate_update",
+		DomainsOnly: true,
+	}
+}

internal/certificatetransparency/errors.go

@@ -7,6 +7,7 @@ var (
 	ErrFetchingSTHFailed       = errors.New("failed to fetch STH")
 	ErrCheckpointInvalidFormat = errors.New("invalid checkpoint format: expected at least 3 lines")
 	ErrInvalidDataTile         = errors.New("invalid data tile precert_entry")
+	ErrInvalidNamesTile        = errors.New("invalid names tile")
 	ErrRequestFailed           = errors.New("request failed")
 	ErrUnknownEntryType        = errors.New("unknown entry type")
 	ErrInvalidFingerprint      = errors.New("invalid fingerprint")

internal/models/certstream.go

@@ -9,6 +9,7 @@ import (
 type Entry struct {
 	Data              Data   `json:"data"`
 	MessageType       string `json:"message_type"`
+	DomainsOnly       bool   `json:"-"` // built from a names tile, so there is no certificate
 	cachedJSON        []byte
 	cachedJSONLite    []byte
 	cachedJSONDomains []byte
@@ -19,6 +20,7 @@ func (e *Entry) Clone() Entry {
 	return Entry{
 		Data:              e.Data,
 		MessageType:       e.MessageType,
+		DomainsOnly:       e.DomainsOnly,
 		cachedJSON:        e.cachedJSON,
 		cachedJSONLite:    e.cachedJSONLite,
 		cachedJSONDomains: e.cachedJSONDomains,