diff --git a/internal/certificatetransparency/ct-names-tile.go b/internal/certificatetransparency/ct-names-tile.go new file mode 100644 index 0000000..1ed2200 --- /dev/null +++ b/internal/certificatetransparency/ct-names-tile.go @@ -0,0 +1,39 @@ +package certificatetransparency + +import ( + "bytes" + "encoding/json" + "fmt" +) + +// NamesEntry is a single entry of a names tile. +// See https://github.com/FiloSottile/sunlight/blob/main/names-tiles.md +type NamesEntry struct { + // Timestamp in milliseconds. + Timestamp int64 `json:"Timestamp"` + Subject struct { + CommonName string `json:"CommonName"` + } `json:"Subject"` + DNS []string `json:"DNS"` + IP []string `json:"IP"` +} + +// ParseNamesTile parses a names tile into NamesEntry values. +func ParseNamesTile(data []byte) ([]NamesEntry, error) { + var entries []NamesEntry + + for line := range bytes.Lines(data) { + if len(bytes.TrimSpace(line)) == 0 { + continue + } + + var entry NamesEntry + if err := json.Unmarshal(line, &entry); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidNamesTile, err) + } + + entries = append(entries, entry) + } + + return entries, nil +} diff --git a/internal/certificatetransparency/ct-names-tile_test.go b/internal/certificatetransparency/ct-names-tile_test.go new file mode 100644 index 0000000..dfc1338 --- /dev/null +++ b/internal/certificatetransparency/ct-names-tile_test.go @@ -0,0 +1,66 @@ +package certificatetransparency + +import ( + "errors" + "slices" + "testing" +) + +const namesTileJSONL = `{"Timestamp":1753375092043,"Subject":{"CommonName":"example.com"},"DNS":["example.com","www.example.com"]} +{"Timestamp":1753375092044,"Subject":{"Organization":["Example Inc"]},"DNS":["mail.example.org"],"IP":["192.0.2.1"]} +` + +func TestParseNamesTile(t *testing.T) { + entries, err := ParseNamesTile([]byte(namesTileJSONL)) + if err != nil { + t.Fatalf("ParseNamesTile: %v", err) + } + + if len(entries) != 2 { + t.Fatalf("got %d entries, want 2", len(entries)) + } + + if entries[0].Timestamp != 1753375092043 || entries[0].Subject.CommonName != "example.com" { + t.Errorf("unexpected first entry: %+v", entries[0]) + } + + if !slices.Equal(entries[0].DNS, []string{"example.com", "www.example.com"}) { + t.Errorf("DNS = %v", entries[0].DNS) + } + + if !slices.Equal(entries[1].IP, []string{"192.0.2.1"}) { + t.Errorf("IP = %v", entries[1].IP) + } +} + +func TestParseNamesTile_Invalid(t *testing.T) { + if _, err := ParseNamesTile([]byte("{not json}\n")); !errors.Is(err, ErrInvalidNamesTile) { + t.Errorf("want ErrInvalidNamesTile, got %v", err) + } +} + +func TestParseNamesEntry(t *testing.T) { + var namesEntry NamesEntry + namesEntry.Timestamp = 1753375092043 + namesEntry.Subject.CommonName = "example.com" + namesEntry.DNS = []string{"www.example.com", "example.com"} + + entry := ParseNamesEntry(namesEntry, 42, "operator", "log", "https://ct.example.com/log/") + + if !entry.DomainsOnly { + t.Error("entry should be marked as domains only") + } + + if entry.Data.CertIndex != 42 { + t.Errorf("CertIndex = %d, want 42", entry.Data.CertIndex) + } + + // The common name is already in the SANs and must not be added twice. + if !slices.Equal(entry.Data.LeafCert.AllDomains, []string{"www.example.com", "example.com"}) { + t.Errorf("AllDomains = %v", entry.Data.LeafCert.AllDomains) + } + + if entry.Data.Source.Timestamp != 1753375092.043 { + t.Errorf("Source.Timestamp = %f", entry.Data.Source.Timestamp) + } +} diff --git a/internal/certificatetransparency/ct-parser.go b/internal/certificatetransparency/ct-parser.go index 02bd580..e40c079 100644 --- a/internal/certificatetransparency/ct-parser.go +++ b/internal/certificatetransparency/ct-parser.go @@ -446,3 +446,33 @@ func ParseCertstreamEntry(rawEntry *ct.RawLogEntry, operatorName, logname, ctURL return entry, nil } + +// ParseNamesEntry creates an Entry from a names tile entry. It only contains the domains, so +// it is only useful for the domains-only stream. Names tiles don't say whether an entry is a +// precertificate, so UpdateType is left empty. +func ParseNamesEntry(namesEntry NamesEntry, index uint64, operatorName, logName, ctURL string) models.Entry { + domains := make([]string, 0, len(namesEntry.DNS)+1) + domains = append(domains, namesEntry.DNS...) + + if cn := namesEntry.Subject.CommonName; cn != "" && !slices.Contains(domains, cn) { + domains = append(domains, cn) + } + + return models.Entry{ + Data: models.Data{ + CertIndex: index, + Seen: float64(time.Now().UnixMilli()) / 1_000, + Source: models.Source{ + Name: logName, + URL: ctURL, + Timestamp: float64(namesEntry.Timestamp) / 1_000, + Operator: operatorName, + NormalizedURL: normalizeCtlogURL(ctURL), + Type: models.SourceIsTiled, + }, + LeafCert: models.LeafCert{AllDomains: domains}, + }, + MessageType: "certificate_update", + DomainsOnly: true, + } +} diff --git a/internal/certificatetransparency/errors.go b/internal/certificatetransparency/errors.go index 626027e..98ace70 100644 --- a/internal/certificatetransparency/errors.go +++ b/internal/certificatetransparency/errors.go @@ -7,6 +7,7 @@ var ( ErrFetchingSTHFailed = errors.New("failed to fetch STH") ErrCheckpointInvalidFormat = errors.New("invalid checkpoint format: expected at least 3 lines") ErrInvalidDataTile = errors.New("invalid data tile precert_entry") + ErrInvalidNamesTile = errors.New("invalid names tile") ErrRequestFailed = errors.New("request failed") ErrUnknownEntryType = errors.New("unknown entry type") ErrInvalidFingerprint = errors.New("invalid fingerprint") diff --git a/internal/models/certstream.go b/internal/models/certstream.go index 83f3312..64c50f8 100644 --- a/internal/models/certstream.go +++ b/internal/models/certstream.go @@ -9,6 +9,7 @@ import ( type Entry struct { Data Data `json:"data"` MessageType string `json:"message_type"` + DomainsOnly bool `json:"-"` // built from a names tile, so there is no certificate cachedJSON []byte cachedJSONLite []byte cachedJSONDomains []byte @@ -19,6 +20,7 @@ func (e *Entry) Clone() Entry { return Entry{ Data: e.Data, MessageType: e.MessageType, + DomainsOnly: e.DomainsOnly, cachedJSON: e.cachedJSON, cachedJSONLite: e.cachedJSONLite, cachedJSONDomains: e.cachedJSONDomains,