From 43d6c4de2ee9bd2b6fac32c6a36dcb137627f6bd Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Thu, 3 Jul 2025 13:33:06 -0400 Subject: [PATCH] Add package for parsing Mozilla's CT log list --- loglist/mozilla/ctlogs.go | 278 ++++++++++++++++++++++++++++++++ loglist/mozilla/ctlogs_test.go | 84 ++++++++++ loglist/mozilla/example_test.go | 55 +++++++ 3 files changed, 417 insertions(+) create mode 100644 loglist/mozilla/ctlogs.go create mode 100644 loglist/mozilla/ctlogs_test.go create mode 100644 loglist/mozilla/example_test.go diff --git a/loglist/mozilla/ctlogs.go b/loglist/mozilla/ctlogs.go new file mode 100644 index 0000000..46d502a --- /dev/null +++ b/loglist/mozilla/ctlogs.go @@ -0,0 +1,278 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +// Package mozilla contains a parser for Mozilla's CTKnownLogs.h file +package mozilla + +import ( + "bufio" + "errors" + "io" + "strconv" + "strings" + "time" +) + +// CTLogInfo describes a certificate transparency log from Mozilla's CTKnownLogs.h +// file. +type CTLogInfo struct { + Name string + State string // Admissible or Retired + Timestamp time.Time + OperatorIndex int + Key []byte +} + +// CTLogOperatorInfo describes a CT log operator from Mozilla's CTKnownLogs.h +// file. +type CTLogOperatorInfo struct { + Name string + ID int +} + +// Parse reads the CTKnownLogs.h content from r and returns the parsed logs and +// operators. Blocks enclosed by `#ifdef DEBUG` and `#endif` are ignored. +func Parse(r io.Reader) ([]CTLogInfo, []CTLogOperatorInfo, error) { + scanner := bufio.NewScanner(r) + skip := 0 + inLogs := false + inOps := false + + var logs []CTLogInfo + var ops []CTLogOperatorInfo + + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + + switch { + case strings.HasPrefix(trimmed, "#ifdef DEBUG"): + skip++ + continue + case strings.HasPrefix(trimmed, "#endif"): + if skip > 0 { + skip-- + continue + } + } + + if skip > 0 { + continue + } + + if strings.HasPrefix(trimmed, "const CTLogInfo kCTLogList[]") { + inLogs = true + continue + } + if strings.HasPrefix(trimmed, "const CTLogOperatorInfo kCTLogOperatorList[]") { + inOps = true + continue + } + + if inLogs { + if trimmed == "};" { + inLogs = false + continue + } + if strings.HasPrefix(trimmed, "{") { + log, err := readLogEntry(trimmed, scanner) + if err != nil { + return nil, nil, err + } + logs = append(logs, log) + } + continue + } + + if inOps { + if trimmed == "};" { + inOps = false + continue + } + if strings.HasPrefix(trimmed, "{") { + op, err := readOperatorEntry(trimmed) + if err != nil { + return nil, nil, err + } + ops = append(ops, op) + } + } + } + + if err := scanner.Err(); err != nil { + return nil, nil, err + } + + return logs, ops, nil +} + +func readLogEntry(firstLine string, s *bufio.Scanner) (CTLogInfo, error) { + var log CTLogInfo + + // Example first line: + // {"Name", CTLogState::Admissible, + firstLine = strings.TrimSpace(firstLine) + if !strings.HasPrefix(firstLine, "{") { + return log, errors.New("invalid log entry start") + } + firstLine = strings.TrimPrefix(firstLine, "{") + firstLine = strings.TrimSuffix(firstLine, ",") + + parts := splitCSV(firstLine) + var statePart string + switch len(parts) { + case 2: + log.Name = trimQuotes(parts[0]) + statePart = parts[1] + case 1: + log.Name = trimQuotes(parts[0]) + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + statePart = strings.TrimSpace(s.Text()) + default: + return log, errors.New("invalid log entry header") + } + statePart = strings.TrimSuffix(strings.TrimSpace(statePart), ",") + log.State = strings.TrimPrefix(statePart, "CTLogState::") + + // Next line: timestamp + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + tsLine := strings.TrimSpace(s.Text()) + tsValue := strings.Split(tsLine, ",")[0] + ts, err := strconv.ParseInt(strings.TrimSpace(tsValue), 10, 64) + if err != nil { + return log, err + } + log.Timestamp = time.Unix(0, ts*int64(time.Millisecond)) + + // Next line: operator index + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + opLine := strings.TrimSpace(s.Text()) + opValue := strings.Split(opLine, ",")[0] + opIndex, err := strconv.Atoi(strings.TrimSpace(opValue)) + if err != nil { + return log, err + } + log.OperatorIndex = opIndex + + // Key lines + var keyHex strings.Builder + for { + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + l := strings.TrimSpace(s.Text()) + if strings.HasPrefix(l, "\"") { + // remove trailing comma if any + trimmed := strings.TrimSuffix(l, ",") + keyHex.WriteString(trimQuotes(trimmed)) + if strings.HasSuffix(l, ",") { + // last key line + break + } + continue + } + return log, errors.New("unexpected line while reading key") + } + + key, err := decodeHexEscapes(keyHex.String()) + if err != nil { + return log, err + } + + // key length line + if !s.Scan() { + return log, io.ErrUnexpectedEOF + } + lenLine := strings.TrimSpace(s.Text()) + lenValue := strings.TrimSuffix(lenLine, "},") + keyLen, err := strconv.Atoi(strings.TrimSpace(lenValue)) + if err != nil { + return log, err + } + if len(key) != keyLen { + // ignore mismatch but continue + } + log.Key = key + return log, nil +} + +func readOperatorEntry(line string) (CTLogOperatorInfo, error) { + var op CTLogOperatorInfo + line = strings.TrimSuffix(strings.TrimSpace(line), ",") + if !strings.HasPrefix(line, "{") || !strings.HasSuffix(line, "}") { + return op, errors.New("invalid operator entry") + } + line = strings.TrimPrefix(line, "{") + line = strings.TrimSuffix(line, "}") + parts := splitCSV(line) + if len(parts) != 2 { + return op, errors.New("invalid operator fields") + } + op.Name = trimQuotes(parts[0]) + id, err := strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil { + return op, err + } + op.ID = id + return op, nil +} + +func splitCSV(s string) []string { + var parts []string + var cur strings.Builder + inQuote := false + for i := 0; i < len(s); i++ { + c := s[i] + if c == '"' { + inQuote = !inQuote + cur.WriteByte(c) + continue + } + if c == ',' && !inQuote { + parts = append(parts, strings.TrimSpace(cur.String())) + cur.Reset() + continue + } + cur.WriteByte(c) + } + if cur.Len() > 0 { + parts = append(parts, strings.TrimSpace(cur.String())) + } + return parts +} + +func trimQuotes(s string) string { + s = strings.TrimSpace(s) + if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' { + return s[1 : len(s)-1] + } + return s +} + +func decodeHexEscapes(s string) ([]byte, error) { + var out []byte + for i := 0; i < len(s); { + if i+3 >= len(s) || s[i] != '\\' || s[i+1] != 'x' { + return nil, errors.New("invalid escape") + } + b, err := strconv.ParseUint(s[i+2:i+4], 16, 8) + if err != nil { + return nil, err + } + out = append(out, byte(b)) + i += 4 + } + return out, nil +} diff --git a/loglist/mozilla/ctlogs_test.go b/loglist/mozilla/ctlogs_test.go new file mode 100644 index 0000000..8c62590 --- /dev/null +++ b/loglist/mozilla/ctlogs_test.go @@ -0,0 +1,84 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package mozilla + +import ( + "crypto/sha256" + "encoding/base64" + "errors" + "net/http" + "testing" +) + +// parseFromURL downloads the CTKnownLogs.h file from the given URL and parses it. +func parseFromURL(url string) ([]CTLogInfo, []CTLogOperatorInfo, error) { + resp, err := http.Get(url) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, nil, errors.New(resp.Status) + } + return Parse(resp.Body) +} + +func TestParseFromURL(t *testing.T) { + logs, ops, err := parseFromURL("https://hg-edge.mozilla.org/mozilla-central/raw-file/tip/security/ct/CTKnownLogs.h") + if err != nil { + t.Fatal(err) + } + if len(ops) == 0 { + t.Fatal("no operators parsed") + } + foundGoogle := false + foundSectigo := false + foundLets := false + for _, op := range ops { + if op.Name == "" { + t.Error("operator with empty name") + } + switch op.Name { + case "Google": + foundGoogle = true + case "Sectigo": + foundSectigo = true + case "Let's Encrypt": + foundLets = true + } + } + if !foundGoogle || !foundSectigo || !foundLets { + t.Errorf("missing expected operators: Google=%v Sectigo=%v Let's=%v", foundGoogle, foundSectigo, foundLets) + } + + if len(logs) == 0 { + t.Fatal("no logs parsed") + } + foundHash := false + targetHash := "1219ENGn9XfCx+lf1wC/+YLJM1pl4dCzAXMXwMjFaXc=" + for _, l := range logs { + if l.Name == "" { + t.Error("log with empty name") + } + if len(l.Key) == 0 { + t.Error("log with empty key") + } + if l.State != "Admissible" && l.State != "Retired" { + t.Errorf("unexpected state %q", l.State) + } + hash := sha256.Sum256(l.Key) + if base64.StdEncoding.EncodeToString(hash[:]) == targetHash { + foundHash = true + } + } + if !foundHash { + t.Errorf("log with key hash %s not found", targetHash) + } +} diff --git a/loglist/mozilla/example_test.go b/loglist/mozilla/example_test.go new file mode 100644 index 0000000..aa7c679 --- /dev/null +++ b/loglist/mozilla/example_test.go @@ -0,0 +1,55 @@ +// Copyright (C) 2025 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package mozilla_test + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" + "log" + "net/http" + "os" + "text/tabwriter" + "time" + + "software.sslmate.com/src/certspotter/loglist/mozilla" +) + +func ExampleParse() { + resp, err := http.Get("https://hg-edge.mozilla.org/mozilla-central/raw-file/tip/security/ct/CTKnownLogs.h") + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + log.Fatal(resp.Status) + } + logs, operators, err := mozilla.Parse(resp.Body) + if err != nil { + log.Fatal(err) + } + + tw := tabwriter.NewWriter(os.Stdout, 0, 8, 1, ' ', 0) + fmt.Fprintln(tw, "Operator\tName") + for _, o := range operators { + fmt.Fprintf(tw, "%d\t%s\n", o.ID, o.Name) + } + tw.Flush() + + tw = tabwriter.NewWriter(os.Stdout, 0, 8, 1, ' ', 0) + fmt.Fprintln(tw, "LogID\tState\tTimestamp\tOperator\tName") + for _, l := range logs { + hash := sha256.Sum256(l.Key) + fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%s\n", + base64.StdEncoding.EncodeToString(hash[:]), + l.State, l.Timestamp.UTC().Format(time.RFC3339), l.OperatorIndex, l.Name) + } + tw.Flush() +}