Add optional rate limiting of log queries

If a log operator publishes a simple rate limit for a log, we can use that
information to avoid sending requests to the log that we know will fail.
This will improve throughput as we won't be wasting time backing off from
failed requests.

go.mod

@@ -8,6 +8,9 @@ require (
 	golang.org/x/sync v0.15.0
 )
 
-require golang.org/x/text v0.26.0 // indirect
+require (
+	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/time v0.14.0 // indirect
+)
 
 retract v0.19.0 // Contains serious bugs.

go.sum

@@ -6,3 +6,5 @@ golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
 golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
 golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
+golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=

loglist/schema.go

@@ -44,8 +44,9 @@ type Log struct {
 	} `json:"temporal_interval,omitzero"`
 
 	// certspotter-specific extensions
-	CertspotterDownloadSize int `json:"certspotter_download_size,omitzero"`
+	CertspotterDownloadSize int     `json:"certspotter_download_size,omitzero"`
-	CertspotterDownloadJobs int `json:"certspotter_download_jobs,omitzero"`
+	CertspotterDownloadJobs int     `json:"certspotter_download_jobs,omitzero"`
+	CertspotterDownloadQPS  float64 `json:"certspotter_download_qps,omitzero"`
 
 	// TODO: add previous_operators
 }

monitor/monitor.go

@@ -14,6 +14,7 @@ import (
 	"errors"
 	"fmt"
 	"golang.org/x/sync/errgroup"
+	"golang.org/x/time/rate"
 	"log"
 	mathrand "math/rand/v2"
 	"net/url"
@@ -44,13 +45,24 @@ func downloadJobSize(ctlog *loglist.Log) uint64 {
 }
 
 func downloadWorkers(ctlog *loglist.Log) int {
-	if ctlog.CertspotterDownloadJobs != 0 {
+	if ctlog.CertspotterDownloadQPS != 0 {
+		// parallelism is effectively governed by the rate limit so for now just hard code a number here
+		return 10
+	} else if ctlog.CertspotterDownloadJobs != 0 {
 		return ctlog.CertspotterDownloadJobs
 	} else {
 		return 1
 	}
 }
 
+func downloadRateLimit(ctlog *loglist.Log) rate.Limit {
+	if ctlog.CertspotterDownloadQPS != 0 {
+		return rate.Limit(ctlog.CertspotterDownloadQPS)
+	} else {
+		return rate.Inf
+	}
+}
+
 type verifyEntriesError struct {
 	sth             *cttypes.SignedTreeHead
 	entriesRootHash merkletree.Hash
@@ -113,10 +125,14 @@ type logClient struct {
 	config *Config
 	log    *loglist.Log
 	client ctclient.Log
+	lim    *rate.Limiter
 }
 
 func (client *logClient) GetSTH(ctx context.Context) (sth *cttypes.SignedTreeHead, url string, err error) {
 	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		if err := client.lim.Wait(ctx); err != nil {
+			return err
+		}
 		sth, url, err = getAuthenticSTH(ctx, client.log, client.client)
 		return err
 	})
@@ -124,6 +140,9 @@ func (client *logClient) GetSTH(ctx context.Context) (sth *cttypes.SignedTreeHea
 }
 func (client *logClient) GetRoots(ctx context.Context) (roots [][]byte, err error) {
 	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		if err := client.lim.Wait(ctx); err != nil {
+			return err
+		}
 		roots, err = client.client.GetRoots(ctx)
 		return err
 	})
@@ -131,6 +150,9 @@ func (client *logClient) GetRoots(ctx context.Context) (roots [][]byte, err erro
 }
 func (client *logClient) GetEntries(ctx context.Context, startInclusive, endInclusive uint64) (entries []ctclient.Entry, err error) {
 	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		if err := client.lim.Wait(ctx); err != nil {
+			return err
+		}
 		entries, err = client.client.GetEntries(ctx, startInclusive, endInclusive)
 		return err
 	})
@@ -138,6 +160,9 @@ func (client *logClient) GetEntries(ctx context.Context, startInclusive, endIncl
 }
 func (client *logClient) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (tree *merkletree.CollapsedTree, err error) {
 	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		if err := client.lim.Wait(ctx); err != nil {
+			return err
+		}
 		tree, err = client.client.ReconstructTree(ctx, sth)
 		return err
 	})
@@ -184,6 +209,7 @@ func newLogClient(config *Config, ctlog *loglist.Log) (ctclient.Log, ctclient.Is
 			config: config,
 			log:    ctlog,
 			client: &ctclient.RFC6962Log{URL: logURL},
+			lim:    rate.NewLimiter(downloadRateLimit(ctlog), 1),
 		}, nil, nil
 	case ctlog.IsStaticCTAPI():
 		submissionURL, err := url.Parse(ctlog.SubmissionURL)
@@ -203,6 +229,7 @@ func newLogClient(config *Config, ctlog *loglist.Log) (ctclient.Log, ctclient.Is
 				config: config,
 				log:    ctlog,
 				client: client,
+				lim:    rate.NewLimiter(downloadRateLimit(ctlog), 1),
 			}, &issuerGetter{
 				config:    config,
 				log:       ctlog,