diff --git a/cmd/feedwitness/configs.go b/cmd/feedwitness/configs.go index 149bdf5..790346e 100644 --- a/cmd/feedwitness/configs.go +++ b/cmd/feedwitness/configs.go @@ -22,7 +22,7 @@ import ( "maps" logfmt "github.com/transparency-dev/formats/log" - "github.com/transparency-dev/witness/omniwitness" + "github.com/transparency-dev/witness/config" "github.com/transparency-dev/formats/note" "gopkg.in/yaml.v3" ) @@ -56,7 +56,7 @@ func newStaticFeederConfig(yamlCfg []byte) (*staticFeederConfig, error) { if err != nil { return nil, fmt.Errorf("failed to create signature verifier: %v", err) } - logCfg := omniwitness.Log{ + logCfg := config.Log{ VKey: log.PublicKey, Verifier: logV, Origin: log.Origin, diff --git a/cmd/feedwitness/run_feeders.go b/cmd/feedwitness/run_feeders.go index 71a6237..a3147e5 100644 --- a/cmd/feedwitness/run_feeders.go +++ b/cmd/feedwitness/run_feeders.go @@ -32,7 +32,7 @@ import ( "github.com/transparency-dev/witness/internal/feeder/sumdb" "github.com/transparency-dev/witness/internal/feeder/tiles" "github.com/transparency-dev/witness/witness" - "github.com/transparency-dev/witness/omniwitness" + "github.com/transparency-dev/witness/config" "go.opentelemetry.io/otel/metric" "golang.org/x/mod/sumdb/note" "golang.org/x/sync/errgroup" @@ -58,7 +58,7 @@ func init() { } type feederConfig struct { - Log omniwitness.Log + Log config.Log Feeder logFeeder } diff --git a/cmd/omniwitness/monolith.go b/cmd/omniwitness/monolith.go index 89e796d..81364ff 100644 --- a/cmd/omniwitness/monolith.go +++ b/cmd/omniwitness/monolith.go @@ -30,6 +30,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" f_note "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/witness/config" "github.com/transparency-dev/witness/omniwitness" "github.com/transparency-dev/witness/persistence/inmemory" psql "github.com/transparency-dev/witness/persistence/sqlite" @@ -160,7 +161,7 @@ func main() { } // Merge embedded configs into persisted configs { - lc := []omniwitness.Log{} + lc := []config.Log{} for c, err := range l.Logs(ctx) { if err != nil { klog.Exitf("Failed to read embedded log config: %v", err) diff --git a/cmd/omniwitness_gcp/main.go b/cmd/omniwitness_gcp/main.go index 14787f6..36bdacd 100644 --- a/cmd/omniwitness_gcp/main.go +++ b/cmd/omniwitness_gcp/main.go @@ -25,6 +25,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/transparency-dev/witness/config" "github.com/transparency-dev/witness/omniwitness" "github.com/transparency-dev/witness/persistence/spanner" "go.opentelemetry.io/otel" @@ -131,7 +132,7 @@ func mustUpdateLogs(ctx context.Context, y []byte, p *spanner.Persistence) { if err != nil { klog.Exitf("Failed to parse YAML logs config: %v", err) } - logs := []omniwitness.Log{} + logs := []config.Log{} for log, err := range l.Logs(ctx) { if err != nil { klog.Exitf("Error iterating over logs: %v", err) diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..369dff1 --- /dev/null +++ b/config/config.go @@ -0,0 +1,34 @@ +// Copyright 2026 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package config holds code related to configuring witnesses. +package config + +import "golang.org/x/mod/sumdb/note" + +// Log describes a verifiable log. +type Log struct { + // VKey is the serialised note-compliant vkey for the log. + VKey string + // Verifier is a signature verifier for log checkpoints. + Verifier note.Verifier + // Origin is the expected first line of checkpoints from the log. + Origin string + // QPD is the expected number of witness requests per day from the log. + QPD float64 + // Contact is an arbitrary string with contact information for the log operator. + Contact string + // URL is the URL of the root of the log. + URL string +} diff --git a/config/public_witness_network.go b/config/public_witness_network.go new file mode 100644 index 0000000..860db41 --- /dev/null +++ b/config/public_witness_network.go @@ -0,0 +1,172 @@ +// Copyright 2026 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package config holds code related to configuring witnesses. +package config + +import ( + "bufio" + "context" + "fmt" + "io" + "iter" + "net/http" + "strconv" + "strings" + + "github.com/transparency-dev/formats/note" +) + +// PublicFetchOpts holds options to be used when fetching the public witness network config. +type PublicFetchOpts struct { + // Client is the HTTP client to be used, if unset uses http.DefaultClient. + Client *http.Client + // URL is the URL of the config file. + URL string +} + +func FetchPublicConfig(ctx context.Context, opts PublicFetchOpts) ([]Log, error) { + if opts.Client == nil { + opts.Client = http.DefaultClient + } + + req, err := http.NewRequest(http.MethodGet, opts.URL, nil) + if err != nil { + return nil, fmt.Errorf("http.NewRequest: %w", err) + } + req = req.WithContext(ctx) + resp, err := opts.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("http.Do: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("got status %q", resp.Status) + } + defer func() { + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + }() + + return ParsePublicWitnessConfig(resp.Body) +} + +// ParsePublicWitnessConfig implements a parser for the public witness config format. +// +// The format is described here: https://github.com/transparency-dev/witness-network/blob/main/log-list-format.md +func ParsePublicWitnessConfig(r io.Reader) ([]Log, error) { + ret := []Log{} + foundHeader := false + var candidate *Log + scanner := bufio.NewScanner(r) + for l := range filteringScan(scanner) { + bits := strings.SplitN(l, " ", 2) + keyword := strings.ToLower(strings.TrimSpace(bits[0])) + if !foundHeader && keyword != "logs/v0" { + return nil, fmt.Errorf("invalid config, 'logs/v0' header should be the first non-comment line") + } + switch keyword { + case "logs/v0": + if foundHeader { + return nil, fmt.Errorf("invalid config, multiple 'logs/v0' headers found") + } + foundHeader = true + continue + case "vkey": + // vkey introduces a new log in the config file. + // The argument is a note-compliant vkey string. + // + // Since vkey is always the first keyword in a new log, we can use this as a trigger to "flush" the + // previous log config, if any, and start a new one. + if candidate != nil { + ret = append(ret, *candidate) + } + candidate = &Log{} + if len(bits) != 2 { + return nil, fmt.Errorf("invalid vkey line %q", l) + } + candidate.VKey = strings.TrimSpace(bits[1]) + v, err := note.NewVerifier(candidate.VKey) + if err != nil { + return nil, fmt.Errorf("note.NewVerifier: %v", err) + } + candidate.Verifier = v + candidate.Origin = v.Name() + case "origin": + // origin is an optional keyword which can be used to set a log origin which is different to the log's + // vkey name. + if candidate == nil { + return nil, fmt.Errorf("invalid config") + } + if len(bits) != 2 { + return nil, fmt.Errorf("invalid origin line %q", l) + } + candidate.Origin = strings.TrimSpace(bits[1]) + case "qpd": + // qpd is the number of queries the log is permitted to send to the witness per day. + // The value is interpreted as a float64. + if candidate == nil { + return nil, fmt.Errorf("invalid config") + } + if len(bits) != 2 { + return nil, fmt.Errorf("invalid qpd line %q", l) + } + v := strings.TrimSpace(bits[1]) + qpd, err := strconv.ParseFloat(v, 64) + if err != nil { + return nil, fmt.Errorf("invalid qpd value %q: %v", v, err) + } + candidate.QPD = qpd + case "contact": + // contact is a free-form text field which contains some sort of contact information for the log operator. + if candidate == nil { + return nil, fmt.Errorf("invalid config") + } + if len(bits) != 2 { + return nil, fmt.Errorf("invalid contact line %q", l) + } + candidate.Contact = strings.TrimSpace(bits[1]) + default: + return nil, fmt.Errorf("unexpected keyword %q", keyword) + } + } + // Include the final entry, if any. + if candidate != nil { + ret = append(ret, *candidate) + candidate = nil + } + if !foundHeader { + return nil, fmt.Errorf("invalid config, no 'logs/v0' header found") + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading config: %v", err) + } + return ret, nil +} + +func filteringScan(s *bufio.Scanner) iter.Seq[string] { + return func(yield func(string) bool) { + for s.Scan() { + l := strings.TrimSpace(s.Text()) + if l == "" { + continue + } + if l[0] == '#' { + continue + } + if !yield(l) { + return + } + } + } +} diff --git a/config/public_witness_network_test.go b/config/public_witness_network_test.go new file mode 100644 index 0000000..fced6c0 --- /dev/null +++ b/config/public_witness_network_test.go @@ -0,0 +1,136 @@ +// Copyright 2023 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config_test + +import ( + _ "embed" + "strings" + "testing" + + "github.com/transparency-dev/witness/config" + "github.com/google/go-cmp/cmp" +) + +func TestParsePublicWitnessConfig(t *testing.T) { + for _, test := range []struct { + name string + config string + want []config.Log + wantErr bool + }{ + { + name: "working example", + config: ` + # + # List: 10qps-100klogs + # Revision: 123 + # Generated: YYYY-MM-DD HH:MM:SS UTC + # Other undefined debug information. + # + logs/v0 + + # 1st list item -- foo's log + vkey sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 + qpd 86400 + contact https://tlog.foo.org/contact + + # 2nd list item -- log with custom origin + vkey sigsum.org/v1/tree/44ad38f8226ff9bd27629a41e55df727308d0a1cd8a2c31d3170048ac1dd22a1+682b49db+AQ7H4WhDEZsSA3enOROsasvC0D2CQy4sNrhBsJqVhB8l + origin something-not-equal-to-vkey-keyname + qpd 24 + contact sysadmin (at) bar.org + + # 3rd list item - minimal log config + vkey tlog.andxor.it+d5e6b3d0+AU6uJ3h8tb+RRMdGjHV4KCrrHoKfIYGbhL2A46thEhKQ + qpd 1 + + # Some trailing comments + `, + want: []config.Log{ + { + VKey: "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8", + Origin: "sum.golang.org", + QPD: 86400, + Contact: "https://tlog.foo.org/contact", + }, { + VKey: "sigsum.org/v1/tree/44ad38f8226ff9bd27629a41e55df727308d0a1cd8a2c31d3170048ac1dd22a1+682b49db+AQ7H4WhDEZsSA3enOROsasvC0D2CQy4sNrhBsJqVhB8l", + Origin: "something-not-equal-to-vkey-keyname", + QPD: 24, + Contact: "sysadmin (at) bar.org", + }, { + VKey: "tlog.andxor.it+d5e6b3d0+AU6uJ3h8tb+RRMdGjHV4KCrrHoKfIYGbhL2A46thEhKQ", + Origin: "tlog.andxor.it", + QPD: 1, + }, + }, + }, { + name: "empty config", + config: "logs/v0", + want: []config.Log{}, + }, { + name: "broken: no header", + config: ` + # 1st list item -- foo's log + vkey sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 + qpd 86400 + contact https://tlog.foo.org/contact + `, + wantErr: true, + }, { + name: "broken: bad ordering, vkey not first", + config: ` + logs/v0 + # 1st list item -- foo's log + qpd 86400 + vkey sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 + contact https://tlog.foo.org/contact + `, + wantErr: true, + }, { + name: "broken: not a vkey", + config: ` + logs/v0 + vkey BANANAS + # 1st list item -- foo's log + qpd 86400 + contact https://tlog.foo.org/contact + `, + wantErr: true, + }, { + name: "broken: qpd not numeric", + config: ` + logs/v0 + vkey sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 + # 1st list item -- foo's log + qpd toast + contact https://tlog.foo.org/contact + `, + wantErr: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + got, err := config.ParsePublicWitnessConfig(strings.NewReader(test.config)) + if gotErr := err != nil; gotErr != test.wantErr { + t.Fatalf("Got %v, want error %t", err, test.wantErr) + } + for i := range got { + got[i].Verifier = nil + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("Got unexpected difference: %v", diff) + } + }) + } +} diff --git a/omniwitness/configs.go b/omniwitness/configs.go index 5cd50ec..695b853 100644 --- a/omniwitness/configs.go +++ b/omniwitness/configs.go @@ -15,18 +15,15 @@ package omniwitness import ( - "bufio" "context" _ "embed" // embed is needed to embed files as constants "errors" "fmt" - "io" "iter" "maps" "net/http" - "strconv" - "strings" + "github.com/transparency-dev/witness/config" logfmt "github.com/transparency-dev/formats/log" "github.com/transparency-dev/formats/note" "gopkg.in/yaml.v3" @@ -58,14 +55,14 @@ func NewStaticLogConfig(yamlCfg []byte) (*staticLogConfig, error) { return nil, fmt.Errorf("failed to unmarshal witness config: %v", err) } r := &staticLogConfig{ - logs: make(map[string]Log), + logs: make(map[string]config.Log), } for _, log := range cfg.Logs { logV, err := note.NewVerifier(log.PublicKey) if err != nil { return nil, fmt.Errorf("failed to create signature verifier: %v", err) } - logCfg := Log{ + logCfg := config.Log{ VKey: log.PublicKey, Verifier: logV, Origin: log.Origin, @@ -84,11 +81,11 @@ func NewStaticLogConfig(yamlCfg []byte) (*staticLogConfig, error) { } type staticLogConfig struct { - logs map[string]Log + logs map[string]config.Log } -func (s *staticLogConfig) Logs(_ context.Context) iter.Seq2[Log, error] { - return func(yield func(Log, error) bool) { +func (s *staticLogConfig) Logs(_ context.Context) iter.Seq2[config.Log, error] { + return func(yield func(config.Log, error) bool) { for _, v := range s.logs { if !yield(v, nil) { return @@ -97,7 +94,7 @@ func (s *staticLogConfig) Logs(_ context.Context) iter.Seq2[Log, error] { } } -func (s *staticLogConfig) Log(_ context.Context, origin string) (Log, bool, error) { +func (s *staticLogConfig) Log(_ context.Context, origin string) (config.Log, bool, error) { logID := logfmt.ID(origin) l, ok := s.logs[logID] return l, ok, nil @@ -110,7 +107,7 @@ func (s *staticLogConfig) Merge(other *staticLogConfig) { maps.Copy(s.logs, other.logs) } -func (s *staticLogConfig) AddLogs(_ context.Context, _ []Log) error { +func (s *staticLogConfig) AddLogs(_ context.Context, _ []config.Log) error { return errors.New("staticLogConfig doesn't support adding logs") } @@ -121,131 +118,3 @@ type PublicFetchOpts struct { // URL is the URL of the config file. URL string } - -func FetchPublicConfig(ctx context.Context, opts PublicFetchOpts) ([]Log, error) { - if opts.Client == nil { - opts.Client = http.DefaultClient - } - - req, err := http.NewRequest(http.MethodGet, opts.URL, nil) - if err != nil { - return nil, fmt.Errorf("http.NewRequest: %w", err) - } - req = req.WithContext(ctx) - resp, err := opts.Client.Do(req) - if err != nil { - return nil, fmt.Errorf("http.Do: %w", err) - } - defer func() { - _, _ = io.ReadAll(resp.Body) - _ = resp.Body.Close() - }() - - return ParsePublicWitnessConfig(resp.Body) -} - -// ParsePublicWitnessConfig implements a parser for the public witness config format. -// -// The format is described here: https://github.com/transparency-dev/witness-network/blob/main/log-list-format.md -func ParsePublicWitnessConfig(r io.Reader) ([]Log, error) { - ret := []Log{} - foundHeader := false - var candidate *Log - scanner := bufio.NewScanner(r) - for l := range filteringScan(scanner) { - bits := strings.SplitN(l, " ", 2) - switch keyword := strings.ToLower(strings.TrimSpace(bits[0])); keyword { - case "logs/v0": - if foundHeader { - return nil, fmt.Errorf("invalid config, multiple 'logs/v0' headers found") - } - foundHeader = true - continue - case "vkey": - // vkey introduces a new log in the config file. - // The argument is a note-compliant vkey string. - // - // Since vkey is always the first keyword in a new log, we can use this as a trigger to "flush" the - // previous log config, if any, and start a new one. - if candidate != nil { - ret = append(ret, *candidate) - } - candidate = &Log{} - if len(bits) != 2 { - return nil, fmt.Errorf("invalid vkey line %q", l) - } - candidate.VKey = strings.TrimSpace(bits[1]) - v, err := note.NewVerifier(candidate.VKey) - if err != nil { - return nil, fmt.Errorf("note.NewVerifier: %v", err) - } - candidate.Verifier = v - candidate.Origin = v.Name() - case "origin": - // origin is an optional keyword which can be used to set a log origin which is different to the log's - // vkey name. - if candidate == nil { - return nil, fmt.Errorf("invalid config") - } - if len(bits) != 2 { - return nil, fmt.Errorf("invalid origin line %q", l) - } - candidate.Origin = strings.TrimSpace(bits[1]) - case "qpd": - // qpd is the number of queries the log is permitted to send to the witness per dat. - // The value is interpreted as a float64. - if candidate == nil { - return nil, fmt.Errorf("invalid config") - } - if len(bits) != 2 { - return nil, fmt.Errorf("invalid qps line %q", l) - } - v := strings.TrimSpace(bits[1]) - qpd, err := strconv.ParseFloat(v, 64) - if err != nil { - return nil, fmt.Errorf("invalid qps value %q: %v", v, err) - } - candidate.QPD = qpd - case "contact": - // contact is a free-form text field which contains some sort of contact information for the log operator. - if candidate == nil { - return nil, fmt.Errorf("invalid config") - } - if len(bits) != 2 { - return nil, fmt.Errorf("invalid contact line %q", l) - } - candidate.Contact = strings.TrimSpace(bits[1]) - default: - return nil, fmt.Errorf("unexpected keyword %q", keyword) - } - } - // Include the final entry, if any. - if candidate != nil { - ret = append(ret, *candidate) - candidate = nil - } - if !foundHeader { - return nil, fmt.Errorf("invalid config, no 'logs/v0' header found") - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading config: %v", err) - } - return ret, nil -} - -func filteringScan(s *bufio.Scanner) iter.Seq[string] { - return func(yield func(string) bool) { - for s.Scan() { - l := strings.TrimSpace(s.Text()) - if l == "" { - continue - } - if l[0] == '#' { - continue - } - if !yield(l) { - return - } - } - } -} diff --git a/omniwitness/configs_test.go b/omniwitness/configs_test.go index 9363ed2..1e7f0fa 100644 --- a/omniwitness/configs_test.go +++ b/omniwitness/configs_test.go @@ -16,10 +16,8 @@ package omniwitness_test import ( _ "embed" - "strings" "testing" - "github.com/google/go-cmp/cmp" "github.com/transparency-dev/witness/omniwitness" ) @@ -103,114 +101,3 @@ Logs: } } -func TestParsePublicWitnessConfig(t *testing.T) { - for _, test := range []struct { - name string - config string - want []omniwitness.Log - wantErr bool - }{ - { - name: "working example", - config: ` - # - # List: 10qps-100klogs - # Revision: 123 - # Generated: YYYY-MM-DD HH:MM:SS UTC - # Other undefined debug information. - # - logs/v0 - - # 1st list item -- foo's log - vkey sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 - qpd 86400 - contact https://tlog.foo.org/contact - - # 2nd list item -- log with custom origin - vkey sigsum.org/v1/tree/44ad38f8226ff9bd27629a41e55df727308d0a1cd8a2c31d3170048ac1dd22a1+682b49db+AQ7H4WhDEZsSA3enOROsasvC0D2CQy4sNrhBsJqVhB8l - origin something-not-equal-to-vkey-keyname - qpd 24 - contact sysadmin (at) bar.org - - # 3rd list item - minimal log config - vkey tlog.andxor.it+d5e6b3d0+AU6uJ3h8tb+RRMdGjHV4KCrrHoKfIYGbhL2A46thEhKQ - qpd 1 - - # Some trailing comments - `, - want: []omniwitness.Log{ - { - VKey: "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8", - Origin: "sum.golang.org", - QPD: 86400, - Contact: "https://tlog.foo.org/contact", - }, { - VKey: "sigsum.org/v1/tree/44ad38f8226ff9bd27629a41e55df727308d0a1cd8a2c31d3170048ac1dd22a1+682b49db+AQ7H4WhDEZsSA3enOROsasvC0D2CQy4sNrhBsJqVhB8l", - Origin: "something-not-equal-to-vkey-keyname", - QPD: 24, - Contact: "sysadmin (at) bar.org", - }, { - VKey: "tlog.andxor.it+d5e6b3d0+AU6uJ3h8tb+RRMdGjHV4KCrrHoKfIYGbhL2A46thEhKQ", - Origin: "tlog.andxor.it", - QPD: 1, - }, - }, - }, { - name: "empty config", - config: "logs/v0", - want: []omniwitness.Log{}, - }, { - name: "broken: no header", - config: ` - # 1st list item -- foo's log - vkey sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 - qpd 86400 - contact https://tlog.foo.org/contact - `, - wantErr: true, - }, { - name: "broken: bad ordering, vkey not first", - config: ` - logs/v0 - # 1st list item -- foo's log - qpd 86400 - vkey sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 - contact https://tlog.foo.org/contact - `, - wantErr: true, - }, { - name: "broken: not a vkey", - config: ` - logs/v0 - vkey BANANAS - # 1st list item -- foo's log - qpd 86400 - contact https://tlog.foo.org/contact - `, - wantErr: true, - }, { - name: "broken: qpd not numeric", - config: ` - logs/v0 - vkey sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 - # 1st list item -- foo's log - qpd toast - contact https://tlog.foo.org/contact - `, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := omniwitness.ParsePublicWitnessConfig(strings.NewReader(test.config)) - if gotErr := err != nil; gotErr != test.wantErr { - t.Fatalf("Got %v, want error %t", err, test.wantErr) - } - for i := range got { - got[i].Verifier = nil - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("Got unexpected difference: %v", diff) - } - }) - } -} diff --git a/omniwitness/distribute.go b/omniwitness/distribute.go index c7bb085..94ff513 100644 --- a/omniwitness/distribute.go +++ b/omniwitness/distribute.go @@ -24,6 +24,7 @@ import ( "net/url" f_log "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/witness/config" "go.opentelemetry.io/otel/metric" "golang.org/x/mod/sumdb/note" "golang.org/x/time/rate" @@ -62,7 +63,7 @@ func init() { // logsFn should return the _current_ set of logs whose checkpoints should be distributed. // This may be called repeatedly by the implementation in order to ensure that changes to the underlying config are reflected in the distribution operation. -type logsFn func(context.Context) iter.Seq2[Log, error] +type logsFn func(context.Context) iter.Seq2[config.Log, error] // newDistributor creates a new Distributor from the given configuration. func newDistributor(baseURL string, client *http.Client, logs logsFn, witSigV note.Verifier, getLatest getLatestCheckpointFn, rateLimit float64) (*distributor, error) { @@ -110,7 +111,7 @@ func (d *distributor) DistributeOnce(ctx context.Context) error { return nil } -func (d *distributor) distributeForLog(ctx context.Context, l Log) error { +func (d *distributor) distributeForLog(ctx context.Context, l config.Log) error { logID := f_log.ID(l.Origin) counterDistRestAttempt.Add(ctx, 1, metric.WithAttributes(logKey.String(l.Origin))) diff --git a/omniwitness/distribute_test.go b/omniwitness/distribute_test.go index 719aa0f..dd088ff 100644 --- a/omniwitness/distribute_test.go +++ b/omniwitness/distribute_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/gorilla/mux" + "github.com/transparency-dev/witness/config" f_log "github.com/transparency-dev/formats/log" f_note "github.com/transparency-dev/formats/note" "golang.org/x/mod/sumdb/note" @@ -45,7 +46,7 @@ func TestDistributeOnce(t *testing.T) { defer ts.Close() lc := &singleLogConfig{ - lc: Log{ + lc: config.Log{ Origin: "Log Checkpoint v0", Verifier: mustVerifier(t, lPK), URL: "http://example.com/log62", @@ -128,11 +129,11 @@ func mustVerifier(t *testing.T, v string) note.Verifier { } type singleLogConfig struct { - lc Log + lc config.Log } -func (l *singleLogConfig) Logs(_ context.Context) iter.Seq2[Log, error] { - return func(yield func(Log, error) bool) { +func (l *singleLogConfig) Logs(_ context.Context) iter.Seq2[config.Log, error] { + return func(yield func(config.Log, error) bool) { yield(l.lc, nil) } } diff --git a/omniwitness/omniwitness.go b/omniwitness/omniwitness.go index d96dc92..d242c6b 100644 --- a/omniwitness/omniwitness.go +++ b/omniwitness/omniwitness.go @@ -27,6 +27,7 @@ import ( "net/http" "time" + "github.com/transparency-dev/witness/config" "github.com/transparency-dev/witness/witness" "golang.org/x/mod/sumdb/note" "golang.org/x/sync/errgroup" @@ -40,27 +41,14 @@ import ( // in order to persist its view of log state type Persistence = witness.LogStatePersistence +//go:fix inline +type Log = config.Log + const ( defaultDistributeInterval = 1 * time.Minute defaultProvisionInterval = 10 * time.Minute ) -// Log describes a verifiable log. -type Log struct { - // VKey is the serialised note-compliant vkey for the log. - VKey string - // Verifier is a signature verifier for log checkpoints. - Verifier note.Verifier - // Origin is the expected first line of checkpoints from the log. - Origin string - // QPD is the expected number of witness requests per day from the log. - QPD float64 - // Contact is an arbitrary string with contact information for the log operator. - Contact string - // URL is the URL of the root of the log. - URL string -} - // OperatorConfig allows the bare minimum operator-specific configuration. // This should only contain configuration details that are custom per-operator. type OperatorConfig struct { @@ -107,12 +95,12 @@ type OperatorConfig struct { // LogConfig is the contract of something which knows how to provide log configuration info for the witness. type LogConfig interface { // Logs returns an iterator of all known logs. - Logs(ctx context.Context) iter.Seq2[Log, error] + Logs(ctx context.Context) iter.Seq2[config.Log, error] // Log returns the configuration info of the log with the specified log ID, if it exists. - Log(ctx context.Context, id string) (Log, bool, error) + Log(ctx context.Context, id string) (config.Log, bool, error) // AddLogs should attempt to merge the provided logs into the current config. // The merge must be additive only with respect to the logs. - AddLogs(ctx context.Context, cfg []Log) error + AddLogs(ctx context.Context, cfg []config.Log) error } // Main runs the omniwitness, with the witness listening using the listener, and all @@ -235,10 +223,9 @@ func runRestDistributors(ctx context.Context, g *errgroup.Group, httpClient *htt func rateLimit(limiter *rate.Limiter, delegate func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { if limiter != nil && !limiter.Allow() { - http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) - return - } + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } delegate(w, r) } } - diff --git a/omniwitness/omniwitness_test.go b/omniwitness/omniwitness_test.go index 9a58e92..e8c7ebd 100644 --- a/omniwitness/omniwitness_test.go +++ b/omniwitness/omniwitness_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/transparency-dev/witness/persistence/inmemory" "github.com/transparency-dev/witness/omniwitness" + "github.com/transparency-dev/witness/persistence/inmemory" "golang.org/x/sync/errgroup" ) diff --git a/omniwitness/otel.go b/omniwitness/otel.go index 4597091..a6bfbf1 100644 --- a/omniwitness/otel.go +++ b/omniwitness/otel.go @@ -22,9 +22,9 @@ import ( const name = "github.com/transparency-dev/witness/omniwitness" var ( - meter = otel.Meter(name) + meter = otel.Meter(name) ) var ( - logKey = attribute.Key("witness.log_origin") + logKey = attribute.Key("witness.log_origin") ) diff --git a/omniwitness/public_witness_network.go b/omniwitness/public_witness_network.go index 192a761..48f78c5 100644 --- a/omniwitness/public_witness_network.go +++ b/omniwitness/public_witness_network.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "github.com/transparency-dev/witness/config" "k8s.io/klog/v2" ) @@ -43,12 +44,12 @@ func provisionFromPublicConfig(ctx context.Context, httpClient *http.Client, url } klog.V(1).Infof("Provisioning from public witness network...") for _, u := range urls { - opts := PublicFetchOpts{ + opts := config.PublicFetchOpts{ Client: httpClient, URL: u, } klog.V(2).Infof("Provisioning from %q...", u) - logs, err := FetchPublicConfig(ctx, opts) + logs, err := config.FetchPublicConfig(ctx, opts) if err != nil { klog.Warningf("Failed to fetch public witness network config from %q: %v", u, err) continue diff --git a/persistence/inmemory/inmemory.go b/persistence/inmemory/inmemory.go index 45e752a..774c1ad 100644 --- a/persistence/inmemory/inmemory.go +++ b/persistence/inmemory/inmemory.go @@ -24,7 +24,7 @@ import ( "sync" "github.com/transparency-dev/formats/log" - "github.com/transparency-dev/witness/omniwitness" + "github.com/transparency-dev/witness/config" ) // New returns a persistence object that lives only in memory. @@ -45,7 +45,7 @@ func (p *Persistence) Init(_ context.Context) error { return nil } -func (p *Persistence) AddLogs(ctx context.Context, lc []omniwitness.Log) error { +func (p *Persistence) AddLogs(ctx context.Context, lc []config.Log) error { for _, l := range lc { logID := log.ID(l.Origin) p.logs.Store(logID, l) @@ -53,21 +53,21 @@ func (p *Persistence) AddLogs(ctx context.Context, lc []omniwitness.Log) error { return nil } -func (p *Persistence) Logs(ctx context.Context) iter.Seq2[omniwitness.Log, error] { - return func(yield func(omniwitness.Log, error) bool) { +func (p *Persistence) Logs(ctx context.Context) iter.Seq2[config.Log, error] { + return func(yield func(config.Log, error) bool) { p.logs.Range(func(key, value any) bool { - return yield(value.(omniwitness.Log), nil) + return yield(value.(config.Log), nil) }) } } -func (p *Persistence) Log(ctx context.Context, origin string) (omniwitness.Log, bool, error) { +func (p *Persistence) Log(ctx context.Context, origin string) (config.Log, bool, error) { logID := log.ID(origin) val, ok := p.logs.Load(logID) if !ok { - return omniwitness.Log{}, false, nil + return config.Log{}, false, nil } - return val.(omniwitness.Log), true, nil + return val.(config.Log), true, nil } func (p *Persistence) Latest(_ context.Context, origin string) ([]byte, error) { diff --git a/persistence/spanner/persistence.go b/persistence/spanner/persistence.go index 7265711..06caa73 100644 --- a/persistence/spanner/persistence.go +++ b/persistence/spanner/persistence.go @@ -28,7 +28,7 @@ import ( "cloud.google.com/go/spanner/apiv1/spannerpb" logfmt "github.com/transparency-dev/formats/log" "github.com/transparency-dev/formats/note" - "github.com/transparency-dev/witness/omniwitness" + "github.com/transparency-dev/witness/config" "google.golang.org/api/iterator" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -102,7 +102,7 @@ func (p *Persistence) createTablesIfNotExist(ctx context.Context) error { return nil } -func (p *Persistence) AddLogs(ctx context.Context, lc []omniwitness.Log) error { +func (p *Persistence) AddLogs(ctx context.Context, lc []config.Log) error { m := []*spanner.MutationGroup{} for _, l := range lc { // Note that it's a deliberate choice here to use Insert so as to guarantee that we will not @@ -143,8 +143,8 @@ func batchWrite(ctx context.Context, s *spanner.Client, m []*spanner.MutationGro return errors.Join(errs...) } -func (p *Persistence) Logs(ctx context.Context) iter.Seq2[omniwitness.Log, error] { - return func(yield func(omniwitness.Log, error) bool) { +func (p *Persistence) Logs(ctx context.Context) iter.Seq2[config.Log, error] { + return func(yield func(config.Log, error) bool) { r := p.spanner.Single().Read(ctx, "logs", spanner.AllKeys(), []string{"origin", "vkey", "contact", "disabled"}) for { row, err := r.Next() @@ -152,15 +152,15 @@ func (p *Persistence) Logs(ctx context.Context) iter.Seq2[omniwitness.Log, error if err == iterator.Done { return } - if !yield(omniwitness.Log{}, fmt.Errorf("failed to read row: %v", err)) { + if !yield(config.Log{}, fmt.Errorf("failed to read row: %v", err)) { return } } - c := omniwitness.Log{} + c := config.Log{} var disabled spanner.NullBool var contact spanner.NullString if err := row.Columns(&c.Origin, &c.VKey, &contact, &disabled); err != nil { - if !yield(omniwitness.Log{}, fmt.Errorf("failed to read columns: %v", err)) { + if !yield(config.Log{}, fmt.Errorf("failed to read columns: %v", err)) { return } } @@ -171,7 +171,7 @@ func (p *Persistence) Logs(ctx context.Context) iter.Seq2[omniwitness.Log, error c.Contact = contact.StringVal c.Verifier, err = note.NewVerifier(c.VKey) if err != nil { - if !yield(omniwitness.Log{}, fmt.Errorf("failed to create verifier: %v", err)) { + if !yield(config.Log{}, fmt.Errorf("failed to create verifier: %v", err)) { return } } @@ -182,20 +182,20 @@ func (p *Persistence) Logs(ctx context.Context) iter.Seq2[omniwitness.Log, error } } -func (p *Persistence) Log(ctx context.Context, origin string) (omniwitness.Log, bool, error) { +func (p *Persistence) Log(ctx context.Context, origin string) (config.Log, bool, error) { logID := logfmt.ID(origin) row, err := p.spanner.Single().ReadRow(ctx, "logs", spanner.Key{logID}, []string{"origin", "vkey", "contact", "disabled"}) if err != nil { if errors.Is(err, spanner.ErrRowNotFound) { - return omniwitness.Log{}, false, nil + return config.Log{}, false, nil } - return omniwitness.Log{}, false, fmt.Errorf("failed to read row: %v", err) + return config.Log{}, false, fmt.Errorf("failed to read row: %v", err) } - c := omniwitness.Log{} + c := config.Log{} var disabled spanner.NullBool var contact spanner.NullString if err := row.Columns(&c.Origin, &c.VKey, &contact, &disabled); err != nil { - return omniwitness.Log{}, false, fmt.Errorf("failed to read columns: %v", err) + return config.Log{}, false, fmt.Errorf("failed to read columns: %v", err) } if disabled.Bool { klog.V(1).Infof("Ignoring disabled log %q", c.Origin) @@ -204,7 +204,7 @@ func (p *Persistence) Log(ctx context.Context, origin string) (omniwitness.Log, c.Contact = contact.StringVal c.Verifier, err = note.NewVerifier(c.VKey) if err != nil { - return omniwitness.Log{}, false, fmt.Errorf("failed to create verifier: %v", err) + return config.Log{}, false, fmt.Errorf("failed to create verifier: %v", err) } return c, true, nil } diff --git a/persistence/spanner/persistence_test.go b/persistence/spanner/persistence_test.go index dc98b51..29d01ff 100644 --- a/persistence/spanner/persistence_test.go +++ b/persistence/spanner/persistence_test.go @@ -27,7 +27,7 @@ import ( "cloud.google.com/go/spanner/spannertest" "github.com/transparency-dev/formats/log" ptest "github.com/transparency-dev/witness/persistence/testonly" - "github.com/transparency-dev/witness/omniwitness" + "github.com/transparency-dev/witness/config" "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -122,7 +122,7 @@ func TestDisableLog(t *testing.T) { sp.batchWrite = testBatchWrite if err := sp.AddLogs(t.Context(), - []omniwitness.Log{ + []config.Log{ { Origin: "log1", VKey: "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8", @@ -169,7 +169,7 @@ func TestDisabledLogStaysDisabled(t *testing.T) { t.Fatalf("Init(): %v", err) } - logs := []omniwitness.Log{ + logs := []config.Log{ { Origin: "log1", VKey: "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8", @@ -213,9 +213,9 @@ func TestDisabledLogStaysDisabled(t *testing.T) { } } -func toSlice(t *testing.T, i iter.Seq2[omniwitness.Log, error]) []omniwitness.Log { +func toSlice(t *testing.T, i iter.Seq2[config.Log, error]) []config.Log { t.Helper() - logs := []omniwitness.Log{} + logs := []config.Log{} for l, err := range i { if err != nil { t.Fatalf("Logs(): %v", err) diff --git a/persistence/sqlite/sql.go b/persistence/sqlite/sql.go index 54c73ae..03694d8 100644 --- a/persistence/sqlite/sql.go +++ b/persistence/sqlite/sql.go @@ -24,7 +24,7 @@ import ( "github.com/transparency-dev/formats/log" "github.com/transparency-dev/formats/note" "github.com/transparency-dev/witness/witness" - "github.com/transparency-dev/witness/omniwitness" + "github.com/transparency-dev/witness/config" "k8s.io/klog/v2" "modernc.org/sqlite" @@ -85,7 +85,7 @@ func (p *Persistence) createTablesIfNotExist(ctx context.Context) error { return nil } -func (p *Persistence) AddLogs(ctx context.Context, lc []omniwitness.Log) error { +func (p *Persistence) AddLogs(ctx context.Context, lc []config.Log) error { for _, l := range lc { // Note that it's a deliberate choice here to use Insert so as to guarantee that we will not // update the stored config for a given log. There may be reasons to revisit this in the future, @@ -100,11 +100,11 @@ func (p *Persistence) AddLogs(ctx context.Context, lc []omniwitness.Log) error { return nil } -func (p *Persistence) Logs(ctx context.Context) iter.Seq2[omniwitness.Log, error] { - return func(yield func(omniwitness.Log, error) bool) { +func (p *Persistence) Logs(ctx context.Context) iter.Seq2[config.Log, error] { + return func(yield func(config.Log, error) bool) { rows, err := p.db.QueryContext(ctx, "SELECT origin, vkey, contact, disabled FROM logs") if err != nil { - if !yield(omniwitness.Log{}, fmt.Errorf("failed to select from logs: %v", err)) { + if !yield(config.Log{}, fmt.Errorf("failed to select from logs: %v", err)) { return } } @@ -112,10 +112,10 @@ func (p *Persistence) Logs(ctx context.Context) iter.Seq2[omniwitness.Log, error _ = rows.Close() }() for rows.Next() { - c := omniwitness.Log{} + c := config.Log{} disabled := false if err := rows.Scan(&c.Origin, &c.VKey, &c.Contact, &disabled); err != nil { - if !yield(omniwitness.Log{}, fmt.Errorf("failed to scan columns: %v", err)) { + if !yield(config.Log{}, fmt.Errorf("failed to scan columns: %v", err)) { return } } @@ -125,7 +125,7 @@ func (p *Persistence) Logs(ctx context.Context) iter.Seq2[omniwitness.Log, error } c.Verifier, err = note.NewVerifier(c.VKey) if err != nil { - if !yield(omniwitness.Log{}, fmt.Errorf("failed to create verifier: %v", err)) { + if !yield(config.Log{}, fmt.Errorf("failed to create verifier: %v", err)) { return } } @@ -136,16 +136,16 @@ func (p *Persistence) Logs(ctx context.Context) iter.Seq2[omniwitness.Log, error } } -func (p *Persistence) Log(ctx context.Context, origin string) (omniwitness.Log, bool, error) { +func (p *Persistence) Log(ctx context.Context, origin string) (config.Log, bool, error) { logID := log.ID(origin) row := p.db.QueryRowContext(ctx, "SELECT origin, vkey, contact, disabled FROM logs WHERE logID = ?", logID) if row.Err() != nil { - return omniwitness.Log{}, false, fmt.Errorf("failed to select from logs: %v", row.Err()) + return config.Log{}, false, fmt.Errorf("failed to select from logs: %v", row.Err()) } - c := omniwitness.Log{} + c := config.Log{} disabled := false if err := row.Scan(&c.Origin, &c.VKey, &c.Contact, &disabled); err != nil { - return omniwitness.Log{}, false, fmt.Errorf("failed to scan columns: %v", err) + return config.Log{}, false, fmt.Errorf("failed to scan columns: %v", err) } if disabled { klog.V(1).Infof("Ignoring disabled log %q", c.Origin) @@ -154,7 +154,7 @@ func (p *Persistence) Log(ctx context.Context, origin string) (omniwitness.Log, var err error c.Verifier, err = note.NewVerifier(c.VKey) if err != nil { - return omniwitness.Log{}, false, fmt.Errorf("failed to create verifier: %v", err) + return config.Log{}, false, fmt.Errorf("failed to create verifier: %v", err) } return c, true, nil } diff --git a/persistence/sqlite/sql_test.go b/persistence/sqlite/sql_test.go index 04dba3f..7c957fb 100644 --- a/persistence/sqlite/sql_test.go +++ b/persistence/sqlite/sql_test.go @@ -22,7 +22,7 @@ import ( "github.com/transparency-dev/formats/log" f_note "github.com/transparency-dev/formats/note" - "github.com/transparency-dev/witness/omniwitness" + "github.com/transparency-dev/witness/config" "github.com/transparency-dev/witness/witness" ptest "github.com/transparency-dev/witness/persistence/testonly" "golang.org/x/mod/sumdb/note" @@ -50,7 +50,7 @@ func TestLogConfig(t *testing.T) { defer func() { _ = shutdown() }() vkey := "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8" - logs := []omniwitness.Log{ + logs := []config.Log{ {Origin: "log1", VKey: vkey}, {Origin: "log2", VKey: vkey}, } @@ -62,7 +62,7 @@ func TestLogConfig(t *testing.T) { // Test Logs (list) sc := p.Logs(t.Context()) - gotLogs := make(map[string]omniwitness.Log) + gotLogs := make(map[string]config.Log) for l, err := range sc { if err != nil { t.Fatalf("Logs() error: %v", err) @@ -166,7 +166,7 @@ func TestDeadlock(t *testing.T) { if err != nil { t.Fatalf("NewVerifier: %v", err) } - err = p.AddLogs(t.Context(), []omniwitness.Log{ + err = p.AddLogs(t.Context(), []config.Log{ {Origin: "monkeys", VKey: mPK, Verifier: logV}, }) if err != nil {