Message validation (#1066)

* Fix issues in tests

* Fix linter

* Fix TestSSVMapping

* Attempt to fix fetchLogsInBatches

* Revert "Attempt to fix fetchLogsInBatches"

This reverts commit 96006c2.

* Attempt to fix fetchLogsInBatches

* Revert "Attempt to fix fetchLogsInBatches"

This reverts commit 5a3d9de.

* Revert "Fix TestSSVMapping"

This reverts commit c3263c1.

* Revert "Fix linter"

This reverts commit 8f72bb2.

* Revert "Fix issues in tests"

This reverts commit 0720abb.

* Change batch verifier params

* Disable signature check

* Revert "Disable signature check"

This reverts commit 311c722.

* Batch size 10

* Disable signature check

* Use BLSU for signature verification

* BLSU aggregate

* Disable partial signature check

* Disable consensus signature check

* Fix linter

* Revert "BLSU aggregate"

* Revert "Use BLSU for signature verification"

This reverts commit 6d4c15f.

* AggregateVerify for partial signature

* Simplify duplicated signer check

* Fix error text

* Fix check order

* Fix issues in tests

* Revert "Fix issues in tests"

This reverts commit b40355c.

* Move deployment to 5-8

* Fix some tests

* Fix data race

* Revert "Fix data race"

This reverts commit e3596a2.

* Attempt to fix data race

* Revert "Attempt to fix data race"

This reverts commit 38cce5f.

* Attempt to fix data race [2]

* Fix spec tests

* Fix a data race

* Fix a data race in tests bootstrapper

* Fix differ config

* Change message size metric to histogram

* Fix some tests

* Fix linter

* Fix unit tests

* More tests

* Improve tests

* More tests

* More tests [2]

* More tests [3]

* Fix incorrect usage of errors.As in tests

* More tests

* Deploy to 1-4 instead of 5-8

* Add logs with message processing duration

* Fix message validation duration buckets

* Add logs/metrics with signature check duration

* Disable duration logs

* Use single verification

* Revert "Use single verification"

This reverts commit cae01d2.

* More tests

* Fix max duties check

* More tests

* Fix duty count validation

* More tests

* More tests

* Fix duty count bug

* Add message queue metrics

* Fix a typo

* Revert "Fix message validation duration buckets"

This reverts commit 269340b.

* Fix validation duration buckets

* Cache serialized signature

* Fix variable name

* Revert "Fix variable name"

This reverts commit 15345a2.

* Revert "Cache serialized signature"

This reverts commit 20245e2.

* Queue len/cap as gauge

* Add message validation dashboard

* Enable signature checks

* Disable signature verification for non-committee non-decided

* Fix bug with committee check

* Verify partial signatures only if in committee

* Fix check

* Simplify code

* Add metrics if in committee

* Fix value of InvalidMessageDeliveriesWeight

* Deploy to 5-8 instead of 1-4

* Implement duty fetcher for message validation

* Get rid of optsTemplate in validatorsMap

* Make validators map exported and pass it to duty fetcher

* Fix panic due to context passing

* Simplify saving validators

* Debug panic

* Revert "Debug panic"

This reverts commit 76c273f.

* Revert "Simplify saving validators"

This reverts commit b24d704.

* Revert "Fix panic due to context passing"

This reverts commit 44b47f3.

* Revert "Make validators map exported and pass it to duty fetcher"

This reverts commit 82ee64d.

* Revert "Get rid of optsTemplate in validatorsMap"

This reverts commit 20cc1b7.

* Revert "Implement duty fetcher for message validation"

This reverts commit dabc10a.

* Implement duty fetcher for message validation

* Fix context panic

* Fix signature in message validation tests

* Get rid of optsTemplate in validatorsMap

* Fix condition

* Make validators map exported and pass it to duty fetcher

* Revert "Make validators map exported and pass it to duty fetcher"

This reverts commit 65729f3.

* Fix logic of validating beacon duty, add test

* Make validators map exported and pass it to duty fetcher

* Start duty fetcher

* Revert "Start duty fetcher"

This reverts commit cf3f51f.

* Revert "Make validators map exported and pass it to duty fetcher"

This reverts commit 563143c.

* Start duty fetcher

* Attempt to make validators map exported

* Move ActiveValidatorIndices to ValidatorsMap

* Use logger in DutyFetcher

* Pass validators map from node to validator controller

* Enable duty fetcher

* Revert "Enable duty fetcher"

This reverts commit 3648788.

* Extract validator creation logic from validators map

* Simplify creating validator

* Delete redundant comment

* Fix linter

* Init and start duty fetcher but don't pass to message validator

* Fix fetcher bug

* Use fetcher

* go mod tidy

* Duty fetcher tests

* Remove redundant file

* Fix imports

* Disable deployment

* Enable signature check for all messages

* Fix issues after merging

* Fix .gitlab-ci.yml

* Enable message validation in sync

* Revert "Enable signature check for all messages"

This reverts commit 8c692b9.

* Deploy to 5-8

* Try blst verify

* Try to fix error

* Enable signature check for all messages

* Revert "Enable signature check for all messages"

This reverts commit b0849c8.

* Revert "Try to fix error"

This reverts commit 0c9d35f.

* Revert "Try blst verify"

This reverts commit 38bbe35.

* Flag for signature verification in message validation

* Optimization for SingleVerifyByOperators

* Remove outdated comment

* Delete unused errors

* Try to fix bug with ErrTooManyDutiesPerEpoch

* Code review requests

* Try to fix bug with ErrTooManyDuties

* Delete ErrUnexpectedMessageOrder

* Delete ErrDecidedSignersSequence

* Try RSA verification instead of BLS in most places

* Enable signature check in message validation

* Revert "Enable signature check in message validation"

This reverts commit ef1a05c.

* Revert "Try RSA verification instead of BLS in most places"

This reverts commit 0e16b11.

* Don't check for signature check flag in protocol

* Revert "Don't check for signature check flag in protocol"

This reverts commit 9f46933.

* Add logs for config

* Debugging

* Debugging [2]

* Debugging [3]

* Debugging [4]

* Debugging [5]

* Debugging [6]

* Add passing signature check for non-committee validators

* Revert "Debugging [6]"

This reverts commit d4d99d0.

* Revert "Debugging [5]"

This reverts commit 425f30b.

* Revert "Debugging [4]"

This reverts commit a5ade77.

* Revert "Debugging [3]"

This reverts commit 97398bc.

* Revert "Debugging [2]"

This reverts commit 91b4507.

* Revert "Debugging"

This reverts commit adb769f.

* Revert "Add logs for config"

This reverts commit f016ace.

* Try RSA verification instead of BLS in most places

* Enable signature check in message validation

* Revert "Try RSA verification instead of BLS in most places"

This reverts commit 29a196e.

* Revert "Enable signature check in message validation"

This reverts commit 080d876.

* Try to fix bug with ErrTooManyDutiesPerEpoch

* fetch metadata for all validators

* LA Audit (#1136)

* Audits Directory

* add to README

* Update dashboard

* Revert "fetch metadata for all validators"

This reverts commit 68df177.

* fetch metadata for all validators

* Fix differ

* Attempt to fix duty fetcher

* Fix panic

* Fix duty fetcher tests

* Fix event handler tests

* Add ErrNoShareMetadata

* Add test for ErrNoShareMetadata

* Revert differ change

* Get rid of type assertions/switches

* Extract duty storage from duty scheduler

* Fetch all duties, process only committee duties for proposer

* Fetch all duties, process only committee duties for sync committee

* Deploy to 1-4 as well

* Fix potential nil ptr dereference

* Forbid consensus message with validator registration role

* Revert message queue changes

* Check message counts for partial signature messages

* Add role and round to metrics

* Update Grafana

* Update Grafana

* WIP on rejection/banning test

* rename `dutystorage` to `dutystore`

* refactor

* refator MessageValidator to an interface

* update Grafana dashboard

* Don't start validator registration duty if not attesting

* Test improvements from Andrew

* Fix docker build

* Update bind-tools

* Cleanup

* rename `CheckSignature` to `VerifySignatures`

* Enable nil config checks

* Create qbft config in message validation

* Revert "Enable nil config checks"

This reverts commit 5b547d9.

* Fix panic

* Fix panic [2]

* Add signature check option

* Add godoc

* Disable deployment

* Reset epoch/period for previous one instead of current one

* Revert "Disable deployment"

This reverts commit cb585d3.

* Fix duplication bug

* keep old duties

* Revert "Reset epoch/period for previous one instead of current one"

This reverts commit 0a803a0.

* Disable deployment

* Revert "Disable deployment"

This reverts commit cfd23f8.

* Cleanup a TODO

* Disable deployment

* Revert "Disable deployment"

This reverts commit cc0ab52.

* Pass context instead of logger to message router

* Add a test for GetSlotEndTime

* performance optimization

* change YAML representation of msg validation sig flag & add log

* extract batch verifier to separate file

* approve spec alignment

* fix test

* Delete debug.log

* Disable deployment

* Disable batch verifier

* Fix compilation

* Delete batch verifier

* Simplify GetSignerState

* Add a note about NTP

* go mod tidy

---------

Co-authored-by: moshe-blox <[email protected]>
Co-authored-by: moshe-blox <[email protected]>

Author: nkryuchkov

Dockerfile

@@ -9,7 +9,6 @@ RUN apt-get update                                                        && \
   git=1:2.39.2-1.1 \
   zip=3.0-13 \
   unzip=6.0-28 \
-  wget=1.21.3-1+b2 \
   g++=4:12.2.0-3 \
   gcc-aarch64-linux-gnu=4:12.2.0-3 \
   bzip2=1.0.8-5+b1 \
@@ -61,7 +60,7 @@ RUN apk -v --update add \
   ca-certificates=20230506-r0 \
   bash=5.2.15-r5 \
   make=4.4.1-r1 \
-  bind-tools=9.18.16-r0 && \
+  bind-tools=9.18.19-r0 && \
   rm /var/cache/apk/*
 
 COPY --from=builder /go/bin/ssvnode /go/bin/ssvnode

cli/operator/node.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/bloxapp/ssv/api/handlers"
 	apiserver "github.com/bloxapp/ssv/api/server"
-
 	"github.com/bloxapp/ssv/beacon/goclient"
 	global_config "github.com/bloxapp/ssv/cli/config"
 	"github.com/bloxapp/ssv/ekm"
@@ -34,6 +33,7 @@ import (
 	ssv_identity "github.com/bloxapp/ssv/identity"
 	"github.com/bloxapp/ssv/logging"
 	"github.com/bloxapp/ssv/logging/fields"
+	"github.com/bloxapp/ssv/message/validation"
 	"github.com/bloxapp/ssv/migrations"
 	"github.com/bloxapp/ssv/monitoring/metrics"
 	"github.com/bloxapp/ssv/monitoring/metricsreporter"
@@ -42,9 +42,11 @@ import (
 	"github.com/bloxapp/ssv/networkconfig"
 	"github.com/bloxapp/ssv/nodeprobe"
 	"github.com/bloxapp/ssv/operator"
+	"github.com/bloxapp/ssv/operator/duties/dutystore"
 	"github.com/bloxapp/ssv/operator/slot_ticker"
 	operatorstorage "github.com/bloxapp/ssv/operator/storage"
 	"github.com/bloxapp/ssv/operator/validator"
+	"github.com/bloxapp/ssv/operator/validatorsmap"
 	beaconprotocol "github.com/bloxapp/ssv/protocol/v2/blockchain/beacon"
 	"github.com/bloxapp/ssv/protocol/v2/types"
 	registrystorage "github.com/bloxapp/ssv/registry/storage"
@@ -60,6 +62,10 @@ type KeyStore struct {
 	PasswordFile   string `yaml:"PasswordFile" env:"PASSWORD_FILE" env-description:"Password for operator private key file decryption"`
 }
 
+type MessageValidation struct {
+	VerifySignatures bool `yaml:"VerifySignatures" env:"MESSAGE_VALIDATION_VERIFY_SIGNATURES" env-default:"false" env-description:"Experimental feature to verify signatures in pubsub's message validation instead of in consensus protocol."`
+}
+
 type config struct {
 	global_config.GlobalConfig `yaml:"global"`
 	DBOptions                  basedb.Options                   `yaml:"db"`
@@ -72,13 +78,11 @@ type config struct {
 	MetricsAPIPort             int                              `yaml:"MetricsAPIPort" env:"METRICS_API_PORT" env-description:"Port to listen on for the metrics API."`
 	EnableProfile              bool                             `yaml:"EnableProfile" env:"ENABLE_PROFILE" env-description:"flag that indicates whether go profiling tools are enabled"`
 	NetworkPrivateKey          string                           `yaml:"NetworkPrivateKey" env:"NETWORK_PRIVATE_KEY" env-description:"private key for network identity"`
-
-	WsAPIPort int  `yaml:"WebSocketAPIPort" env:"WS_API_PORT" env-description:"Port to listen on for the websocket API."`
-	WithPing  bool `yaml:"WithPing" env:"WITH_PING" env-description:"Whether to send websocket ping messages'"`
-
-	SSVAPIPort int `yaml:"SSVAPIPort" env:"SSV_API_PORT" env-description:"Port to listen on for the SSV API."`
-
-	LocalEventsPath string `yaml:"LocalEventsPath" env:"EVENTS_PATH" env-description:"path to local events"`
+	WsAPIPort                  int                              `yaml:"WebSocketAPIPort" env:"WS_API_PORT" env-description:"Port to listen on for the websocket API."`
+	WithPing                   bool                             `yaml:"WithPing" env:"WITH_PING" env-description:"Whether to send websocket ping messages'"`
+	SSVAPIPort                 int                              `yaml:"SSVAPIPort" env:"SSV_API_PORT" env-description:"Port to listen on for the SSV API."`
+	LocalEventsPath            string                           `yaml:"LocalEventsPath" env:"EVENTS_PATH" env-description:"path to local events"`
+	MessageValidation          MessageValidation                `yaml:"MessageValidation"`
 }
 
 var cfg config
@@ -97,6 +101,11 @@ var StartNodeCmd = &cobra.Command{
 			log.Fatal("could not create logger", err)
 		}
 		defer logging.CapturePanic(logger)
+
+		metricsReporter := metricsreporter.New(
+			metricsreporter.WithLogger(logger),
+		)
+
 		networkConfig, err := setupSSVNetwork(logger)
 		if err != nil {
 			logger.Fatal("could not setup network", zap.Error(err))
@@ -128,23 +137,9 @@ var StartNodeCmd = &cobra.Command{
 			return currentEpoch >= cfg.P2pNetworkConfig.PermissionedActivateEpoch && currentEpoch < cfg.P2pNetworkConfig.PermissionedDeactivateEpoch
 		}
 
-		cfg.P2pNetworkConfig.Permissioned = permissioned
-		cfg.P2pNetworkConfig.WhitelistedOperatorKeys = append(cfg.P2pNetworkConfig.WhitelistedOperatorKeys, networkConfig.WhitelistedOperatorKeys...)
-		cfg.P2pNetworkConfig.NodeStorage = nodeStorage
-		cfg.P2pNetworkConfig.OperatorID = format.OperatorID(operatorData.PublicKey)
-		cfg.P2pNetworkConfig.FullNode = cfg.SSVOptions.ValidatorOptions.FullNode
-		cfg.P2pNetworkConfig.Network = networkConfig
-
-		p2pNetwork := setupP2P(logger, db)
-
 		slotTicker := slot_ticker.NewTicker(cmd.Context(), networkConfig)
 
-		metricsReporter := metricsreporter.New(
-			metricsreporter.WithLogger(logger),
-		)
-
 		cfg.ConsensusClient.Context = cmd.Context()
-
 		cfg.ConsensusClient.Graffiti = []byte("SSV.Network")
 		cfg.ConsensusClient.GasLimit = spectypes.DefaultGasLimit
 		cfg.ConsensusClient.Network = networkConfig.Beacon.GetNetwork()
@@ -166,6 +161,36 @@ var StartNodeCmd = &cobra.Command{
 			logger.Fatal("could not connect to execution client", zap.Error(err))
 		}
 
+		cfg.P2pNetworkConfig.Permissioned = permissioned
+		cfg.P2pNetworkConfig.WhitelistedOperatorKeys = append(cfg.P2pNetworkConfig.WhitelistedOperatorKeys, networkConfig.WhitelistedOperatorKeys...)
+		cfg.P2pNetworkConfig.NodeStorage = nodeStorage
+		cfg.P2pNetworkConfig.OperatorID = format.OperatorID(operatorData.PublicKey)
+		cfg.P2pNetworkConfig.FullNode = cfg.SSVOptions.ValidatorOptions.FullNode
+		cfg.P2pNetworkConfig.Network = networkConfig
+
+		validatorsMap := validatorsmap.New(cmd.Context())
+
+		dutyStore := dutystore.New()
+		cfg.SSVOptions.DutyStore = dutyStore
+
+		messageValidator := validation.NewMessageValidator(
+			networkConfig,
+			validation.WithShareStorage(nodeStorage.Shares()),
+			validation.WithLogger(logger),
+			validation.WithMetrics(metricsReporter),
+			validation.WithDutyStore(dutyStore),
+			validation.WithOwnOperatorID(operatorData.ID),
+			validation.WithSignatureVerification(cfg.MessageValidation.VerifySignatures),
+		)
+
+		cfg.P2pNetworkConfig.Metrics = metricsReporter
+		cfg.P2pNetworkConfig.MessageValidator = messageValidator
+		cfg.SSVOptions.ValidatorOptions.MessageValidator = messageValidator
+		// if signature check is enabled in message validation then it's disabled in validator controller and vice versa
+		cfg.SSVOptions.ValidatorOptions.VerifySignatures = !cfg.MessageValidation.VerifySignatures
+
+		p2pNetwork := setupP2P(logger, db)
+
 		cfg.SSVOptions.Context = cmd.Context()
 		cfg.SSVOptions.DB = db
 		cfg.SSVOptions.BeaconNode = consensusClient
@@ -178,6 +203,7 @@ var StartNodeCmd = &cobra.Command{
 		cfg.SSVOptions.ValidatorOptions.Network = p2pNetwork
 		cfg.SSVOptions.ValidatorOptions.Beacon = consensusClient
 		cfg.SSVOptions.ValidatorOptions.KeyManager = keyManager
+		cfg.SSVOptions.ValidatorOptions.ValidatorsMap = validatorsMap
 
 		cfg.SSVOptions.ValidatorOptions.ShareEncryptionKeyProvider = nodeStorage.GetPrivateKey
 		cfg.SSVOptions.ValidatorOptions.OperatorData = operatorData
@@ -209,10 +235,10 @@ var StartNodeCmd = &cobra.Command{
 
 		cfg.SSVOptions.ValidatorOptions.StorageMap = storageMap
 		cfg.SSVOptions.ValidatorOptions.Metrics = metricsReporter
+		cfg.SSVOptions.Metrics = metricsReporter
 
 		validatorCtrl := validator.NewController(logger, cfg.SSVOptions.ValidatorOptions)
 		cfg.SSVOptions.ValidatorController = validatorCtrl
-		cfg.SSVOptions.Metrics = metricsReporter
 
 		operatorNode = operator.New(logger, cfg.SSVOptions, slotTicker)
 
@@ -477,10 +503,7 @@ func setupSSVNetwork(logger *zap.Logger) (networkconfig.NetworkConfig, error) {
 	return networkConfig, nil
 }
 
-func setupP2P(
-	logger *zap.Logger,
-	db basedb.Database,
-) network.P2PNetwork {
+func setupP2P(logger *zap.Logger, db basedb.Database) network.P2PNetwork {
 	istore := ssv_identity.NewIdentityStore(db)
 	netPrivKey, err := istore.SetupNetworkKey(logger, cfg.NetworkPrivateKey)
 	if err != nil {