From 92b6aa4ea665734207c13155f3cf3fcf8d3f9c1f Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Wed, 24 Sep 2025 17:51:21 +0200 Subject: [PATCH 01/14] add prometheus setup --- cli/flags.go | 20 ++++++++++++++++++++ cli/main.go | 12 ++++++++++++ go.mod | 9 +++++++++ go.sum | 16 ++++++++++++++++ server/service.go | 25 +++++++++++++++++++++++++ 5 files changed, 82 insertions(+) diff --git a/cli/flags.go b/cli/flags.go index ab8760bd..dd182be8 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -7,6 +7,7 @@ const ( GenesisCategory = "GENESIS" RelayCategory = "RELAYS" GeneralCategory = "GENERAL" + Metrics = "METRICS" ) var flags = []cli.Flag{ @@ -35,6 +36,10 @@ var flags = []cli.Flag{ timeoutGetPayloadFlag, timeoutRegValFlag, maxRetriesFlag, + + //metrics + metricsFlag, + metricsAddrFlag, } var ( @@ -178,4 +183,19 @@ var ( Value: 5, Category: RelayCategory, } + + //metrics + metricsFlag = &cli.BoolFlag{ + Name: "metrics", + Sources: cli.EnvVars("METRICS_ENABLED"), + Usage: "enables a metrics server", + Category: Metrics, + } + metricsAddrFlag = &cli.StringFlag{ + Name: "metrics-addr", + Sources: cli.EnvVars("METRICS_ADDR"), + Value: "localhost:18551", + Usage: "listening address for the metrics server", + Category: Metrics, + } ) diff --git a/cli/main.go b/cli/main.go index 9de3664b..b7f7ef60 100644 --- a/cli/main.go +++ b/cli/main.go @@ -68,6 +68,8 @@ func start(_ context.Context, cmd *cli.Command) error { genesisForkVersion, genesisTime = setupGenesis(cmd) relays, minBid, relayCheck = setupRelays(cmd) listenAddr = cmd.String(addrFlag.Name) + metricsEnabled = cmd.Bool(metricsFlag.Name) + metricsAddr = cmd.String(metricsAddrFlag.Name) ) opts := server.BoostServiceOpts{ @@ -82,6 +84,7 @@ func start(_ context.Context, cmd *cli.Command) error { RequestTimeoutGetPayload: time.Duration(cmd.Int(timeoutGetPayloadFlag.Name)) * time.Millisecond, RequestTimeoutRegVal: time.Duration(cmd.Int(timeoutRegValFlag.Name)) * time.Millisecond, RequestMaxRetries: cmd.Int(maxRetriesFlag.Name), + MetricsAddr: metricsAddr, } service, err := server.NewBoostService(opts) if err != nil { @@ -92,6 +95,15 @@ func start(_ context.Context, cmd *cli.Command) error { log.Error("no relay passed the health-check!") } + if metricsEnabled { + go func() { + log.Infof("metrics server listening on %v", opts.MetricsAddr) + if err := service.StartMetricsServer(); err != nil { + log.WithError(err).Error("metrics server exited with error") + } + }() + } + log.Infof("listening on %v", listenAddr) return service.StartHTTPServer() } diff --git a/go.mod b/go.mod index c7a038a8..d7b4a28f 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/holiman/uint256 v1.3.2 + github.com/prometheus/client_golang v1.20.5 github.com/prysmaticlabs/go-bitfield v0.0.0-20240618144021-706c95b2dd15 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.10.0 @@ -17,7 +18,9 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/bavard v0.1.30 // indirect github.com/consensys/gnark-crypto v0.17.0 // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect @@ -27,11 +30,17 @@ require ( github.com/ethereum/go-verkle v0.2.2 // indirect github.com/goccy/go-yaml v1.17.1 // indirect github.com/gofrs/flock v0.12.1 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/supranational/blst v0.3.14 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/sync v0.13.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index b48e3377..a533275a 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/attestantio/go-builder-client v0.7.2 h1:bOrtysEIZd9bEM+mAeT6OtAo6LSAf github.com/attestantio/go-builder-client v0.7.2/go.mod h1:+NADxbaknI5yxl+0mCkMa/VciVsesxRMGNP/poDfV08= github.com/attestantio/go-eth2-client v0.27.1 h1:g7bm+gG/p+gfzYdEuxuAepVWYb8EO+2KojV5/Lo2BxM= github.com/attestantio/go-eth2-client v0.27.1/go.mod h1:fvULSL9WtNskkOB4i+Yyr6BKpNHXvmpGZj9969fCrfY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -59,6 +61,8 @@ github.com/huandu/go-clone v1.7.2 h1:3+Aq0Ed8XK+zKkLjE2dfHg0XrpIfcohBE1K+c8Usxoo github.com/huandu/go-clone v1.7.2/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= github.com/huandu/go-clone/generic v1.6.0 h1:Wgmt/fUZ28r16F2Y3APotFD59sHk1p78K0XLdbUYN5U= github.com/huandu/go-clone/generic v1.6.0/go.mod h1:xgd9ZebcMsBWWcBx5mVMCoqMX24gLWr5lQicr+nVXNs= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -79,12 +83,22 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prysmaticlabs/go-bitfield v0.0.0-20240618144021-706c95b2dd15 h1:lC8kiphgdOBTcbTvo8MwkvpKjO0SlAgjv4xIK5FGJ94= github.com/prysmaticlabs/go-bitfield v0.0.0-20240618144021-706c95b2dd15/go.mod h1:8svFBIKKu31YriBG/pNizo9N0Jr9i5PQ+dFkxWg3x5k= github.com/prysmaticlabs/gohashtree v0.0.4-beta h1:H/EbCuXPeTV3lpKeXGPpEV9gsUpkqOOVnWapUyeWro4= @@ -131,6 +145,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/server/service.go b/server/service.go index 60e2a6db..c70e89e0 100644 --- a/server/service.go +++ b/server/service.go @@ -21,6 +21,9 @@ import ( "github.com/flashbots/mev-boost/server/types" "github.com/google/uuid" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" goacceptheaders "github.com/timewasted/go-accept-headers" ) @@ -63,6 +66,8 @@ type BoostServiceOpts struct { RequestTimeoutGetPayload time.Duration RequestTimeoutRegVal time.Duration RequestMaxRetries int + + MetricsAddr string } // BoostService - the mev-boost service @@ -86,6 +91,8 @@ type BoostService struct { slotUID *slotUID slotUIDLock sync.Mutex + + metricsAddr string } // NewBoostService created a new BoostService @@ -187,6 +194,24 @@ func (m *BoostService) StartHTTPServer() error { return err } +// StartMetricsServer starts the HTTP server for exporting open-metrics +func (m *BoostService) StartMetricsServer() error { + prometheusRegistry := prometheus.NewRegistry() + if err := prometheusRegistry.Register(collectors.NewGoCollector()); err != nil { + m.log.WithError(err).Error("failed to register metrics for GoCollector") + } + if err := prometheusRegistry.Register(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})); err != nil { + m.log.WithError(err).Error("failed to register ProcessCollector") + } + + serveMux := http.NewServeMux() + serveMux.Handle("/metrics", promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{ + ErrorLog: m.log, + EnableOpenMetrics: true, + })) + return http.ListenAndServe(m.metricsAddr, serveMux) +} + func (m *BoostService) startBidCacheCleanupTask() { for { time.Sleep(1 * time.Minute) From f3a9e174e7bf1681224c6abe6bfb9dfadd87f07d Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Thu, 25 Sep 2025 14:23:58 +0200 Subject: [PATCH 02/14] add beacon node metrics --- server/metrics.go | 64 +++++++++++++++++++++++++++++++++++++++++++++++ server/service.go | 59 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 server/metrics.go diff --git a/server/metrics.go b/server/metrics.go new file mode 100644 index 00000000..a4b542ae --- /dev/null +++ b/server/metrics.go @@ -0,0 +1,64 @@ +package server + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" +) + +const Namespace = "mev_boost" + +var ( + metricsInitOnce sync.Once + BeaconNodeStatus *prometheus.CounterVec + RelayHeaderValue *prometheus.GaugeVec + RelayLatency *prometheus.HistogramVec + RelayStatusCode *prometheus.CounterVec +) + +func RegisterMetrics(registry *prometheus.Registry) { + metricsInitOnce.Do(func() { + BeaconNodeStatus = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Name: "beacon_node_status_code", + Help: "http status code returned to beacon node", + }, + []string{"http_status_code", "endpoint"}, + ) + + RelayHeaderValue = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Name: "relay_header_value", + Help: "header value in gwei delivered by relay", + }, + []string{"relay"}, + ) + + RelayLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: Namespace, + Name: "relay_latency", + Help: "http latency by relay", + }, + []string{"endpoint", "relay"}, + ) + + RelayStatusCode = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Name: "relay_status_code", + Help: "http status code received by relay", + }, + []string{"http_status_code", "endpoint", "relay"}, + ) + + registry.MustRegister( + BeaconNodeStatus, + RelayHeaderValue, + RelayLatency, + RelayStatusCode, + ) + }) +} diff --git a/server/service.go b/server/service.go index c70e89e0..d3c957bb 100644 --- a/server/service.go +++ b/server/service.go @@ -203,6 +203,7 @@ func (m *BoostService) StartMetricsServer() error { if err := prometheusRegistry.Register(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})); err != nil { m.log.WithError(err).Error("failed to register ProcessCollector") } + RegisterMetrics(prometheusRegistry) serveMux := http.NewServeMux() serveMux.Handle("/metrics", promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{ @@ -234,8 +235,14 @@ func (m *BoostService) handleRoot(w http.ResponseWriter, _ *http.Request) { func (m *BoostService) handleStatus(w http.ResponseWriter, _ *http.Request) { w.Header().Set(HeaderKeyVersion, config.Version) if !m.relayCheck || m.CheckRelays() > 0 { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathStatus).Inc() + } m.respondOK(w, nilResponse) } else { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusServiceUnavailable), params.PathStatus).Inc() + } m.respondError(w, http.StatusServiceUnavailable, "all relays are unavailable") } } @@ -259,6 +266,9 @@ func (m *BoostService) handleRegisterValidator(w http.ResponseWriter, req *http. // Read the validator registrations regBytes, err := io.ReadAll(req.Body) if err != nil { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusInternalServerError), params.PathRegisterValidator).Inc() + } m.respondError(w, http.StatusInternalServerError, err.Error()) return } @@ -268,9 +278,15 @@ func (m *BoostService) handleRegisterValidator(w http.ResponseWriter, req *http. err = m.registerValidator(log, regBytes, header) if err == nil { // One of the relays responded OK + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathRegisterValidator).Inc() + } m.respondOK(w, nilResponse) } else { // None of the relays responded OK + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadGateway), params.PathRegisterValidator).Inc() + } m.respondError(w, http.StatusBadGateway, err.Error()) } } @@ -290,6 +306,9 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) // Parse the slot slotValue, err := strconv.ParseUint(vars["slot"], 10, 64) if err != nil { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadRequest), params.PathGetHeader).Inc() + } m.respondError(w, http.StatusBadRequest, errInvalidSlot.Error()) return } @@ -309,6 +328,9 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) // Query the relays for the header result, err := m.getHeader(log, slot, pubkey, parentHashHex, ua, rawProposerAcceptContentTypes) if err != nil { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadRequest), params.PathGetHeader).Inc() + } m.respondError(w, http.StatusBadRequest, err.Error()) return } @@ -316,6 +338,9 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) // Bail if none of the relays returned a bid if result.response.IsEmpty() { log.Info("no bid received") + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusNoContent), params.PathGetHeader).Inc() + } w.WriteHeader(http.StatusNoContent) return } @@ -351,12 +376,22 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) // Respond appropriately switch proposerPreferredContentType { case MediaTypeJSON: + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathGetHeader).Inc() + } log.Debug("responding with JSON") m.respondGetHeaderJSON(w, &result) + case MediaTypeOctetStream: + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathGetHeader).Inc() + } log.Debug("responding with SSZ") m.respondGetHeaderSSZ(w, &result) default: + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusNotAcceptable), params.PathGetHeader).Inc() + } message := fmt.Sprintf("unsupported media type: %s", proposerPreferredContentType) log.Error(message) m.respondError(w, http.StatusNotAcceptable, message) @@ -387,6 +422,9 @@ func (m *BoostService) handleGetPayload(w http.ResponseWriter, req *http.Request // Read the body first, so we can log it later on error signedBlindedBlockBytes, err := io.ReadAll(req.Body) if err != nil { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadRequest), params.PathGetPayload).Inc() + } log.WithError(err).Error("could not read body of request from the beacon node") m.respondError(w, http.StatusBadRequest, err.Error()) return @@ -397,6 +435,9 @@ func (m *BoostService) handleGetPayload(w http.ResponseWriter, req *http.Request // If no payload has been received from relay, log loudly about withholding! if result == nil || getPayloadResponseIsEmpty(result) { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadGateway), params.PathGetPayload).Inc() + } originRelays := types.RelayEntriesToStrings(originalBid.relays) log.WithField("relaysWithBid", strings.Join(originRelays, ", ")).Error("no payload received from relay!") m.respondError(w, http.StatusBadGateway, errNoSuccessfulRelayResponse.Error()) @@ -419,12 +460,21 @@ func (m *BoostService) handleGetPayload(w http.ResponseWriter, req *http.Request // Respond appropriately switch proposerPreferredContentType { case MediaTypeJSON: + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathGetPayload).Inc() + } log.Debug("responding with JSON") m.respondGetPayloadJSON(w, result) case MediaTypeOctetStream: + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathGetPayload).Inc() + } log.Debug("responding with SSZ") m.respondGetPayloadSSZ(w, result) default: + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusNotAcceptable), params.PathGetPayload).Inc() + } message := fmt.Sprintf("unsupported media type: %s", proposerPreferredContentType) log.Error(message) m.respondError(w, http.StatusNotAcceptable, message) @@ -452,6 +502,9 @@ func (m *BoostService) handleGetPayloadV2(w http.ResponseWriter, req *http.Reque // Read the body first, so we can log it later on error signedBlindedBlockBytes, err := io.ReadAll(req.Body) if err != nil { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadRequest), params.PathGetPayloadV2).Inc() + } log.WithError(err).Error("could not read body of request from the beacon node") m.respondError(w, http.StatusBadRequest, err.Error()) return @@ -462,12 +515,18 @@ func (m *BoostService) handleGetPayloadV2(w http.ResponseWriter, req *http.Reque // If no relay accepted the submission, log about the failure if !success { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadGateway), params.PathGetPayloadV2).Inc() + } originRelays := types.RelayEntriesToStrings(originalBid.relays) log.WithField("relaysWithBid", strings.Join(originRelays, ", ")).Error("no relay accepted the signed blinded beacon block submission!") m.respondError(w, http.StatusBadGateway, errNoSuccessfulRelayResponse.Error()) return } + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusAccepted), params.PathGetPayloadV2).Inc() + } log.Info("successfully submitted signed blinded beacon block to relay") w.WriteHeader(http.StatusAccepted) } From 5b8b81780c20810a070ea00031bf1342f060a138 Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Thu, 25 Sep 2025 14:39:29 +0200 Subject: [PATCH 03/14] add metrics for relay header --- server/get_header.go | 5 +++++ server/metrics.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server/get_header.go b/server/get_header.go index ee1aec8c..9fa72245 100644 --- a/server/get_header.go +++ b/server/get_header.go @@ -207,6 +207,11 @@ func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, pa log.Debug("bid received") + if RelayHeaderValue != nil { + valueEthFloat64, _ := valueEth.Float64() + RelayHeaderValue.WithLabelValues(relay.String()).Set(valueEthFloat64) + } + // Skip if value is lower than the minimum bid if bidInfo.value.CmpBig(m.relayMinBid.BigInt()) == -1 { log.Debug("ignoring bid below min-bid value") diff --git a/server/metrics.go b/server/metrics.go index a4b542ae..dafc5ca2 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -31,7 +31,7 @@ func RegisterMetrics(registry *prometheus.Registry) { prometheus.GaugeOpts{ Namespace: Namespace, Name: "relay_header_value", - Help: "header value in gwei delivered by relay", + Help: "header value delivered by relay", }, []string{"relay"}, ) From 42a06e7aed090287aefbdff216bb819022312a4a Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Thu, 25 Sep 2025 15:05:39 +0200 Subject: [PATCH 04/14] add relay latency metric --- server/get_header.go | 5 +++++ server/get_payload.go | 10 +++++++++- server/register_validator.go | 5 +++++ server/service.go | 4 ++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/server/get_header.go b/server/get_header.go index 9fa72245..0e7f3b0c 100644 --- a/server/get_header.go +++ b/server/get_header.go @@ -19,6 +19,7 @@ import ( "github.com/attestantio/go-eth2-client/spec" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/flashbots/mev-boost/config" + "github.com/flashbots/mev-boost/server/params" "github.com/flashbots/mev-boost/server/types" "github.com/google/uuid" "github.com/sirupsen/logrus" @@ -94,7 +95,11 @@ func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, pa // Send the request log.Debug("requesting header") + start := time.Now() resp, err := m.httpClientGetHeader.Do(req) + if RelayLatency != nil { + RelayLatency.WithLabelValues(params.PathGetHeader, relay.String()).Observe(float64(time.Since(start).Milliseconds())) + } if err != nil { log.WithError(err).Warn("error calling getHeader on relay") return diff --git a/server/get_payload.go b/server/get_payload.go index eba025d0..60afef0a 100644 --- a/server/get_payload.go +++ b/server/get_payload.go @@ -212,9 +212,17 @@ func (m *BoostService) innerGetPayload(log *logrus.Entry, signedBlindedBeaconBlo req.Header.Set(HeaderDateMilliseconds, fmt.Sprintf("%d", time.Now().UTC().UnixMilli())) req.Header.Set(HeaderUserAgent, userAgent) - // Send the request + // Send the request and record latency log.Debug("submitting signed blinded block") + start := time.Now() resp, err := m.httpClientGetPayload.Do(req) + if RelayLatency != nil { + endpoint := params.PathGetPayload + if version == GetPayloadV2 { + endpoint = params.PathGetPayloadV2 + } + RelayLatency.WithLabelValues(endpoint, relay.String()).Observe(float64(time.Since(start).Milliseconds())) + } if err != nil { log.WithError(err).Warnf("error calling getPayload%s on relay", version) return nil, err diff --git a/server/register_validator.go b/server/register_validator.go index 08bae69c..7d67030b 100644 --- a/server/register_validator.go +++ b/server/register_validator.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "net/http" + "time" "github.com/flashbots/mev-boost/server/params" "github.com/flashbots/mev-boost/server/types" @@ -45,7 +46,11 @@ func (m *BoostService) registerValidator(log *logrus.Entry, regBytes []byte, hea }).Debug("sending the registerValidator request") // Send the request + start := time.Now() resp, err := m.httpClientRegVal.Do(req) + if RelayLatency != nil { + RelayLatency.WithLabelValues(params.PathRegisterValidator, relay.String()).Observe(float64(time.Since(start).Milliseconds())) + } if err != nil { log.WithError(err).Warn("error calling registerValidator on relay") respErrCh <- err diff --git a/server/service.go b/server/service.go index d3c957bb..1084cc55 100644 --- a/server/service.go +++ b/server/service.go @@ -545,7 +545,11 @@ func (m *BoostService) CheckRelays() int { log := m.log.WithField("url", url) log.Debug("checking relay status") + start := time.Now() code, err := SendHTTPRequest(context.Background(), m.httpClientGetHeader, http.MethodGet, url, "", nil, nil, nil) + if RelayLatency != nil { + RelayLatency.WithLabelValues(params.PathStatus, relay.String()).Observe(float64(time.Since(start).Milliseconds())) + } if err != nil { log.WithError(err).Error("relay status error - request failed") return From c6096a4fba394d3bcd17eddc9f671ed5e8bd5ce5 Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Thu, 25 Sep 2025 15:22:00 +0200 Subject: [PATCH 05/14] add relays status code metrics --- server/get_header.go | 3 +++ server/get_payload.go | 18 +++++++++--------- server/register_validator.go | 4 ++++ server/service.go | 3 +++ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/server/get_header.go b/server/get_header.go index 0e7f3b0c..b7cb5b2f 100644 --- a/server/get_header.go +++ b/server/get_header.go @@ -106,6 +106,9 @@ func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, pa } defer resp.Body.Close() + if RelayStatusCode != nil { + RelayStatusCode.WithLabelValues(strconv.Itoa(resp.StatusCode), params.PathGetHeader, relay.String()).Inc() + } // Check if no header is available if resp.StatusCode == http.StatusNoContent { log.Debug("no-content response") diff --git a/server/get_payload.go b/server/get_payload.go index 60afef0a..60be1060 100644 --- a/server/get_payload.go +++ b/server/get_payload.go @@ -10,6 +10,7 @@ import ( "mime" "net/http" "slices" + "strconv" "sync/atomic" "time" @@ -212,15 +213,17 @@ func (m *BoostService) innerGetPayload(log *logrus.Entry, signedBlindedBeaconBlo req.Header.Set(HeaderDateMilliseconds, fmt.Sprintf("%d", time.Now().UTC().UnixMilli())) req.Header.Set(HeaderUserAgent, userAgent) + statusCode := http.StatusOK + endpoint := params.PathGetPayload + if version == GetPayloadV2 { + statusCode = http.StatusAccepted + endpoint = params.PathGetPayloadV2 + } // Send the request and record latency log.Debug("submitting signed blinded block") start := time.Now() resp, err := m.httpClientGetPayload.Do(req) if RelayLatency != nil { - endpoint := params.PathGetPayload - if version == GetPayloadV2 { - endpoint = params.PathGetPayloadV2 - } RelayLatency.WithLabelValues(endpoint, relay.String()).Observe(float64(time.Since(start).Milliseconds())) } if err != nil { @@ -228,11 +231,8 @@ func (m *BoostService) innerGetPayload(log *logrus.Entry, signedBlindedBeaconBlo return nil, err } - var statusCode int - if version == GetPayloadV1 { - statusCode = http.StatusOK - } else { - statusCode = http.StatusAccepted + if RelayStatusCode != nil { + RelayStatusCode.WithLabelValues(strconv.Itoa(statusCode), endpoint, relay.String()).Inc() } // Check that the response was successful if resp.StatusCode != statusCode { diff --git a/server/register_validator.go b/server/register_validator.go index 7d67030b..7fd188df 100644 --- a/server/register_validator.go +++ b/server/register_validator.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "net/http" + "strconv" "time" "github.com/flashbots/mev-boost/server/params" @@ -58,6 +59,9 @@ func (m *BoostService) registerValidator(log *logrus.Entry, regBytes []byte, hea } resp.Body.Close() + if RelayStatusCode != nil { + RelayStatusCode.WithLabelValues(strconv.Itoa(resp.StatusCode), params.PathRegisterValidator, relay.String()).Inc() + } // Check if response is successful if resp.StatusCode == http.StatusOK { log.Debug("relay accepted registrations") diff --git a/server/service.go b/server/service.go index 1084cc55..4e8486fd 100644 --- a/server/service.go +++ b/server/service.go @@ -554,6 +554,9 @@ func (m *BoostService) CheckRelays() int { log.WithError(err).Error("relay status error - request failed") return } + if RelayStatusCode != nil { + RelayStatusCode.WithLabelValues(strconv.Itoa(code), params.PathStatus, relay.String()).Inc() + } if code == http.StatusOK { log.Debug("relay status OK") } else { From 7b272c5606b2d69f5b11b0fa6b8250be3071dc2e Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Thu, 25 Sep 2025 15:27:30 +0200 Subject: [PATCH 06/14] fix lint --- cli/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/flags.go b/cli/flags.go index dd182be8..f147c12b 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -37,7 +37,7 @@ var flags = []cli.Flag{ timeoutRegValFlag, maxRetriesFlag, - //metrics + // metrics metricsFlag, metricsAddrFlag, } @@ -184,7 +184,7 @@ var ( Category: RelayCategory, } - //metrics + // metrics metricsFlag = &cli.BoolFlag{ Name: "metrics", Sources: cli.EnvVars("METRICS_ENABLED"), From 3da177639ec45633459b7d9dfdb0b53994d8337e Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Thu, 25 Sep 2025 16:42:49 +0200 Subject: [PATCH 07/14] fix lint --- server/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/metrics.go b/server/metrics.go index dafc5ca2..dd0d9014 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -21,7 +21,7 @@ func RegisterMetrics(registry *prometheus.Registry) { BeaconNodeStatus = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: Namespace, - Name: "beacon_node_status_code", + Name: "beacon_node_status_code_total", Help: "http status code returned to beacon node", }, []string{"http_status_code", "endpoint"}, @@ -48,7 +48,7 @@ func RegisterMetrics(registry *prometheus.Registry) { RelayStatusCode = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: Namespace, - Name: "relay_status_code", + Name: "relay_status_code_total", Help: "http status code received by relay", }, []string{"http_status_code", "endpoint", "relay"}, From 14c071b29e906900047e563b7fbdcc49ad6fe7a8 Mon Sep 17 00:00:00 2001 From: Bharath Vedartham Date: Thu, 25 Sep 2025 22:27:17 +0530 Subject: [PATCH 08/14] add prometheus setup --- cli/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/flags.go b/cli/flags.go index f147c12b..dd182be8 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -37,7 +37,7 @@ var flags = []cli.Flag{ timeoutRegValFlag, maxRetriesFlag, - // metrics + //metrics metricsFlag, metricsAddrFlag, } @@ -184,7 +184,7 @@ var ( Category: RelayCategory, } - // metrics + //metrics metricsFlag = &cli.BoolFlag{ Name: "metrics", Sources: cli.EnvVars("METRICS_ENABLED"), From c494d4aa5671139513ab1185cb96c8f847c9928d Mon Sep 17 00:00:00 2001 From: Bharath Vedartham Date: Thu, 25 Sep 2025 22:21:25 +0530 Subject: [PATCH 09/14] add some metrics --- server/get_header.go | 24 ++++---- server/get_payload.go | 8 +-- server/metrics.go | 103 +++++++++++++++++++++++++++++++---- server/register_validator.go | 8 +-- server/service.go | 84 +++++++--------------------- 5 files changed, 131 insertions(+), 96 deletions(-) diff --git a/server/get_header.go b/server/get_header.go index b7cb5b2f..c70d768f 100644 --- a/server/get_header.go +++ b/server/get_header.go @@ -97,18 +97,14 @@ func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, pa log.Debug("requesting header") start := time.Now() resp, err := m.httpClientGetHeader.Do(req) - if RelayLatency != nil { - RelayLatency.WithLabelValues(params.PathGetHeader, relay.String()).Observe(float64(time.Since(start).Milliseconds())) - } + RecordRelayLatency(params.PathGetHeader, relay.String(), float64(time.Since(start).Microseconds())) if err != nil { log.WithError(err).Warn("error calling getHeader on relay") return } defer resp.Body.Close() - if RelayStatusCode != nil { - RelayStatusCode.WithLabelValues(strconv.Itoa(resp.StatusCode), params.PathGetHeader, relay.String()).Inc() - } + RecordRelayStatusCode(strconv.Itoa(resp.StatusCode), params.PathGetHeader, relay.String()) // Check if no header is available if resp.StatusCode == http.StatusNoContent { log.Debug("no-content response") @@ -215,14 +211,15 @@ func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, pa log.Debug("bid received") - if RelayHeaderValue != nil { - valueEthFloat64, _ := valueEth.Float64() - RelayHeaderValue.WithLabelValues(relay.String()).Set(valueEthFloat64) - } + RecordRelayLastSlot(relay.String(), uint64(slot)) + + valueEthFloat64, _ := valueEth.Float64() + RecordBidValue(relay.String(), valueEthFloat64) // Skip if value is lower than the minimum bid if bidInfo.value.CmpBig(m.relayMinBid.BigInt()) == -1 { log.Debug("ignoring bid below min-bid value") + IncrementBidBelowMinBid(relay.String()) return } @@ -261,13 +258,18 @@ func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, pa log.Debug("new best bid") result.response = *bid result.bidInfo = bidInfo + result.t = time.Now() }(relay) } wg.Wait() - // Set the winning relays before returning result.relays = relays[BlockHashHex(result.bidInfo.blockHash.String())] + + if len(result.relays) > 0 { + RecordWinningBidValue(result.bidInfo.value.Float64()) + } + return result, nil } diff --git a/server/get_payload.go b/server/get_payload.go index 60be1060..b298a65b 100644 --- a/server/get_payload.go +++ b/server/get_payload.go @@ -223,17 +223,13 @@ func (m *BoostService) innerGetPayload(log *logrus.Entry, signedBlindedBeaconBlo log.Debug("submitting signed blinded block") start := time.Now() resp, err := m.httpClientGetPayload.Do(req) - if RelayLatency != nil { - RelayLatency.WithLabelValues(endpoint, relay.String()).Observe(float64(time.Since(start).Milliseconds())) - } + RecordRelayLatency(endpoint, relay.String(), float64(time.Since(start).Microseconds())) if err != nil { log.WithError(err).Warnf("error calling getPayload%s on relay", version) return nil, err } - if RelayStatusCode != nil { - RelayStatusCode.WithLabelValues(strconv.Itoa(statusCode), endpoint, relay.String()).Inc() - } + RecordRelayStatusCode(strconv.Itoa(statusCode), endpoint, relay.String()) // Check that the response was successful if resp.StatusCode != statusCode { err = fmt.Errorf("%w: %d", errHTTPErrorResponse, resp.StatusCode) diff --git a/server/metrics.go b/server/metrics.go index dd0d9014..4e4b8f44 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -8,12 +8,23 @@ import ( const Namespace = "mev_boost" +// labels + +const ( + Endpoint = "endpoint" + Relay = "relay" + HTTPStatusCode = "http_status_code" +) + var ( metricsInitOnce sync.Once BeaconNodeStatus *prometheus.CounterVec - RelayHeaderValue *prometheus.GaugeVec + BidValues *prometheus.HistogramVec + BidsBelowMinBid *prometheus.CounterVec + WinningBidValue *prometheus.HistogramVec RelayLatency *prometheus.HistogramVec RelayStatusCode *prometheus.CounterVec + RelayLastSlot *prometheus.GaugeVec ) func RegisterMetrics(registry *prometheus.Registry) { @@ -24,16 +35,34 @@ func RegisterMetrics(registry *prometheus.Registry) { Name: "beacon_node_status_code_total", Help: "http status code returned to beacon node", }, - []string{"http_status_code", "endpoint"}, + []string{HTTPStatusCode, Endpoint}, ) - RelayHeaderValue = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + BidValues = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ Namespace: Namespace, - Name: "relay_header_value", - Help: "header value delivered by relay", + Name: "bid_values", + Help: "Value of the bids seen per relay", }, - []string{"relay"}, + []string{Relay}, + ) + + BidsBelowMinBid = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Name: "bids_below_min_bid", + Help: "Number of bids per relay which are below the min bid", + }, + []string{Relay}, + ) + + WinningBidValue = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: Namespace, + Name: "winning_bid_value", + Help: "Value of the winning bid", + }, + []string{}, ) RelayLatency = prometheus.NewHistogramVec( @@ -42,7 +71,7 @@ func RegisterMetrics(registry *prometheus.Registry) { Name: "relay_latency", Help: "http latency by relay", }, - []string{"endpoint", "relay"}, + []string{Endpoint, Relay}, ) RelayStatusCode = prometheus.NewCounterVec( @@ -51,14 +80,68 @@ func RegisterMetrics(registry *prometheus.Registry) { Name: "relay_status_code_total", Help: "http status code received by relay", }, - []string{"http_status_code", "endpoint", "relay"}, + []string{HTTPStatusCode, Endpoint, Relay}, + ) + + RelayLastSlot = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Name: "relay_last_slot", + Help: "Last slot for which relay delivered a header", + }, + []string{Relay}, ) registry.MustRegister( BeaconNodeStatus, - RelayHeaderValue, + BidValues, + BidsBelowMinBid, + WinningBidValue, RelayLatency, RelayStatusCode, + RelayLastSlot, ) }) } + +func IncrementBeaconNodeStatus(status, endpoint string) { + if BeaconNodeStatus != nil { + BeaconNodeStatus.WithLabelValues(status, endpoint).Inc() + } +} + +func RecordBidValue(relay string, value float64) { + if BidValues != nil { + BidValues.WithLabelValues(relay).Observe(value) + } +} + +func IncrementBidBelowMinBid(relay string) { + if BidsBelowMinBid != nil { + BidsBelowMinBid.WithLabelValues(relay).Inc() + } +} + +func RecordWinningBidValue(value float64) { + if WinningBidValue != nil { + WinningBidValue.WithLabelValues().Observe(value) + } +} + +func RecordRelayLatency(endpoint, relay string, latency float64) { + if RelayLatency != nil { + RelayLatency.WithLabelValues(endpoint, relay).Observe(latency) + } +} + +func RecordRelayStatusCode(httpStatus, endpoint, relay string) { + if RelayStatusCode != nil { + RelayStatusCode.WithLabelValues(httpStatus, endpoint, relay).Inc() + } +} + +func RecordRelayLastSlot(relay string, slot uint64) { + if RelayLastSlot != nil { + RelayLastSlot.WithLabelValues(relay).Set(float64(slot)) + } +} diff --git a/server/register_validator.go b/server/register_validator.go index 7fd188df..cb31fd10 100644 --- a/server/register_validator.go +++ b/server/register_validator.go @@ -49,9 +49,7 @@ func (m *BoostService) registerValidator(log *logrus.Entry, regBytes []byte, hea // Send the request start := time.Now() resp, err := m.httpClientRegVal.Do(req) - if RelayLatency != nil { - RelayLatency.WithLabelValues(params.PathRegisterValidator, relay.String()).Observe(float64(time.Since(start).Milliseconds())) - } + RecordRelayLatency(params.PathRegisterValidator, relay.String(), float64(time.Since(start).Microseconds())) if err != nil { log.WithError(err).Warn("error calling registerValidator on relay") respErrCh <- err @@ -59,9 +57,7 @@ func (m *BoostService) registerValidator(log *logrus.Entry, regBytes []byte, hea } resp.Body.Close() - if RelayStatusCode != nil { - RelayStatusCode.WithLabelValues(strconv.Itoa(resp.StatusCode), params.PathRegisterValidator, relay.String()).Inc() - } + RecordRelayStatusCode(strconv.Itoa(resp.StatusCode), params.PathRegisterValidator, relay.String()) // Check if response is successful if resp.StatusCode == http.StatusOK { log.Debug("relay accepted registrations") diff --git a/server/service.go b/server/service.go index 4e8486fd..17588e6b 100644 --- a/server/service.go +++ b/server/service.go @@ -235,14 +235,10 @@ func (m *BoostService) handleRoot(w http.ResponseWriter, _ *http.Request) { func (m *BoostService) handleStatus(w http.ResponseWriter, _ *http.Request) { w.Header().Set(HeaderKeyVersion, config.Version) if !m.relayCheck || m.CheckRelays() > 0 { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathStatus).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusOK), params.PathStatus) m.respondOK(w, nilResponse) } else { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusServiceUnavailable), params.PathStatus).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusServiceUnavailable), params.PathStatus) m.respondError(w, http.StatusServiceUnavailable, "all relays are unavailable") } } @@ -266,9 +262,7 @@ func (m *BoostService) handleRegisterValidator(w http.ResponseWriter, req *http. // Read the validator registrations regBytes, err := io.ReadAll(req.Body) if err != nil { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusInternalServerError), params.PathRegisterValidator).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusInternalServerError), params.PathRegisterValidator) m.respondError(w, http.StatusInternalServerError, err.Error()) return } @@ -278,15 +272,11 @@ func (m *BoostService) handleRegisterValidator(w http.ResponseWriter, req *http. err = m.registerValidator(log, regBytes, header) if err == nil { // One of the relays responded OK - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathRegisterValidator).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusOK), params.PathRegisterValidator) m.respondOK(w, nilResponse) } else { // None of the relays responded OK - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadGateway), params.PathRegisterValidator).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusBadGateway), params.PathRegisterValidator) m.respondError(w, http.StatusBadGateway, err.Error()) } } @@ -306,9 +296,7 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) // Parse the slot slotValue, err := strconv.ParseUint(vars["slot"], 10, 64) if err != nil { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadRequest), params.PathGetHeader).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusBadRequest), params.PathGetHeader) m.respondError(w, http.StatusBadRequest, errInvalidSlot.Error()) return } @@ -328,9 +316,7 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) // Query the relays for the header result, err := m.getHeader(log, slot, pubkey, parentHashHex, ua, rawProposerAcceptContentTypes) if err != nil { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadRequest), params.PathGetHeader).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusBadRequest), params.PathGetHeader) m.respondError(w, http.StatusBadRequest, err.Error()) return } @@ -338,9 +324,7 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) // Bail if none of the relays returned a bid if result.response.IsEmpty() { log.Info("no bid received") - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusNoContent), params.PathGetHeader).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusNoContent), params.PathGetHeader) w.WriteHeader(http.StatusNoContent) return } @@ -376,22 +360,16 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) // Respond appropriately switch proposerPreferredContentType { case MediaTypeJSON: - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathGetHeader).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusOK), params.PathGetHeader) log.Debug("responding with JSON") m.respondGetHeaderJSON(w, &result) case MediaTypeOctetStream: - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathGetHeader).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusOK), params.PathGetHeader) log.Debug("responding with SSZ") m.respondGetHeaderSSZ(w, &result) default: - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusNotAcceptable), params.PathGetHeader).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusNotAcceptable), params.PathGetHeader) message := fmt.Sprintf("unsupported media type: %s", proposerPreferredContentType) log.Error(message) m.respondError(w, http.StatusNotAcceptable, message) @@ -422,9 +400,7 @@ func (m *BoostService) handleGetPayload(w http.ResponseWriter, req *http.Request // Read the body first, so we can log it later on error signedBlindedBlockBytes, err := io.ReadAll(req.Body) if err != nil { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadRequest), params.PathGetPayload).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusBadRequest), params.PathGetPayload) log.WithError(err).Error("could not read body of request from the beacon node") m.respondError(w, http.StatusBadRequest, err.Error()) return @@ -435,9 +411,7 @@ func (m *BoostService) handleGetPayload(w http.ResponseWriter, req *http.Request // If no payload has been received from relay, log loudly about withholding! if result == nil || getPayloadResponseIsEmpty(result) { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadGateway), params.PathGetPayload).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusBadGateway), params.PathGetPayload) originRelays := types.RelayEntriesToStrings(originalBid.relays) log.WithField("relaysWithBid", strings.Join(originRelays, ", ")).Error("no payload received from relay!") m.respondError(w, http.StatusBadGateway, errNoSuccessfulRelayResponse.Error()) @@ -460,21 +434,15 @@ func (m *BoostService) handleGetPayload(w http.ResponseWriter, req *http.Request // Respond appropriately switch proposerPreferredContentType { case MediaTypeJSON: - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathGetPayload).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusOK), params.PathGetPayload) log.Debug("responding with JSON") m.respondGetPayloadJSON(w, result) case MediaTypeOctetStream: - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusOK), params.PathGetPayload).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusOK), params.PathGetPayload) log.Debug("responding with SSZ") m.respondGetPayloadSSZ(w, result) default: - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusNotAcceptable), params.PathGetPayload).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusNotAcceptable), params.PathGetPayload) message := fmt.Sprintf("unsupported media type: %s", proposerPreferredContentType) log.Error(message) m.respondError(w, http.StatusNotAcceptable, message) @@ -502,9 +470,7 @@ func (m *BoostService) handleGetPayloadV2(w http.ResponseWriter, req *http.Reque // Read the body first, so we can log it later on error signedBlindedBlockBytes, err := io.ReadAll(req.Body) if err != nil { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadRequest), params.PathGetPayloadV2).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusBadRequest), params.PathGetPayloadV2) log.WithError(err).Error("could not read body of request from the beacon node") m.respondError(w, http.StatusBadRequest, err.Error()) return @@ -515,18 +481,14 @@ func (m *BoostService) handleGetPayloadV2(w http.ResponseWriter, req *http.Reque // If no relay accepted the submission, log about the failure if !success { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusBadGateway), params.PathGetPayloadV2).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusBadGateway), params.PathGetPayloadV2) originRelays := types.RelayEntriesToStrings(originalBid.relays) log.WithField("relaysWithBid", strings.Join(originRelays, ", ")).Error("no relay accepted the signed blinded beacon block submission!") m.respondError(w, http.StatusBadGateway, errNoSuccessfulRelayResponse.Error()) return } - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(strconv.Itoa(http.StatusAccepted), params.PathGetPayloadV2).Inc() - } + IncrementBeaconNodeStatus(strconv.Itoa(http.StatusAccepted), params.PathGetPayloadV2) log.Info("successfully submitted signed blinded beacon block to relay") w.WriteHeader(http.StatusAccepted) } @@ -547,16 +509,12 @@ func (m *BoostService) CheckRelays() int { start := time.Now() code, err := SendHTTPRequest(context.Background(), m.httpClientGetHeader, http.MethodGet, url, "", nil, nil, nil) - if RelayLatency != nil { - RelayLatency.WithLabelValues(params.PathStatus, relay.String()).Observe(float64(time.Since(start).Milliseconds())) - } + RecordRelayLatency(params.PathStatus, relay.String(), float64(time.Since(start).Microseconds())) if err != nil { log.WithError(err).Error("relay status error - request failed") return } - if RelayStatusCode != nil { - RelayStatusCode.WithLabelValues(strconv.Itoa(code), params.PathStatus, relay.String()).Inc() - } + RecordRelayStatusCode(strconv.Itoa(code), params.PathStatus, relay.String()) if code == http.StatusOK { log.Debug("relay status OK") } else { From e7beab88709e9bf8ecdbe772120383ed8c8dd42d Mon Sep 17 00:00:00 2001 From: Bharath Vedartham Date: Thu, 25 Sep 2025 22:28:14 +0530 Subject: [PATCH 10/14] fix lint --- cli/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/flags.go b/cli/flags.go index dd182be8..f147c12b 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -37,7 +37,7 @@ var flags = []cli.Flag{ timeoutRegValFlag, maxRetriesFlag, - //metrics + // metrics metricsFlag, metricsAddrFlag, } @@ -184,7 +184,7 @@ var ( Category: RelayCategory, } - //metrics + // metrics metricsFlag = &cli.BoolFlag{ Name: "metrics", Sources: cli.EnvVars("METRICS_ENABLED"), From 146779a1d47cb080ea9f9e0a8579f5b29d40a771 Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Fri, 26 Sep 2025 12:13:21 +0200 Subject: [PATCH 11/14] upd readme --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 177f3915..0b5832cf 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,10 @@ Usage of mev-boost: use Sepolia -version only print version + -metrics + enables a metrics server (default: false) + -metrics-addr string + listening address for the metrics server (default: "localhost:18551") ``` ### `-relays` vs `-relay` @@ -309,6 +313,11 @@ Example for setting a minimum bid value of 0.06 ETH: -relay $YOUR_RELAY_CHOICE_C ``` + +### Enabling metrics + +Optionally, the `-metrics` flag can be provided to expose a prometheus metrics server. The metrics server address/port can be changed with the `-metrics-addr` (e.g., `-metrics-addr localhost:9009`) flag. + --- # API From 1769b72299ff961686000a728d7ae942b43bfae4 Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Fri, 26 Sep 2025 12:14:26 +0200 Subject: [PATCH 12/14] rem line break --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 0b5832cf..d831cdd6 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,6 @@ Example for setting a minimum bid value of 0.06 ETH: -relay $YOUR_RELAY_CHOICE_C ``` - ### Enabling metrics Optionally, the `-metrics` flag can be provided to expose a prometheus metrics server. The metrics server address/port can be changed with the `-metrics-addr` (e.g., `-metrics-addr localhost:9009`) flag. From 1f8ac47dd7c1eb56f503fd81f7f5cd30b8784e40 Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Fri, 26 Sep 2025 15:57:47 +0200 Subject: [PATCH 13/14] fix spelling --- server/metrics.go | 10 ++++++++++ server/service.go | 1 + 2 files changed, 11 insertions(+) diff --git a/server/metrics.go b/server/metrics.go index 4e4b8f44..2bdbd1a2 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -25,6 +25,7 @@ var ( RelayLatency *prometheus.HistogramVec RelayStatusCode *prometheus.CounterVec RelayLastSlot *prometheus.GaugeVec + MsIntoSlot *prometheus.HistogramVec ) func RegisterMetrics(registry *prometheus.Registry) { @@ -92,6 +93,14 @@ func RegisterMetrics(registry *prometheus.Registry) { []string{Relay}, ) + MsIntoSlot = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: Namespace, + Name: "millisec_into_slot", + Help: "Milliseconds into the slot when endpoint was called", + }, + []string{Endpoint}, + ) registry.MustRegister( BeaconNodeStatus, BidValues, @@ -100,6 +109,7 @@ func RegisterMetrics(registry *prometheus.Registry) { RelayLatency, RelayStatusCode, RelayLastSlot, + MsIntoSlot, ) }) } diff --git a/server/service.go b/server/service.go index 17588e6b..3023efd4 100644 --- a/server/service.go +++ b/server/service.go @@ -115,6 +115,7 @@ func NewBoostService(opts BoostServiceOpts) (*BoostService, error) { genesisTime: opts.GenesisTime, bids: make(map[string]bidResp), slotUID: &slotUID{}, + metricsAddr: opts.MetricsAddr, builderSigningDomain: builderSigningDomain, httpClientGetHeader: http.Client{ From 75ef1955c8c61027ec936cf956ccabbe6b3bf56e Mon Sep 17 00:00:00 2001 From: faheelsattar Date: Sun, 28 Sep 2025 20:19:09 +0200 Subject: [PATCH 14/14] port to victoria metrics --- go.mod | 12 +--- go.sum | 22 ++----- server/metrics.go | 155 +++++++++------------------------------------- server/service.go | 22 ++----- 4 files changed, 42 insertions(+), 169 deletions(-) diff --git a/go.mod b/go.mod index d7b4a28f..cdf8b56f 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/flashbots/mev-boost go 1.24.0 require ( + github.com/VictoriaMetrics/metrics v1.40.1 github.com/ethereum/go-ethereum v1.15.9 github.com/flashbots/go-boost-utils v1.9.0 github.com/flashbots/go-utils v0.10.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/holiman/uint256 v1.3.2 - github.com/prometheus/client_golang v1.20.5 github.com/prysmaticlabs/go-bitfield v0.0.0-20240618144021-706c95b2dd15 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.10.0 @@ -18,9 +18,7 @@ require ( ) require ( - github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/bavard v0.1.30 // indirect github.com/consensys/gnark-crypto v0.17.0 // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect @@ -30,17 +28,13 @@ require ( github.com/ethereum/go-verkle v0.2.2 // indirect github.com/goccy/go-yaml v1.17.1 // indirect github.com/gofrs/flock v0.12.1 // indirect - github.com/klauspost/compress v1.17.9 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/supranational/blst v0.3.14 // indirect + github.com/valyala/fastrand v1.1.0 // indirect + github.com/valyala/histogram v1.2.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/sync v0.13.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index a533275a..431e31a1 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,11 @@ github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= +github.com/VictoriaMetrics/metrics v1.40.1 h1:FrF5uJRpIVj9fayWcn8xgiI+FYsKGMslzPuOXjdeyR4= +github.com/VictoriaMetrics/metrics v1.40.1/go.mod h1:XE4uudAAIRaJE614Tl5HMrtoEU6+GDZO4QTnNSsZRuA= github.com/attestantio/go-builder-client v0.7.2 h1:bOrtysEIZd9bEM+mAeT6OtAo6LSAft/qylBLwFoFwZ0= github.com/attestantio/go-builder-client v0.7.2/go.mod h1:+NADxbaknI5yxl+0mCkMa/VciVsesxRMGNP/poDfV08= github.com/attestantio/go-eth2-client v0.27.1 h1:g7bm+gG/p+gfzYdEuxuAepVWYb8EO+2KojV5/Lo2BxM= github.com/attestantio/go-eth2-client v0.27.1/go.mod h1:fvULSL9WtNskkOB4i+Yyr6BKpNHXvmpGZj9969fCrfY= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -61,8 +61,6 @@ github.com/huandu/go-clone v1.7.2 h1:3+Aq0Ed8XK+zKkLjE2dfHg0XrpIfcohBE1K+c8Usxoo github.com/huandu/go-clone v1.7.2/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= github.com/huandu/go-clone/generic v1.6.0 h1:Wgmt/fUZ28r16F2Y3APotFD59sHk1p78K0XLdbUYN5U= github.com/huandu/go-clone/generic v1.6.0/go.mod h1:xgd9ZebcMsBWWcBx5mVMCoqMX24gLWr5lQicr+nVXNs= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -83,22 +81,12 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prysmaticlabs/go-bitfield v0.0.0-20240618144021-706c95b2dd15 h1:lC8kiphgdOBTcbTvo8MwkvpKjO0SlAgjv4xIK5FGJ94= github.com/prysmaticlabs/go-bitfield v0.0.0-20240618144021-706c95b2dd15/go.mod h1:8svFBIKKu31YriBG/pNizo9N0Jr9i5PQ+dFkxWg3x5k= github.com/prysmaticlabs/gohashtree v0.0.4-beta h1:H/EbCuXPeTV3lpKeXGPpEV9gsUpkqOOVnWapUyeWro4= @@ -128,6 +116,10 @@ github.com/trailofbits/go-fuzz-utils v0.0.0-20240830175354-474de707d2aa h1:jXdW8 github.com/trailofbits/go-fuzz-utils v0.0.0-20240830175354-474de707d2aa/go.mod h1:/7KgvY5ghyUsjocUh9dMkLCwKtNxqe0kWl5SIdpLtO8= github.com/urfave/cli/v3 v3.2.0 h1:m8WIXY0U9LCuUl5r+0fqLWDhNYWt6qvlW+GcF4EoXf8= github.com/urfave/cli/v3 v3.2.0/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= +github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= +github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= +github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -145,8 +137,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/server/metrics.go b/server/metrics.go index 2bdbd1a2..fb6df1cf 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -1,157 +1,58 @@ package server import ( - "sync" + "fmt" - "github.com/prometheus/client_golang/prometheus" + "github.com/VictoriaMetrics/metrics" ) -const Namespace = "mev_boost" - -// labels +var winningBidValue = metrics.NewHistogram("mev_boost_winning_bid_value") const ( - Endpoint = "endpoint" - Relay = "relay" - HTTPStatusCode = "http_status_code" -) - -var ( - metricsInitOnce sync.Once - BeaconNodeStatus *prometheus.CounterVec - BidValues *prometheus.HistogramVec - BidsBelowMinBid *prometheus.CounterVec - WinningBidValue *prometheus.HistogramVec - RelayLatency *prometheus.HistogramVec - RelayStatusCode *prometheus.CounterVec - RelayLastSlot *prometheus.GaugeVec - MsIntoSlot *prometheus.HistogramVec + beaconNodeStatusLabel = `mev_boost_beacon_node_status_code_total{http_status_code="%s",endpoint="%s"}` + bidValuesLabel = `mev_boost_bid_values{relay="%s"}` + bidsBelowMinBidLabel = `mev_boost_bids_below_min_bid_total{relay="%s"}` + relayLatencyLabel = `mev_boost_relay_latency{endpoint="%s",relay="%s"}` + relayStatusCodeLabel = `mev_boost_relay_status_code_total{http_status_code="%s",endpoint="%s",relay="%s"}` + relayLastSlotLabel = `mev_boost_relay_last_slot{relay="%s"}` + msIntoSlotLabel = `mev_boost_millisec_into_slot{endpoint="%s"}` ) -func RegisterMetrics(registry *prometheus.Registry) { - metricsInitOnce.Do(func() { - BeaconNodeStatus = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: Namespace, - Name: "beacon_node_status_code_total", - Help: "http status code returned to beacon node", - }, - []string{HTTPStatusCode, Endpoint}, - ) - - BidValues = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: Namespace, - Name: "bid_values", - Help: "Value of the bids seen per relay", - }, - []string{Relay}, - ) - - BidsBelowMinBid = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: Namespace, - Name: "bids_below_min_bid", - Help: "Number of bids per relay which are below the min bid", - }, - []string{Relay}, - ) - - WinningBidValue = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: Namespace, - Name: "winning_bid_value", - Help: "Value of the winning bid", - }, - []string{}, - ) - - RelayLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: Namespace, - Name: "relay_latency", - Help: "http latency by relay", - }, - []string{Endpoint, Relay}, - ) - - RelayStatusCode = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: Namespace, - Name: "relay_status_code_total", - Help: "http status code received by relay", - }, - []string{HTTPStatusCode, Endpoint, Relay}, - ) - - RelayLastSlot = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: Namespace, - Name: "relay_last_slot", - Help: "Last slot for which relay delivered a header", - }, - []string{Relay}, - ) - - MsIntoSlot = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: Namespace, - Name: "millisec_into_slot", - Help: "Milliseconds into the slot when endpoint was called", - }, - []string{Endpoint}, - ) - registry.MustRegister( - BeaconNodeStatus, - BidValues, - BidsBelowMinBid, - WinningBidValue, - RelayLatency, - RelayStatusCode, - RelayLastSlot, - MsIntoSlot, - ) - }) -} - func IncrementBeaconNodeStatus(status, endpoint string) { - if BeaconNodeStatus != nil { - BeaconNodeStatus.WithLabelValues(status, endpoint).Inc() - } + l := fmt.Sprintf(beaconNodeStatusLabel, status, endpoint) + metrics.GetOrCreateCounter(l).Inc() } func RecordBidValue(relay string, value float64) { - if BidValues != nil { - BidValues.WithLabelValues(relay).Observe(value) - } + l := fmt.Sprintf(bidValuesLabel, relay) + metrics.GetOrCreateHistogram(l).Update(value) } func IncrementBidBelowMinBid(relay string) { - if BidsBelowMinBid != nil { - BidsBelowMinBid.WithLabelValues(relay).Inc() - } + l := fmt.Sprintf(bidsBelowMinBidLabel, relay) + metrics.GetOrCreateCounter(l).Inc() } func RecordWinningBidValue(value float64) { - if WinningBidValue != nil { - WinningBidValue.WithLabelValues().Observe(value) - } + winningBidValue.Update(value) } func RecordRelayLatency(endpoint, relay string, latency float64) { - if RelayLatency != nil { - RelayLatency.WithLabelValues(endpoint, relay).Observe(latency) - } + l := fmt.Sprintf(relayLatencyLabel, endpoint, relay) + metrics.GetOrCreateHistogram(l).Update(latency) } func RecordRelayStatusCode(httpStatus, endpoint, relay string) { - if RelayStatusCode != nil { - RelayStatusCode.WithLabelValues(httpStatus, endpoint, relay).Inc() - } + l := fmt.Sprintf(relayStatusCodeLabel, httpStatus, endpoint, relay) + metrics.GetOrCreateCounter(l).Inc() } func RecordRelayLastSlot(relay string, slot uint64) { - if RelayLastSlot != nil { - RelayLastSlot.WithLabelValues(relay).Set(float64(slot)) - } + l := fmt.Sprintf(relayLastSlotLabel, relay) + metrics.GetOrCreateGauge(l, nil).Set(float64(slot)) +} + +func RecordMsIntoSlot(endpoint string, ms float64) { + l := fmt.Sprintf(msIntoSlotLabel, endpoint) + metrics.GetOrCreateHistogram(l).Update(ms) } diff --git a/server/service.go b/server/service.go index 3023efd4..172535bd 100644 --- a/server/service.go +++ b/server/service.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "time" + "github.com/VictoriaMetrics/metrics" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/flashbots/go-boost-utils/ssz" "github.com/flashbots/go-utils/httplogger" @@ -21,9 +22,6 @@ import ( "github.com/flashbots/mev-boost/server/types" "github.com/google/uuid" "github.com/gorilla/mux" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" goacceptheaders "github.com/timewasted/go-accept-headers" ) @@ -195,22 +193,12 @@ func (m *BoostService) StartHTTPServer() error { return err } -// StartMetricsServer starts the HTTP server for exporting open-metrics +// StartMetricsServer starts the HTTP server for exporting metrics func (m *BoostService) StartMetricsServer() error { - prometheusRegistry := prometheus.NewRegistry() - if err := prometheusRegistry.Register(collectors.NewGoCollector()); err != nil { - m.log.WithError(err).Error("failed to register metrics for GoCollector") - } - if err := prometheusRegistry.Register(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})); err != nil { - m.log.WithError(err).Error("failed to register ProcessCollector") - } - RegisterMetrics(prometheusRegistry) - serveMux := http.NewServeMux() - serveMux.Handle("/metrics", promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{ - ErrorLog: m.log, - EnableOpenMetrics: true, - })) + serveMux.HandleFunc("/metrics", func(w http.ResponseWriter, _ *http.Request) { + metrics.WritePrometheus(w, true) + }) return http.ListenAndServe(m.metricsAddr, serveMux) }