From 46cfbfe0b5e9b9ec9981ea96a534a8215d8c424b Mon Sep 17 00:00:00 2001 From: Cedric Cordenier Date: Tue, 21 Jul 2026 16:35:23 +0100 Subject: [PATCH 1/2] Add metric to measure don.SendToNode latency to Gateway --- core/cmd/shell.go | 2 ++ .../capabilities/v2/http_trigger_handler.go | 2 ++ .../capabilities/v2/metrics/metrics.go | 32 +++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/core/cmd/shell.go b/core/cmd/shell.go index 4494588bdbd..376655b35c2 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -56,6 +56,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/ccv/ccvcommon" "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" "github.com/smartcontractkit/chainlink/v2/core/services/cre" + gatewayv2metrics "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/capabilities/v2/metrics" gatewaynetwork "github.com/smartcontractkit/chainlink/v2/core/services/gateway/network" "github.com/smartcontractkit/chainlink/v2/core/services/keystore" "github.com/smartcontractkit/chainlink/v2/core/services/llo" @@ -85,6 +86,7 @@ func metricViews() []sdkmetric.View { ocr3beholderwrapper.MetricViews(), ocr3_1beholderwrapper.MetricViews(), gatewaynetwork.HTTPClientMetricViews(), + gatewayv2metrics.MetricViews(), ) } diff --git a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go index 477085c7244..07234d47317 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go +++ b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go @@ -590,7 +590,9 @@ func (h *httpTriggerHandler) sendWithRetries(ctx context.Context, legacyExecutio continue } h.metrics.IncrementTriggerCapabilityRequestCount(ctx, member.Address, gateway_common.MethodWorkflowExecute, h.lggr) + sendStart := time.Now() err := h.don.SendToNode(ctxWithTimeout, member.Address, req) + h.metrics.RecordTriggerCapabilityRequestLatency(ctx, time.Since(sendStart).Milliseconds(), member.Address, gateway_common.MethodWorkflowExecute, h.lggr) if err != nil { allNodesSucceeded = false h.metrics.IncrementTriggerCapabilityRequestFailures(ctx, member.Address, gateway_common.MethodWorkflowExecute, h.lggr) diff --git a/core/services/gateway/handlers/capabilities/v2/metrics/metrics.go b/core/services/gateway/handlers/capabilities/v2/metrics/metrics.go index 1a608365312..93a63c1fd55 100644 --- a/core/services/gateway/handlers/capabilities/v2/metrics/metrics.go +++ b/core/services/gateway/handlers/capabilities/v2/metrics/metrics.go @@ -6,6 +6,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -59,6 +60,7 @@ type TriggerMetrics struct { requestHandlerLatency metric.Int64Histogram capabilityRequestCount metric.Int64Counter capabilityRequestFailures metric.Int64Counter + capabilityRequestLatency metric.Int64Histogram metadataProcessingFailures metric.Int64Counter metadataRequestCount metric.Int64Counter metadataObservationsCleanUpCount metric.Int64Counter @@ -332,6 +334,14 @@ func newTriggerMetrics(meter metric.Meter) (*TriggerMetrics, error) { return nil, fmt.Errorf("failed to create HTTP trigger gateway capability request failures metric: %w", err) } + m.capabilityRequestLatency, err = meter.Int64Histogram( + "http_trigger_gateway_capability_request_latency_ms", + metric.WithDescription("Latency in milliseconds of sending HTTP trigger requests from gateway node to capability nodes (don.SendToNode), broken down by target node"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP trigger gateway capability request latency metric: %w", err) + } + m.metadataProcessingFailures, err = meter.Int64Counter( "http_trigger_gateway_metadata_processing_failures", metric.WithDescription("Number of HTTP trigger gateway metadata processing failures"), @@ -534,6 +544,14 @@ func (m *Metrics) IncrementTriggerCapabilityRequestFailures(ctx context.Context, )) } +func (m *Metrics) RecordTriggerCapabilityRequestLatency(ctx context.Context, latencyMs int64, nodeAddress string, methodName string, lggr logger.Logger) { + m.trigger.capabilityRequestLatency.Record(ctx, latencyMs, metric.WithAttributes( + attribute.String(AttrNodeAddress, nodeAddress), + attribute.String(AttrNodeName, m.nodeAddressToNodeName[nodeAddress]), + attribute.String(AttrMethodName, methodName), + )) +} + func (m *Metrics) IncrementMetadataProcessingFailures(ctx context.Context, nodeAddress string, methodName string, lggr logger.Logger) { m.trigger.metadataProcessingFailures.Add(ctx, 1, metric.WithAttributes( attribute.String(AttrNodeAddress, nodeAddress), @@ -573,3 +591,17 @@ func (m *Metrics) RecordMetadataSyncStartupLatency(ctx context.Context, latencyM func (m *Metrics) RecordLoadedMetadataSize(ctx context.Context, size int64, lggr logger.Logger) { m.trigger.loadedMetadataSize.Record(ctx, size) } + +// MetricViews returns histogram bucket definitions for this package's metrics. +// Due to the OTEL specification, all histogram buckets must be defined when the beholder client is created. +func MetricViews() []sdkmetric.View { + return []sdkmetric.View{ + sdkmetric.NewView( + sdkmetric.Instrument{Name: "http_trigger_gateway_capability_request_latency_ms"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{ + // 10ms up to 90s (max trigger request duration is on this order), with finer granularity at the lower end + Boundaries: []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 25000, 50000, 90000}, + }}, + ), + } +} From e8d7a0c430476ab8df21bcb700c0daf825833acc Mon Sep 17 00:00:00 2001 From: Cedric Cordenier Date: Wed, 22 Jul 2026 12:23:19 +0100 Subject: [PATCH 2/2] Log out response --- .../handlers/capabilities/v2/http_trigger_handler.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go index 07234d47317..4632138ceb2 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go +++ b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go @@ -444,8 +444,13 @@ func (h *httpTriggerHandler) HandleNodeTriggerResponse(ctx context.Context, resp if err != nil { return err } + + d, err := resp.Digest() + if err != nil { + h.lggr.Debugw("could not calculate digest for resp", "requestID", resp.ID) + } if aggResp == nil { - h.lggr.Debugw("Not enough responses to aggregate", "requestID", resp.ID, "nodeAddress", nodeAddr) + h.lggr.Debugw("Not enough responses to aggregate", "requestID", resp.ID, "nodeAddress", nodeAddr, "resp", resp, "digest", d) return nil } rawResp, err := json.Marshal(aggResp)