Add metric to measure don.SendToNode latency to Gateway

core/cmd/shell.go

@@ -56,6 +56,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/services/ccv/ccvcommon"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/services/cre"
+	gatewayv2metrics "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/capabilities/v2/metrics"
 	gatewaynetwork "github.com/smartcontractkit/chainlink/v2/core/services/gateway/network"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore"
 	"github.com/smartcontractkit/chainlink/v2/core/services/llo"
@@ -85,6 +86,7 @@ func metricViews() []sdkmetric.View {
 		ocr3beholderwrapper.MetricViews(),
 		ocr3_1beholderwrapper.MetricViews(),
 		gatewaynetwork.HTTPClientMetricViews(),
+		gatewayv2metrics.MetricViews(),
 	)
 }
 

core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go

@@ -444,8 +444,13 @@ func (h *httpTriggerHandler) HandleNodeTriggerResponse(ctx context.Context, resp
 	if err != nil {
 		return err
 	}
+
+	d, err := resp.Digest()
+	if err != nil {
+		h.lggr.Debugw("could not calculate digest for resp", "requestID", resp.ID)
+	}
 	if aggResp == nil {
-		h.lggr.Debugw("Not enough responses to aggregate", "requestID", resp.ID, "nodeAddress", nodeAddr)
+		h.lggr.Debugw("Not enough responses to aggregate", "requestID", resp.ID, "nodeAddress", nodeAddr, "resp", resp, "digest", d)
 		return nil
 	}
 	rawResp, err := json.Marshal(aggResp)
@@ -590,7 +595,9 @@ func (h *httpTriggerHandler) sendWithRetries(ctx context.Context, legacyExecutio
 				continue
 			}
 			h.metrics.IncrementTriggerCapabilityRequestCount(ctx, member.Address, gateway_common.MethodWorkflowExecute, h.lggr)
+			sendStart := time.Now()
 			err := h.don.SendToNode(ctxWithTimeout, member.Address, req)
+			h.metrics.RecordTriggerCapabilityRequestLatency(ctx, time.Since(sendStart).Milliseconds(), member.Address, gateway_common.MethodWorkflowExecute, h.lggr)
 			if err != nil {
 				allNodesSucceeded = false
 				h.metrics.IncrementTriggerCapabilityRequestFailures(ctx, member.Address, gateway_common.MethodWorkflowExecute, h.lggr)

core/services/gateway/handlers/capabilities/v2/metrics/metrics.go

@@ -6,6 +6,7 @@ import (
 
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/beholder"
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
@@ -59,6 +60,7 @@ type TriggerMetrics struct {
 	requestHandlerLatency            metric.Int64Histogram
 	capabilityRequestCount           metric.Int64Counter
 	capabilityRequestFailures        metric.Int64Counter
+	capabilityRequestLatency         metric.Int64Histogram
 	metadataProcessingFailures       metric.Int64Counter
 	metadataRequestCount             metric.Int64Counter
 	metadataObservationsCleanUpCount metric.Int64Counter
@@ -332,6 +334,14 @@ func newTriggerMetrics(meter metric.Meter) (*TriggerMetrics, error) {
 		return nil, fmt.Errorf("failed to create HTTP trigger gateway capability request failures metric: %w", err)
 	}
 
+	m.capabilityRequestLatency, err = meter.Int64Histogram(
+		"http_trigger_gateway_capability_request_latency_ms",
+		metric.WithDescription("Latency in milliseconds of sending HTTP trigger requests from gateway node to capability nodes (don.SendToNode), broken down by target node"),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create HTTP trigger gateway capability request latency metric: %w", err)
+	}
+
 	m.metadataProcessingFailures, err = meter.Int64Counter(
 		"http_trigger_gateway_metadata_processing_failures",
 		metric.WithDescription("Number of HTTP trigger gateway metadata processing failures"),
@@ -534,6 +544,14 @@ func (m *Metrics) IncrementTriggerCapabilityRequestFailures(ctx context.Context,
 	))
 }
 
+func (m *Metrics) RecordTriggerCapabilityRequestLatency(ctx context.Context, latencyMs int64, nodeAddress string, methodName string, lggr logger.Logger) {
+	m.trigger.capabilityRequestLatency.Record(ctx, latencyMs, metric.WithAttributes(
+		attribute.String(AttrNodeAddress, nodeAddress),
+		attribute.String(AttrNodeName, m.nodeAddressToNodeName[nodeAddress]),
+		attribute.String(AttrMethodName, methodName),
+	))
+}
+
 func (m *Metrics) IncrementMetadataProcessingFailures(ctx context.Context, nodeAddress string, methodName string, lggr logger.Logger) {
 	m.trigger.metadataProcessingFailures.Add(ctx, 1, metric.WithAttributes(
 		attribute.String(AttrNodeAddress, nodeAddress),
@@ -573,3 +591,17 @@ func (m *Metrics) RecordMetadataSyncStartupLatency(ctx context.Context, latencyM
 func (m *Metrics) RecordLoadedMetadataSize(ctx context.Context, size int64, lggr logger.Logger) {
 	m.trigger.loadedMetadataSize.Record(ctx, size)
 }
+
+// MetricViews returns histogram bucket definitions for this package's metrics.
+// Due to the OTEL specification, all histogram buckets must be defined when the beholder client is created.
+func MetricViews() []sdkmetric.View {
+	return []sdkmetric.View{
+		sdkmetric.NewView(
+			sdkmetric.Instrument{Name: "http_trigger_gateway_capability_request_latency_ms"},
+			sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{
+				// 10ms up to 90s (max trigger request duration is on this order), with finer granularity at the lower end
+				Boundaries: []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 25000, 50000, 90000},
+			}},
+		),
+	}
+}