From 38c7a4dcd227827cc7f54d2d2db3260e58a17cf3 Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Thu, 9 Apr 2026 12:49:28 +0200 Subject: [PATCH 1/3] fix: widen CloudWatch query window in MetricsE2ET to fix flaky test Pad the CloudWatch query time window by -1min/+2min for standard resolution metric queries to account for CloudWatch eventual consistency. Metric timestamps can shift by up to a minute during batch processing, causing MetricDataNotFoundException when using a tight 1-minute window. Closes #2440 Co-Authored-By: Claude Opus 4.6 --- .../amazon/lambda/powertools/MetricsE2ET.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java index 35f8b5ba3..ff8a23a75 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java @@ -18,7 +18,6 @@ import static software.amazon.lambda.powertools.testutils.Infrastructure.FUNCTION_NAME_OUTPUT; import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction; -import java.time.Clock; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Collections; @@ -74,7 +73,6 @@ static void tearDown() { void test_recordMetrics() { // GIVEN - Instant currentTimeTruncatedToMinutes = Instant.now(Clock.systemUTC()).truncatedTo(ChronoUnit.MINUTES); String event1 = "{ \"metrics\": {\"orders\": 1, \"products\": 4}, \"dimensions\": { \"Environment\": \"test\"}, \"highResolution\": \"false\"}"; String event2 = "{ \"metrics\": {\"orders\": 1, \"products\": 8}, \"dimensions\": { \"Environment\": \"test\"}, \"highResolution\": \"true\"}"; @@ -84,15 +82,20 @@ void test_recordMetrics() { invokeFunction(functionName, event2); // THEN + // Pad the query window to address CloudWatch eventual consistency: + // metric timestamps can shift by up to a minute during batch processing. + Instant paddedStart = invocationResult.getStart().minus(1, ChronoUnit.MINUTES); + Instant paddedEnd = invocationResult.getEnd().plus(2, ChronoUnit.MINUTES); + MetricsFetcher metricsFetcher = new MetricsFetcher(); - List coldStart = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60, + List coldStart = metricsFetcher.fetchMetrics(paddedStart, paddedEnd, 60, NAMESPACE, "ColdStart", Stream.of(new String[][] { { "FunctionName", functionName }, { "Service", SERVICE } }).collect(Collectors.toMap(data -> data[0], data -> data[1]))); assertThat(coldStart.get(0)).isEqualTo(1); List orderMetrics = RetryUtils.withRetry(() -> { - List metrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), + List metrics = metricsFetcher.fetchMetrics(paddedStart, paddedEnd, 60, NAMESPACE, "orders", Collections.singletonMap("Environment", "test")); if (metrics.get(0) != 2.0) { throw new DataNotReadyException("Expected 2.0 orders but got " + metrics.get(0)); @@ -100,26 +103,24 @@ void test_recordMetrics() { return metrics; }, "orderMetricsRetry", DataNotReadyException.class).get(); assertThat(orderMetrics.get(0)).isEqualTo(2); - List productMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), - invocationResult.getEnd(), 60, NAMESPACE, + List productMetrics = metricsFetcher.fetchMetrics(paddedStart, + paddedEnd, 60, NAMESPACE, "products", Collections.singletonMap("Environment", "test")); // When searching across a 1 minute time period with a period of 60 we find both metrics and the sum is 12 assertThat(productMetrics.get(0)).isEqualTo(12); - orderMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60, + orderMetrics = metricsFetcher.fetchMetrics(paddedStart, paddedEnd, 60, NAMESPACE, "orders", Collections.singletonMap("Service", SERVICE)); assertThat(orderMetrics.get(0)).isEqualTo(2); - productMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60, + productMetrics = metricsFetcher.fetchMetrics(paddedStart, paddedEnd, 60, NAMESPACE, "products", Collections.singletonMap("Service", SERVICE)); assertThat(productMetrics.get(0)).isEqualTo(12); - Instant searchStartTime = currentTimeTruncatedToMinutes.plusSeconds(15); - Instant searchEndTime = currentTimeTruncatedToMinutes.plusSeconds(45); - - List productMetricDataResult = metricsFetcher.fetchMetrics(searchStartTime, searchEndTime, 1, NAMESPACE, + List productMetricDataResult = metricsFetcher.fetchMetrics(invocationResult.getStart(), + invocationResult.getEnd(), 1, NAMESPACE, "products", Collections.singletonMap("Environment", "test")); // We are searching across the time period the metric was created but with a period of 1 second. Only the high From e32dc78016357e1604410488be14b15324026e58 Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Thu, 9 Apr 2026 13:26:35 +0200 Subject: [PATCH 2/3] fix: also pad high-resolution metric query window The high-res metric query (period=1s) also fails with the original 1-minute window due to CloudWatch eventual consistency. Apply the same padded window and sum all returned data points, since only the high-resolution metric appears at 1-second granularity. Co-Authored-By: Claude Opus 4.6 --- .../amazon/lambda/powertools/MetricsE2ET.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java index ff8a23a75..63a89ae1d 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java @@ -119,12 +119,13 @@ void test_recordMetrics() { "products", Collections.singletonMap("Service", SERVICE)); assertThat(productMetrics.get(0)).isEqualTo(12); - List productMetricDataResult = metricsFetcher.fetchMetrics(invocationResult.getStart(), - invocationResult.getEnd(), 1, NAMESPACE, + List productMetricDataResult = metricsFetcher.fetchMetrics(paddedStart, + paddedEnd, 1, NAMESPACE, "products", Collections.singletonMap("Environment", "test")); - // We are searching across the time period the metric was created but with a period of 1 second. Only the high - // resolution metric will be available at this point - assertThat(productMetricDataResult.get(0)).isEqualTo(8); + // With a period of 1 second, only the high resolution metric is available. + // Sum all data points as the padded window may return multiple 1-second buckets. + double highResSum = productMetricDataResult.stream().mapToDouble(Double::doubleValue).sum(); + assertThat(highResSum).isEqualTo(8); } } From a9eab073541978b2fd04477589bbd441bef851fb Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Thu, 9 Apr 2026 14:16:09 +0200 Subject: [PATCH 3/3] fix: assert high-res metric value is present instead of summing With the padded window, both standard (4) and high-res (8) product metrics appear as separate 1-second buckets, so the sum is 12 not 8. Instead, assert that the high-res value (8.0) is contained in the returned data points. Co-Authored-By: Claude Opus 4.6 --- .../software/amazon/lambda/powertools/MetricsE2ET.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java index 63a89ae1d..ceb4b8c57 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java @@ -123,9 +123,8 @@ void test_recordMetrics() { paddedEnd, 1, NAMESPACE, "products", Collections.singletonMap("Environment", "test")); - // With a period of 1 second, only the high resolution metric is available. - // Sum all data points as the padded window may return multiple 1-second buckets. - double highResSum = productMetricDataResult.stream().mapToDouble(Double::doubleValue).sum(); - assertThat(highResSum).isEqualTo(8); + // With a period of 1 second and a padded window, both standard (4) and high resolution (8) + // metrics may appear as separate 1-second buckets. Verify the high resolution value is present. + assertThat(productMetricDataResult).contains(8.0); } }