Improving smoke test capabilities

.gitlab-ci.yml

@@ -1115,7 +1115,7 @@ test_debugger_arm64:
     - !reference [.test_job_arm64, script]
 
 test_smoke:
-  extends: .test_job
+  extends: .test_job_with_test_agent
   # needs:parallel:matrix limits this job to a specific build_tests combination.
   # Keep matrix vars exact and in build_tests declaration order:
   # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
@@ -1135,7 +1135,7 @@ test_smoke:
     matrix: *test_matrix_8
 
 test_smoke_arm64:
-  extends: .test_job_arm64
+  extends: .test_job_arm64_with_test_agent
   variables:
     <<: *tier_l_variables
     GRADLE_TARGET: "stageMainDist :smokeTest"

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -26,6 +26,8 @@
 public class SharedCommunicationObjects {
   private static final Logger log = LoggerFactory.getLogger(SharedCommunicationObjects.class);
 
+  private static final String X_DATADOG_TEST_SESSION_TOKEN = "X-Datadog-Test-Session-Token";
+
   private final List<Runnable> pausedComponents = new ArrayList<>();
   private volatile boolean paused;
 
@@ -90,9 +92,27 @@ public void createRemaining(Config config) {
       agentHttpClient =
           OkHttpUtils.buildHttpClient(
               OkHttpUtils.isPlainHttp(agentUrl), unixDomainSocket, namedPipe, httpClientTimeout);
+      String testSessionToken = config.getTestAgentSessionToken();
+      if (testSessionToken != null) {
+        agentHttpClient = injectTestAgentSessionHeaderInterceptor(testSessionToken);
+      }
     }
   }
 
+  private OkHttpClient injectTestAgentSessionHeaderInterceptor(String testSessionToken) {
+    return agentHttpClient
+        .newBuilder()
+        .addInterceptor(
+            chain ->
+                chain.proceed(
+                    chain
+                        .request()
+                        .newBuilder()
+                        .header(X_DATADOG_TEST_SESSION_TOKEN, testSessionToken)
+                        .build()))
+        .build();
+  }
+
   /** Registers a callback to be called when remote communications resume. */
   public void whenReady(Runnable callback) {
     if (paused) {

dd-java-agent/instrumentation-testing/src/main/java/datadog/trace/agent/test/assertions/SpanMatcher.java

@@ -451,6 +451,7 @@ private void assertSpanLinks(List<AgentSpanLink> links) {
           .expected(expectedLinkCount)
           .actual(linkCount)
           .buildAndThrow();
+      return;
     }
     for (int i = 0; i < expectedLinkCount; i++) {
       SpanLinkMatcher linkMatcher = this.linkMatchers[i];

dd-smoke-tests/appsec/springboot/src/test/groovy/datadog/smoketest/appsec/ExtendedDataCollectionSmokeTest.groovy

@@ -303,7 +303,7 @@ class ExtendedDataCollectionSmokeTest extends AbstractAppSecServerSmokeTest {
     }
     assert trigger != null, 'test trigger not found'
 
-    rootSpan.span.metaStruct != null
+    !rootSpan.span.metaStruct.isEmpty()
     def requestBody = rootSpan.span.metaStruct.get('http.request.body')
     assert requestBody != null, 'request body is not set'
     !rootSpan.meta.containsKey('_dd.appsec.request_body_size.exceeded')
@@ -343,7 +343,7 @@ class ExtendedDataCollectionSmokeTest extends AbstractAppSecServerSmokeTest {
     }
     assert trigger != null, 'test trigger not found'
 
-    rootSpan.span.metaStruct != null
+    !rootSpan.span.metaStruct.isEmpty()
     def requestBody = rootSpan.span.metaStruct.get('http.request.body')
     assert requestBody != null, 'request body is not set'
     !rootSpan.meta.containsKey('_dd.appsec.request_body_size.exceeded')
@@ -383,7 +383,7 @@ class ExtendedDataCollectionSmokeTest extends AbstractAppSecServerSmokeTest {
     }
     assert trigger != null, 'test trigger not found'
 
-    rootSpan.span.metaStruct != null
+    !rootSpan.span.metaStruct.isEmpty()
     def requestBody = rootSpan.span.metaStruct.get('http.request.body')
     assert requestBody != null, 'request body is not set'
     rootSpan.meta.containsKey('_dd.appsec.request_body_size.exceeded')
@@ -421,7 +421,7 @@ class ExtendedDataCollectionSmokeTest extends AbstractAppSecServerSmokeTest {
     }
     assert trigger == null, 'test trigger found'
 
-    rootSpan.span.metaStruct == null
+    rootSpan.span.metaStruct.isEmpty()
   }
 
   void 'test request body collection if WAF event with default-config'(){
@@ -457,7 +457,7 @@ class ExtendedDataCollectionSmokeTest extends AbstractAppSecServerSmokeTest {
     }
     assert trigger != null, 'test trigger not found'
 
-    rootSpan.span.metaStruct != null
+    !rootSpan.span.metaStruct.isEmpty()
     def requestBody = rootSpan.span.metaStruct.get('http.request.body')
     assert requestBody != null, 'request body is not set'
     !rootSpan.meta.containsKey('_dd.appsec.request_body_size.exceeded')

dd-smoke-tests/appsec/springboot/src/test/groovy/datadog/smoketest/appsec/SpringBootSmokeTest.groovy

@@ -699,7 +699,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
-    rootSpan.span.metaStruct != null
+    !rootSpan.span.metaStruct.isEmpty()
     def stack = rootSpan.span.metaStruct.get('_dd.stack')
     assert stack != null, 'stack is not set'
     def exploit = stack.get('exploit')
@@ -774,7 +774,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
-    rootSpan.span.metaStruct == null
+    rootSpan.span.metaStruct.isEmpty()
 
     where:
     variant               | _
@@ -814,7 +814,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
-    rootSpan.span.metaStruct == null
+    rootSpan.span.metaStruct.isEmpty()
 
     where:
     variant | _
@@ -853,7 +853,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
-    rootSpan.span.metaStruct == null
+    rootSpan.span.metaStruct.isEmpty()
   }
 
   def findFirstMatchingSpan(String resource) {
@@ -939,7 +939,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
-    rootSpan.span.metaStruct == null
+    rootSpan.span.metaStruct.isEmpty()
 
     where:
     endpoint                    | cmd                              | params
@@ -990,7 +990,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
-    rootSpan.span.metaStruct == null
+    rootSpan.span.metaStruct.isEmpty()
 
     where:
     endpoint           | cmd                                  | params

dd-smoke-tests/build.gradle

@@ -11,8 +11,20 @@ dependencies {
 
   compileOnly(libs.junit.jupiter)
 
+  // Testcontainers backs the test-agent .container() backend (TestAgentBackend). Kept compileOnly
+  // so it is NOT forced onto the ~79 modules that depend on this base for the mock/.external()
+  // backends; modules using .container() opt in with their own testImplementation dependency.
+  compileOnly(libs.testcontainers)
+
   compileOnly(libs.bundles.groovy)
   compileOnly(libs.bundles.spock)
+
+  testImplementation(libs.testcontainers)
+}
+
+tasks.withType(Test).configureEach {
+  // Cap concurrent Testcontainers usage across the build (TestAgentBackend container tests).
+  usesService(testcontainersLimit)
 }
 
 tasks.withType(GroovyCompile).configureEach {
@@ -42,6 +54,13 @@ subprojects { Project subProj ->
 
     // Tests depend on this to know where to run things and what agent jar to use
     jvmArgs "-Ddatadog.smoketest.builddir=${buildDir}"
+    // Forward an optional test-agent image override to the test JVM, so TraceBackend.testAgent()
+    // runs locally without internal-registry access, e.g.
+    // -Ddatadog.smoketest(.testagent.image=ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.44.0
+    def testAgentImage = System.getProperty("datadog.smoketest.testagent.image")
+    if (testAgentImage != null) {
+      jvmArgs "-Ddatadog.smoketest.testagent.image=${testAgentImage}"
+    }
     jvmArgumentProviders.add(new CommandLineArgumentProvider() {
         @Override
         Iterable<String> asArguments() {

dd-smoke-tests/gradle.lockfile

@@ -13,6 +13,10 @@ com.datadoghq:dd-instrument-java:0.0.4=compileClasspath,runtimeClasspath,testCom
 com.datadoghq:dd-javac-plugin-client:0.2.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.datadoghq:java-dogstatsd-client:4.4.5=runtimeClasspath,testRuntimeClasspath
 com.datadoghq:sketches-java:0.8.3=runtimeClasspath,testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-annotations:2.10.3=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.docker-java:docker-java-api:3.4.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.docker-java:docker-java-transport-zerodep:3.4.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.docker-java:docker-java-transport:3.4.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.javaparser:javaparser-core:3.25.6=codenarc
 com.github.jnr:jffi:1.3.15=runtimeClasspath,testRuntimeClasspath
 com.github.jnr:jnr-a64asm:1.0.0=runtimeClasspath,testRuntimeClasspath
@@ -47,20 +51,22 @@ io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath
 io.sqreen:libsqreen:17.4.0=runtimeClasspath,testRuntimeClasspath
 javax.servlet:javax.servlet-api:3.1.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 jaxen:jaxen:2.0.0=spotbugs
-junit:junit:4.13.2=runtimeClasspath,testRuntimeClasspath
+junit:junit:4.13.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 net.bytebuddy:byte-buddy-agent:1.18.10=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 net.bytebuddy:byte-buddy:1.18.10=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 net.java.dev.jna:jna-platform:5.8.0=runtimeClasspath,testRuntimeClasspath
-net.java.dev.jna:jna:5.8.0=runtimeClasspath,testRuntimeClasspath
+net.java.dev.jna:jna:5.13.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+net.java.dev.jna:jna:5.8.0=runtimeClasspath
 net.sf.saxon:Saxon-HE:12.9=spotbugs
 org.apache.ant:ant-antlr:1.10.14=codenarc
 org.apache.ant:ant-junit:1.10.14=codenarc
 org.apache.bcel:bcel:6.11.0=spotbugs
+org.apache.commons:commons-compress:1.24.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.commons:commons-lang3:3.19.0=spotbugs
 org.apache.commons:commons-text:1.14.0=spotbugs
 org.apache.logging.log4j:log4j-api:2.25.2=spotbugs
 org.apache.logging.log4j:log4j-core:2.25.2=spotbugs
-org.apiguardian:apiguardian-api:1.1.2=compileClasspath,testCompileClasspath
+org.apiguardian:apiguardian-api:1.1.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.codehaus.groovy:groovy-ant:3.0.23=codenarc
 org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc
 org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc
@@ -73,10 +79,11 @@ org.codehaus.groovy:groovy:3.0.25=compileClasspath,testCompileClasspath,testRunt
 org.codenarc:CodeNarc:3.7.0=codenarc
 org.dom4j:dom4j:2.2.0=spotbugs
 org.gmetrics:GMetrics:2.1.0=codenarc
-org.hamcrest:hamcrest-core:1.3=runtimeClasspath,testRuntimeClasspath
+org.hamcrest:hamcrest-core:1.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.hamcrest:hamcrest:3.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.jctools:jctools-core-jdk11:4.0.6=runtimeClasspath,testRuntimeClasspath
 org.jctools:jctools-core:4.0.6=runtimeClasspath,testRuntimeClasspath
+org.jetbrains:annotations:17.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.jspecify:jspecify:1.0.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.junit.jupiter:junit-jupiter-api:5.14.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath
@@ -104,15 +111,18 @@ org.ow2.asm:asm-util:9.7.1=runtimeClasspath,testRuntimeClasspath
 org.ow2.asm:asm-util:9.9=spotbugs
 org.ow2.asm:asm:9.10.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm:9.9=spotbugs
+org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:jcl-over-slf4j:1.7.30=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:jul-to-slf4j:1.7.30=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:log4j-over-slf4j:1.7.30=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.slf4j:slf4j-api:1.7.32=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.slf4j:slf4j-api:1.7.32=runtimeClasspath
+org.slf4j:slf4j-api:1.7.36=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j
 org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j
 org.spockframework:spock-bom:2.4-groovy-3.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.spockframework:spock-core:2.4-groovy-3.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath
 org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath
+org.testcontainers:testcontainers:1.21.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.xmlresolver:xmlresolver:5.3.3=spotbugs
 empty=annotationProcessor,spotbugsPlugins,testAnnotationProcessor

dd-smoke-tests/opentelemetry/build.gradle

@@ -15,9 +15,13 @@ dependencies {
   implementation group: 'io.opentelemetry', name: 'opentelemetry-api', version: '1.4.0'
   implementation group: 'io.opentelemetry.instrumentation', name: 'opentelemetry-instrumentation-annotations', version: '1.20.0'
   testImplementation project(':dd-smoke-tests')
+  // Needed to use TraceBackend.testAgent(): Testcontainers is compileOnly on the smoke-test base
+  // (so it isn't forced on every consumer), so a module using the container backend opts in here.
+  testImplementation libs.testcontainers
 }
 
 tasks.withType(Test).configureEach {
+  usesService(testcontainersLimit)
   dependsOn 'shadowJar'
   def shadowJarTask = tasks.named('shadowJar', ShadowJar)
   jvmArgumentProviders.add(new CommandLineArgumentProvider() {

dd-smoke-tests/opentelemetry/gradle.lockfile

@@ -13,6 +13,10 @@ com.datadoghq:dd-instrument-java:0.0.4=testCompileClasspath,testRuntimeClasspath
 com.datadoghq:dd-javac-plugin-client:0.2.2=testCompileClasspath,testRuntimeClasspath
 com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath
 com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-annotations:2.10.3=testCompileClasspath,testRuntimeClasspath
+com.github.docker-java:docker-java-api:3.4.2=testCompileClasspath,testRuntimeClasspath
+com.github.docker-java:docker-java-transport-zerodep:3.4.2=testCompileClasspath,testRuntimeClasspath
+com.github.docker-java:docker-java-transport:3.4.2=testCompileClasspath,testRuntimeClasspath
 com.github.javaparser:javaparser-core:3.25.6=codenarc
 com.github.jnr:jffi:1.3.15=testRuntimeClasspath
 com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath
@@ -50,20 +54,21 @@ io.opentelemetry:opentelemetry-context:1.19.0=compileClasspath,runtimeClasspath,
 io.sqreen:libsqreen:17.4.0=testRuntimeClasspath
 javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath
 jaxen:jaxen:2.0.0=spotbugs
-junit:junit:4.13.2=testRuntimeClasspath
+junit:junit:4.13.2=testCompileClasspath,testRuntimeClasspath
 net.bytebuddy:byte-buddy-agent:1.18.10=testCompileClasspath,testRuntimeClasspath
 net.bytebuddy:byte-buddy:1.18.10=testCompileClasspath,testRuntimeClasspath
 net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath
-net.java.dev.jna:jna:5.8.0=testRuntimeClasspath
+net.java.dev.jna:jna:5.13.0=testCompileClasspath,testRuntimeClasspath
 net.sf.saxon:Saxon-HE:12.9=spotbugs
 org.apache.ant:ant-antlr:1.10.14=codenarc
 org.apache.ant:ant-junit:1.10.14=codenarc
 org.apache.bcel:bcel:6.11.0=spotbugs
+org.apache.commons:commons-compress:1.24.0=testCompileClasspath,testRuntimeClasspath
 org.apache.commons:commons-lang3:3.19.0=spotbugs
 org.apache.commons:commons-text:1.14.0=spotbugs
 org.apache.logging.log4j:log4j-api:2.25.2=spotbugs
 org.apache.logging.log4j:log4j-core:2.25.2=spotbugs
-org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
+org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testRuntimeClasspath
 org.codehaus.groovy:groovy-ant:3.0.23=codenarc
 org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc
 org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc
@@ -76,10 +81,11 @@ org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath
 org.codenarc:CodeNarc:3.7.0=codenarc
 org.dom4j:dom4j:2.2.0=spotbugs
 org.gmetrics:GMetrics:2.1.0=codenarc
-org.hamcrest:hamcrest-core:1.3=testRuntimeClasspath
+org.hamcrest:hamcrest-core:1.3=testCompileClasspath,testRuntimeClasspath
 org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath
 org.jctools:jctools-core-jdk11:4.0.6=testRuntimeClasspath
 org.jctools:jctools-core:4.0.6=testRuntimeClasspath
+org.jetbrains:annotations:17.0.0=testCompileClasspath,testRuntimeClasspath
 org.jspecify:jspecify:1.0.0=testCompileClasspath,testRuntimeClasspath
 org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath
 org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath
@@ -107,15 +113,17 @@ org.ow2.asm:asm-util:9.7.1=testRuntimeClasspath
 org.ow2.asm:asm-util:9.9=spotbugs
 org.ow2.asm:asm:9.10.1=testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm:9.9=spotbugs
+org.rnorth.duct-tape:duct-tape:1.0.8=testCompileClasspath,testRuntimeClasspath
 org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath
 org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath
 org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath
-org.slf4j:slf4j-api:1.7.32=testCompileClasspath,testRuntimeClasspath
+org.slf4j:slf4j-api:1.7.36=testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j
 org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j
 org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath
 org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath
 org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath
 org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath
+org.testcontainers:testcontainers:1.21.4=testCompileClasspath,testRuntimeClasspath
 org.xmlresolver:xmlresolver:5.3.3=spotbugs
 empty=annotationProcessor,shadow,spotbugsPlugins,testAnnotationProcessor