Skip to content

Truncate CI Visibility meta tag values #11485

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
juan-fernandez wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Truncate CI Visibility meta tag values #11485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions ...ce-core/src/main/java/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import static datadog.communication.http.OkHttpUtils.gzippedMsgpackRequestBodyOf;
import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf;
import static datadog.json.JsonMapper.toJson;
import static datadog.trace.common.writer.CiVisibilityMetaTruncation.truncate;

import datadog.communication.serialization.GrowableBuffer;
import datadog.communication.serialization.Writable;
Expand Down Expand Up @@ -240,34 +241,34 @@ private void writeHeader() {
headerWriter.startMap(10);
/* 2,1,1 */
headerWriter.writeUTF8(ENV);
headerWriter.writeUTF8(wellKnownTags.getEnv());
headerWriter.writeUTF8(truncate(wellKnownTags.getEnv()));
Copy link
Copy Markdown

@chatgpt-codex-connector chatgpt-codex-connector Bot Jun 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Convert well-known tags before truncating

In this header path, CiVisibilityWellKnownTags#getEnv() and the following well-known tag getters return UTF8BytesString, but the new truncate helper only accepts String. Java has to resolve truncate(...) before it can call writeUTF8, so this makes dd-trace-core fail to compile for the CI test-cycle mapper; convert these values to strings first or make the helper accept CharSequence.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor

@mcculls mcculls Jun 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For well-known tags that will never change I'd recommend creating your own CI-Vis holder class containing truncated well-known tags. That would be more efficient and could also help address the comment above, because you could ensure that any truncated well-known tag values stay as the more efficient UTF8BytesString for marshalling purposes.

I'd also suggest looking at the existing Strings utility class that has truncate methods for String and CharSequence. You could add a similar truncate utility method that accepts UTF8BytesString and returns a UTF8BytesString.

/* 2,1,2 */
headerWriter.writeUTF8(RUNTIME_ID);
headerWriter.writeUTF8(wellKnownTags.getRuntimeId());
headerWriter.writeUTF8(truncate(wellKnownTags.getRuntimeId()));
/* 2,1,3 */
headerWriter.writeUTF8(LANGUAGE);
headerWriter.writeUTF8(wellKnownTags.getLanguage());
headerWriter.writeUTF8(truncate(wellKnownTags.getLanguage()));
/* 2,1,4 */
headerWriter.writeUTF8(RUNTIME_NAME);
headerWriter.writeUTF8(wellKnownTags.getRuntimeName());
headerWriter.writeUTF8(truncate(wellKnownTags.getRuntimeName()));
/* 2,1,5 */
headerWriter.writeUTF8(RUNTIME_VENDOR);
headerWriter.writeUTF8(wellKnownTags.getRuntimeVendor());
headerWriter.writeUTF8(truncate(wellKnownTags.getRuntimeVendor()));
/* 2,1,6 */
headerWriter.writeUTF8(RUNTIME_VERSION);
headerWriter.writeUTF8(wellKnownTags.getRuntimeVersion());
headerWriter.writeUTF8(truncate(wellKnownTags.getRuntimeVersion()));
/* 2,1,7 */
headerWriter.writeUTF8(OS_ARCHITECTURE);
headerWriter.writeUTF8(wellKnownTags.getOsArch());
headerWriter.writeUTF8(truncate(wellKnownTags.getOsArch()));
/* 2,1,8 */
headerWriter.writeUTF8(OS_PLATFORM);
headerWriter.writeUTF8(wellKnownTags.getOsPlatform());
headerWriter.writeUTF8(truncate(wellKnownTags.getOsPlatform()));
/* 2,1,9 */
headerWriter.writeUTF8(OS_VERSION);
headerWriter.writeUTF8(wellKnownTags.getOsVersion());
headerWriter.writeUTF8(truncate(wellKnownTags.getOsVersion()));
/* 2,1,10 */
headerWriter.writeUTF8(TEST_IS_USER_PROVIDED_SERVICE);
headerWriter.writeUTF8(wellKnownTags.getIsUserProvidedService());
headerWriter.writeUTF8(truncate(wellKnownTags.getIsUserProvidedService()));
/* 3 */
headerWriter.writeUTF8(EVENTS);
headerWriter.startArray(eventCount);
Expand Down Expand Up @@ -350,21 +351,21 @@ public void accept(Metadata metadata) {
// we just need to be sure that the size is the same as the number of elements
for (Map.Entry<String, String> entry : metadata.getBaggage().entrySet()) {
writable.writeString(entry.getKey(), null);
writable.writeString(entry.getValue(), null);
writable.writeString(truncate(entry.getValue()), null);
}
if (null != metadata.getHttpStatusCode()) {
writable.writeUTF8(HTTP_STATUS);
writable.writeUTF8(metadata.getHttpStatusCode());
writable.writeUTF8(truncate(metadata.getHttpStatusCode().toString()));
}
for (Map.Entry<String, Object> entry : tags.entrySet()) {
Object value = entry.getValue();
if (!(value instanceof Number)) {
writable.writeString(entry.getKey(), null);
if (!(value instanceof Iterable)) {
writable.writeObjectString(value, null);
writable.writeString(truncate(String.valueOf(value)), null);
} else {
String serializedValue = toJson((Collection<String>) value);
writable.writeString(serializedValue, null);
writable.writeString(truncate(serializedValue), null);
}
}
}
Expand Down
14 changes: 14 additions & 0 deletions dd-trace-core/src/main/java/datadog/trace/common/writer/CiVisibilityMetaTruncation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package datadog.trace.common.writer;

public final class CiVisibilityMetaTruncation {
public static final int MAX_META_STRING_VALUE_LENGTH = 5000;

private CiVisibilityMetaTruncation() {}

public static String truncate(String value) {
if (value == null || value.length() <= MAX_META_STRING_VALUE_LENGTH) {
return value;
}
return value.substring(0, MAX_META_STRING_VALUE_LENGTH);
}
}
15 changes: 8 additions & 7 deletions dd-trace-core/src/main/java/datadog/trace/common/writer/FileBasedPayloadDispatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package datadog.trace.common.writer;

import static datadog.json.JsonMapper.toJson;
import static datadog.trace.common.writer.CiVisibilityMetaTruncation.truncate;

import datadog.json.JsonWriter;
import datadog.trace.api.Config;
Expand Down Expand Up @@ -153,10 +154,10 @@ public void flush() {
doc.beginObject();
doc.name("*");
doc.beginObject();
doc.name("env").value(wellKnownTags.getEnv().toString());
doc.name("language").value(wellKnownTags.getLanguage().toString());
doc.name("env").value(truncate(wellKnownTags.getEnv().toString()));
doc.name("language").value(truncate(wellKnownTags.getLanguage().toString()));
doc.name("test_is_user_provided_service")
.value(wellKnownTags.getIsUserProvidedService().toString());
.value(truncate(wellKnownTags.getIsUserProvidedService().toString()));
doc.endObject();
doc.endObject();
doc.name("events");
Expand Down Expand Up @@ -383,20 +384,20 @@ public void accept(Metadata metadata) {
w.beginObject();
for (Map.Entry<String, String> entry : metadata.getBaggage().entrySet()) {
if (!isExcludedTag(entry.getKey())) {
w.name(entry.getKey()).value(entry.getValue());
w.name(entry.getKey()).value(truncate(entry.getValue()));
}
}
if (metadata.getHttpStatusCode() != null) {
w.name(Tags.HTTP_STATUS).value(metadata.getHttpStatusCode().toString());
w.name(Tags.HTTP_STATUS).value(truncate(metadata.getHttpStatusCode().toString()));
}
for (Map.Entry<String, Object> entry : tags.entrySet()) {
Object value = entry.getValue();
if (!(value instanceof Number) && !isExcludedTag(entry.getKey())) {
w.name(entry.getKey());
if (value instanceof Iterable) {
w.value(toJson((Collection<String>) value));
w.value(truncate(toJson((Collection<String>) value)));
} else {
w.value(String.valueOf(value));
w.value(truncate(String.valueOf(value)));
}
}
}
Expand Down
72 changes: 62 additions & 10 deletions ...t/groovy/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1PayloadTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import java.nio.ByteBuffer
import java.nio.channels.WritableByteChannel

import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_MEASURED
import static datadog.trace.common.writer.CiVisibilityMetaTruncation.MAX_META_STRING_VALUE_LENGTH
import static datadog.trace.common.writer.CiVisibilityMetaTruncation.truncate
import static datadog.trace.common.writer.TraceGenerator.generateRandomSpan
import static datadog.trace.common.writer.TraceGenerator.generateRandomTraces
import static org.junit.jupiter.api.Assertions.assertEquals
Expand Down Expand Up @@ -110,6 +112,56 @@ class CiTestCycleMapperV1PayloadTest extends DDSpecification {
assert spanContent.containsKey("parent_id")
}

def "truncates meta string values and preserves metrics and top level ids"() {
Copy link
Copy Markdown

@chatgpt-codex-connector chatgpt-codex-connector Bot Jun 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Migrate the new mapper tests to JUnit 5

This adds new Spock/Groovy coverage in a file under /workspace/dd-trace-java, but the repository instructions in /workspace/dd-trace-java/AGENTS.md explicitly say not to write new Groovy/Spock tests and to migrate existing Groovy tests to JUnit 5. Please move this new coverage to a JUnit 5 Java test instead of expanding the Spock suite.

Useful? React with 👍 / 👎.

setup:
String longValue = "a" * (MAX_META_STRING_VALUE_LENGTH + 1)
String exactValue = "b" * MAX_META_STRING_VALUE_LENGTH
def span = generateRandomSpan(InternalSpanTypes.TEST, [
(Tags.TEST_SESSION_ID): DDTraceId.from(123),
(Tags.TEST_MODULE_ID) : 456,
(Tags.TEST_SUITE_ID) : 789,
"custom.tag" : longValue,
"exact.tag" : exactValue,
"custom.metric" : 42,
])

when:
Map<String, Object> deserializedSpan = whenASpanIsWritten(span)

then:
verifyTopLevelTags(deserializedSpan, DDTraceId.from(123), 456, 789)

def spanContent = (Map<String, Object>) deserializedSpan.get("content")
def deserializedMetrics = (Map<String, Object>) spanContent.get("metrics")
def deserializedMeta = (Map<String, Object>) spanContent.get("meta")

assert deserializedMeta.get("custom.tag") == longValue.substring(0, MAX_META_STRING_VALUE_LENGTH)
assert deserializedMeta.get("custom.tag").length() == MAX_META_STRING_VALUE_LENGTH
assert deserializedMeta.get("exact.tag") == exactValue
assert deserializedMetrics.get("custom.metric") == 42
}

def "truncates payload metadata values"() {
setup:
String longValue = "m" * (MAX_META_STRING_VALUE_LENGTH + 1)
CiVisibilityWellKnownTags wellKnownTags = new CiVisibilityWellKnownTags(
longValue, longValue, longValue,
longValue, longValue, longValue,
longValue, longValue, longValue, longValue)
CiTestCycleMapperV1 mapper = new CiTestCycleMapperV1(wellKnownTags, false)
List<List<TraceGenerator.PojoSpan>> traces = Collections.singletonList(
Collections.singletonList(generateRandomSpan(InternalSpanTypes.TEST, Collections.emptyMap())))
PayloadVerifier verifier = new PayloadVerifier(wellKnownTags, traces, mapper)
MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(100 << 10, verifier))

when:
packer.format(traces.get(0), mapper)
packer.flush()

then:
verifier.verifyTracesConsumed()
}

def "verify test_suite_end event is written correctly"() {
setup:
def span = generateRandomSpan(InternalSpanTypes.TEST_SUITE_END, [
Expand Down Expand Up @@ -275,25 +327,25 @@ class CiTestCycleMapperV1PayloadTest extends DDSpecification {

assertEquals(10, unpacker.unpackMapHeader())
assertEquals("env", unpacker.unpackString())
assertEquals(wellKnownTags.env as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.env as String), unpacker.unpackString())
assertEquals("runtime-id", unpacker.unpackString())
assertEquals(wellKnownTags.runtimeId as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.runtimeId as String), unpacker.unpackString())
assertEquals("language", unpacker.unpackString())
assertEquals(wellKnownTags.language as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.language as String), unpacker.unpackString())
assertEquals(Tags.RUNTIME_NAME, unpacker.unpackString())
assertEquals(wellKnownTags.runtimeName as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.runtimeName as String), unpacker.unpackString())
assertEquals(Tags.RUNTIME_VENDOR, unpacker.unpackString())
assertEquals(wellKnownTags.runtimeVendor as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.runtimeVendor as String), unpacker.unpackString())
assertEquals(Tags.RUNTIME_VERSION, unpacker.unpackString())
assertEquals(wellKnownTags.runtimeVersion as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.runtimeVersion as String), unpacker.unpackString())
assertEquals(Tags.OS_ARCHITECTURE, unpacker.unpackString())
assertEquals(wellKnownTags.osArch as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.osArch as String), unpacker.unpackString())
assertEquals(Tags.OS_PLATFORM, unpacker.unpackString())
assertEquals(wellKnownTags.osPlatform as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.osPlatform as String), unpacker.unpackString())
assertEquals(Tags.OS_VERSION, unpacker.unpackString())
assertEquals(wellKnownTags.osVersion as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.osVersion as String), unpacker.unpackString())
assertEquals(DDTags.TEST_IS_USER_PROVIDED_SERVICE, unpacker.unpackString())
assertEquals(wellKnownTags.isUserProvidedService as String, unpacker.unpackString())
assertEquals(truncate(wellKnownTags.isUserProvidedService as String), unpacker.unpackString())

assertEquals("events", unpacker.unpackString())

Expand Down
47 changes: 47 additions & 0 deletions dd-trace-core/src/test/java/datadog/trace/common/writer/FileBasedPayloadDispatcherTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,47 @@ void citestcycleStripsCiGitOsRuntimeTagsAndWellKnownFields(@TempDir Path outputD
assertEquals(99L, metrics.get("kept.metric").asLong());
}

@Test
void citestcycleTruncatesMetaValuesAndPreservesMetricsAndTopLevelIds(@TempDir Path outputDir)
throws IOException {
FileBasedPayloadDispatcher dispatcher =
new FileBasedPayloadDispatcher(outputDir.toString(), "tests", TrackType.CITESTCYCLE);
String longValue =
longString(CiVisibilityMetaTruncation.MAX_META_STRING_VALUE_LENGTH + 1, 'a');
String exactValue = longString(CiVisibilityMetaTruncation.MAX_META_STRING_VALUE_LENGTH, 'b');
Map<String, Object> tags = new HashMap<>();
tags.put(Tags.TEST_SESSION_ID, DDTraceId.from(123));
tags.put(Tags.TEST_MODULE_ID, 456L);
tags.put(Tags.TEST_SUITE_ID, 789L);
tags.put("custom.tag", longValue);
tags.put("exact.tag", exactValue);
tags.put("custom.metric", 42L);
CoreSpan<?> span = mockSpan(InternalSpanTypes.TEST, tags);

dispatcher.addTrace(Collections.singletonList(span));
dispatcher.flush();

JsonNode content =
JSON.readTree(listFiles(outputDir).get(0).toFile()).get("events").get(0).get("content");
JsonNode meta = content.get("meta");
JsonNode metrics = content.get("metrics");

assertEquals(
longValue.substring(0, CiVisibilityMetaTruncation.MAX_META_STRING_VALUE_LENGTH),
meta.get("custom.tag").asText());
assertEquals(
CiVisibilityMetaTruncation.MAX_META_STRING_VALUE_LENGTH,
meta.get("custom.tag").asText().length());
assertEquals(exactValue, meta.get("exact.tag").asText());
assertEquals(42L, metrics.get("custom.metric").asLong());
assertFalse(meta.has(Tags.TEST_SESSION_ID));
assertFalse(meta.has(Tags.TEST_MODULE_ID));
assertFalse(meta.has(Tags.TEST_SUITE_ID));
assertEquals(123L, content.get(Tags.TEST_SESSION_ID).asLong());
assertEquals(456L, content.get(Tags.TEST_MODULE_ID).asLong());
assertEquals(789L, content.get(Tags.TEST_SUITE_ID).asLong());
}

@Test
void citestcycleAssignsEventTypesForSessionModuleSuiteTestSpanSpans(@TempDir Path outputDir)
throws IOException {
Expand Down Expand Up @@ -295,4 +336,10 @@ private static List<Path> listFiles(Path dir) throws IOException {
}
return files;
}

private static String longString(int length, char value) {
char[] chars = new char[length];
Arrays.fill(chars, value);
return new String(chars);
}
}
Loading