From 0efcc7c518513cedd12b5201856106a3765663ca Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Wed, 24 Jun 2026 14:02:06 +0200 Subject: [PATCH] feat: replace v04 usage by v1 in sidecar --- datadog-sidecar-ffi/src/lib.rs | 46 ++++++++++ datadog-sidecar/src/service/blocking.rs | 24 ++++++ datadog-sidecar/src/service/sender.rs | 27 ++++++ .../src/service/sidecar_interface.rs | 32 +++++++ datadog-sidecar/src/service/sidecar_server.rs | 83 ++++++++++++++++++- .../src/trace_exporter/trace_serializer.rs | 8 +- libdd-trace-utils/src/send_data/mod.rs | 27 ++++++ libdd-trace-utils/src/trace_utils.rs | 62 +++++--------- libdd-trace-utils/src/tracer_payload.rs | 31 +++++-- 9 files changed, 287 insertions(+), 53 deletions(-) diff --git a/datadog-sidecar-ffi/src/lib.rs b/datadog-sidecar-ffi/src/lib.rs index 25fa4550bc..566ef08d38 100644 --- a/datadog-sidecar-ffi/src/lib.rs +++ b/datadog-sidecar-ffi/src/lib.rs @@ -1109,6 +1109,52 @@ pub unsafe extern "C" fn ddog_sidecar_send_trace_v04_bytes( MaybeError::None } +/// Sends a v0.4-encoded trace to the sidecar via shared memory; the sidecar will re-encode +/// it as a V1 msgpack payload before forwarding to the agent. +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_sidecar_send_trace_v1_shm( + transport: &mut Box, + instance_id: &InstanceId, + shm_handle: Box, + len: usize, + tracer_header_tags: &TracerHeaderTags, +) -> MaybeError { + let tracer_header_tags = try_c!(tracer_header_tags.try_into()); + + try_c!(blocking::send_trace_v1_shm( + transport, + instance_id, + *shm_handle, + len, + tracer_header_tags, + )); + + MaybeError::None +} + +/// Sends a v0.4-encoded trace as bytes to the sidecar; the sidecar will re-encode it as a +/// V1 msgpack payload before forwarding to the agent. +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_sidecar_send_trace_v1_bytes( + transport: &mut Box, + instance_id: &InstanceId, + data: ffi::CharSlice, + tracer_header_tags: &TracerHeaderTags, +) -> MaybeError { + let tracer_header_tags = try_c!(tracer_header_tags.try_into()); + + try_c!(blocking::send_trace_v1_bytes( + transport, + instance_id, + data.as_bytes().to_vec(), + tracer_header_tags, + )); + + MaybeError::None +} + #[no_mangle] #[allow(clippy::missing_safety_doc)] #[allow(improper_ctypes_definitions)] // DebuggerPayload is just a pointer, we hide its internals diff --git a/datadog-sidecar/src/service/blocking.rs b/datadog-sidecar/src/service/blocking.rs index 80a9257384..9506e575bd 100644 --- a/datadog-sidecar/src/service/blocking.rs +++ b/datadog-sidecar/src/service/blocking.rs @@ -294,6 +294,30 @@ pub fn send_trace_v04_shm( Ok(()) } +/// Sends a v0.4-encoded trace as bytes; the sidecar re-encodes it as V1 before forwarding. +pub fn send_trace_v1_bytes( + transport: &mut SidecarTransport, + instance_id: &InstanceId, + data: Vec, + headers: SerializedTracerHeaderTags, +) -> io::Result<()> { + lock_sender(transport)?.send_trace_v1_bytes(instance_id.clone(), data, headers); + Ok(()) +} + +/// Sends a v0.4-encoded trace via shared memory; the sidecar re-encodes it as V1 before +/// forwarding. +pub fn send_trace_v1_shm( + transport: &mut SidecarTransport, + instance_id: &InstanceId, + handle: ShmHandle, + len: usize, + headers: SerializedTracerHeaderTags, +) -> io::Result<()> { + lock_sender(transport)?.send_trace_v1_shm(instance_id.clone(), handle, len, headers); + Ok(()) +} + /// Sends raw data from shared memory to the debugger endpoint. pub fn send_debugger_data_shm( transport: &mut SidecarTransport, diff --git a/datadog-sidecar/src/service/sender.rs b/datadog-sidecar/src/service/sender.rs index e88720e23f..e66e7396eb 100644 --- a/datadog-sidecar/src/service/sender.rs +++ b/datadog-sidecar/src/service/sender.rs @@ -380,6 +380,33 @@ impl SidecarSender { .try_send_send_trace_v04_bytes(instance_id, data, headers); } + pub fn send_trace_v1_shm( + &mut self, + instance_id: InstanceId, + handle: ShmHandle, + len: usize, + headers: SerializedTracerHeaderTags, + ) { + if !self.try_drain_outbox() { + return; + } + self.channel + .try_send_send_trace_v1_shm(instance_id, handle, len, headers); + } + + pub fn send_trace_v1_bytes( + &mut self, + instance_id: InstanceId, + data: Vec, + headers: SerializedTracerHeaderTags, + ) { + if !self.try_drain_outbox() { + return; + } + self.channel + .try_send_send_trace_v1_bytes(instance_id, data, headers); + } + pub fn send_debugger_data_shm( &mut self, instance_id: InstanceId, diff --git a/datadog-sidecar/src/service/sidecar_interface.rs b/datadog-sidecar/src/service/sidecar_interface.rs index 6f6244b258..7e54d260d2 100644 --- a/datadog-sidecar/src/service/sidecar_interface.rs +++ b/datadog-sidecar/src/service/sidecar_interface.rs @@ -129,6 +129,38 @@ pub trait SidecarInterface { headers: SerializedTracerHeaderTags, ); + /// Sends a v0.4-encoded trace via shared memory; the sidecar re-encodes it as a V1 + /// msgpack payload before forwarding to the agent. Use this when the upstream SDK only + /// speaks v0.4 but the agent advertises `/v1.0/traces`. + /// + /// # Arguments + /// + /// * `instance_id` - The ID of the instance. + /// * `handle` - The handle to the shared memory. + /// * `len` - The size of the shared memory data. + /// * `headers` - The serialized headers from the tracer. + async fn send_trace_v1_shm( + instance_id: InstanceId, + #[SerializedHandle] handle: ShmHandle, + len: usize, + headers: SerializedTracerHeaderTags, + ); + + /// Sends a v0.4-encoded trace as bytes; the sidecar re-encodes it as a V1 msgpack payload + /// before forwarding to the agent. Use this when the upstream SDK only speaks v0.4 but + /// the agent advertises `/v1.0/traces`. + /// + /// # Arguments + /// + /// * `instance_id` - The ID of the instance. + /// * `data` - The v0.4 trace data serialized as bytes. + /// * `headers` - The serialized headers from the tracer. + async fn send_trace_v1_bytes( + instance_id: InstanceId, + data: Vec, + headers: SerializedTracerHeaderTags, + ); + /// Transfers raw data to a live-debugger endpoint. /// /// # Arguments diff --git a/datadog-sidecar/src/service/sidecar_server.rs b/datadog-sidecar/src/service/sidecar_server.rs index 7a5e095d13..5cdac56cc3 100644 --- a/datadog-sidecar/src/service/sidecar_server.rs +++ b/datadog-sidecar/src/service/sidecar_server.rs @@ -249,6 +249,30 @@ impl SidecarServer { data: tinybytes::Bytes, target: &Endpoint, retry_interval: u64, + ) { + self.send_trace(headers, data, target, retry_interval, TraceEncoding::V04) + } + + /// Re-encode entry point for the V1 path. Input bytes are still v0.4 msgpack from the SDK; + /// the [`TraceEncoding::V1`] tag tells [`SendData`] to encode the wire payload as V1 before + /// forwarding to the agent. + fn send_trace_v1( + &self, + headers: &SerializedTracerHeaderTags, + data: tinybytes::Bytes, + target: &Endpoint, + retry_interval: u64, + ) { + self.send_trace(headers, data, target, retry_interval, TraceEncoding::V1) + } + + fn send_trace( + &self, + headers: &SerializedTracerHeaderTags, + data: tinybytes::Bytes, + target: &Endpoint, + retry_interval: u64, + encoding: TraceEncoding, ) { let headers: TracerHeaderTags = match headers.try_into() { Ok(headers) => headers, @@ -265,7 +289,7 @@ impl SidecarServer { headers ); - match decode_to_trace_chunks(data, TraceEncoding::V04) { + match decode_to_trace_chunks(data, encoding) { Ok((payload, size)) => { trace!("Parsed the trace payload and enqueuing it for sending: {payload:?}"); let mut data = SendData::new( @@ -898,6 +922,63 @@ impl SidecarInterface for ConnectionSidecarHandler { } } + async fn send_trace_v1_shm( + &self, + _peer: PeerCredentials, + instance_id: InstanceId, + handle: ShmHandle, + _len: usize, + headers: SerializedTracerHeaderTags, + ) { + self.track_instance(&instance_id); + let session = self.server.get_session(&instance_id.session_id); + let trace_config = session.get_trace_config(); + if let Some(endpoint) = trace_config.endpoint.clone() { + let server = self.server.clone(); + let retry_interval = trace_config.retry_interval; + tokio::spawn(async move { + match handle.map() { + Ok(mapped) => { + let bytes = tinybytes::Bytes::from(mapped); + server.send_trace_v1(&headers, bytes, &endpoint, retry_interval); + } + Err(e) => error!("Failed mapping shared trace data memory: {}", e), + } + }); + } else { + warn!( + "Received trace data ({handle:?}) for missing session {}", + instance_id.session_id + ); + } + } + + async fn send_trace_v1_bytes( + &self, + _peer: PeerCredentials, + instance_id: InstanceId, + data: Vec, + headers: SerializedTracerHeaderTags, + ) { + self.track_instance(&instance_id); + let session = self.server.get_session(&instance_id.session_id); + let trace_config = session.get_trace_config(); + + if let Some(endpoint) = trace_config.endpoint.clone() { + let server = self.server.clone(); + let retry_interval = trace_config.retry_interval; + tokio::spawn(async move { + let bytes = tinybytes::Bytes::from(data); + server.send_trace_v1(&headers, bytes, &endpoint, retry_interval); + }); + } else { + warn!( + "Received trace data for missing session {}", + instance_id.session_id + ); + } + } + async fn send_debugger_data_shm( &self, _peer: PeerCredentials, diff --git a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs index 7cfd62a5a0..30dc497c83 100644 --- a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs +++ b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs @@ -17,7 +17,7 @@ use libdd_trace_utils::msgpack_encoder; use libdd_trace_utils::span::{v04::Span, TraceData}; use libdd_trace_utils::trace_utils::{self, TracerHeaderTags}; use libdd_trace_utils::tracer_metadata::TracerMetadata; -use libdd_trace_utils::tracer_payload::{self, TraceEncoding}; +use libdd_trace_utils::tracer_payload::{self}; /// Minimal capacity of fresh buffers allocated to encode traces, in bytes. const MIN_BUFFER_CAPACITY: usize = 1024; @@ -79,11 +79,9 @@ impl TraceSerializer { }; match output_format { TraceExporterOutputFormat::V1 => Ok(tracer_payload::TraceChunks::V1(traces)), - TraceExporterOutputFormat::V04 => { - trace_utils::collect_trace_chunks(traces, TraceEncoding::V04).map_err(map_err) - } + TraceExporterOutputFormat::V04 => Ok(tracer_payload::TraceChunks::V04(traces)), TraceExporterOutputFormat::V05 => { - trace_utils::collect_trace_chunks(traces, TraceEncoding::V05).map_err(map_err) + trace_utils::convert_trace_chunks_v04_to_v05(traces).map_err(map_err) } } } diff --git a/libdd-trace-utils/src/send_data/mod.rs b/libdd-trace-utils/src/send_data/mod.rs index 53d14d8ecd..5a219aa665 100644 --- a/libdd-trace-utils/src/send_data/mod.rs +++ b/libdd-trace-utils/src/send_data/mod.rs @@ -394,6 +394,29 @@ impl SendData { endpoint.as_ref(), )); } + // TracerPayloadCollection::V1(payload) => { + // // V0.4-shaped spans re-encoded as a V1 msgpack payload at send time. Used by + // // the sidecar when the upstream SDK only speaks v0.4 but the agent advertises + // // `/v1.0/traces`. `extract_payload_attrs` will pull env/hostname/app_version + // // from span meta tags since the SDK propagates them there for v0.4 payloads. + // #[allow(clippy::unwrap_used)] + // let chunks = u64::try_from(self.tracer_payloads.size()).unwrap(); + // let mut headers = self.headers.clone(); + // headers.reserve(2); + // headers.insert(DATADOG_TRACE_COUNT, chunks.into()); + // headers.insert(CONTENT_TYPE, APPLICATION_MSGPACK); + + // let metadata = crate::tracer_metadata::TracerMetadata::default(); + // let payload = msgpack_encoder::v1::to_vec_from_payload_v1(payload); + + // futures.push(self.send_payload( + // capabilities, + // chunks, + // payload, + // headers, + // endpoint.as_ref(), + // )); + // } TracerPayloadCollection::V05(payload) => { #[allow(clippy::unwrap_used)] let chunks = u64::try_from(self.tracer_payloads.size()).unwrap(); @@ -537,6 +560,10 @@ mod tests { msgpack_encoder::v04::to_encoded_byte_len(payloads) as usize } TracerPayloadCollection::V05(payloads) => rmp_serde::to_vec(payloads).unwrap().len(), + // TracerPayloadCollection::V1(payloads) => { + // let metadata = crate::tracer_metadata::TracerMetadata::default(); + // msgpack_encoder::v1::to_encoded_byte_len(payloads, &metadata) as usize + // } } } diff --git a/libdd-trace-utils/src/trace_utils.rs b/libdd-trace-utils/src/trace_utils.rs index 851ac54beb..f0398235f6 100644 --- a/libdd-trace-utils/src/trace_utils.rs +++ b/libdd-trace-utils/src/trace_utils.rs @@ -7,7 +7,7 @@ use crate::span::v05::dict::SharedDict; use crate::span::{v05, TraceData}; pub use crate::tracer_header_tags::TracerHeaderTags; use crate::tracer_payload::TracerPayloadCollection; -use crate::tracer_payload::{self, TraceChunks, TraceEncoding}; +use crate::tracer_payload::{self, TraceChunks}; use anyhow::anyhow; use bytes::buf::Reader; use bytes::Buf; @@ -583,25 +583,28 @@ pub fn enrich_span_with_azure_function_metadata(span: &mut pb::Span) { } } -pub fn collect_trace_chunks( +/// Converts v0.4-shaped span chunks into the v0.5 wire representation. +/// +/// v0.5 deduplicates every string field across the whole payload through a shared dictionary +/// and replaces them with `u32` indices. This walks each span via [`v05::from_v04_span`], +/// interning strings into the [`SharedDict`] as it goes, and returns the resulting +/// `(dict, traces)` pair wrapped in [`TraceChunks::V05`]. +/// +/// Returns `Err` if any span fails to convert (e.g. unsupported field value); the partial +/// dictionary built so far is discarded. +pub fn convert_trace_chunks_v04_to_v05( traces: Vec>>, - format: TraceEncoding, ) -> anyhow::Result> { - match format { - TraceEncoding::V05 => { - let mut shared_dict = SharedDict::default(); - let mut v05_traces: Vec> = Vec::with_capacity(traces.len()); - for trace in traces { - let v05_trace = trace - .into_iter() - .map(|span| v05::from_v04_span(span, &mut shared_dict)) - .collect::>>()?; - v05_traces.push(v05_trace); - } - Ok(TraceChunks::V05((shared_dict, v05_traces))) - } - TraceEncoding::V04 => Ok(TraceChunks::V04(traces)), + let mut shared_dict = SharedDict::default(); + let mut v05_traces: Vec> = Vec::with_capacity(traces.len()); + for trace in traces { + let v05_trace = trace + .into_iter() + .map(|span| v05::from_v04_span(span, &mut shared_dict)) + .collect::>>()?; + v05_traces.push(v05_trace); } + Ok(TraceChunks::V05((shared_dict, v05_traces))) } pub fn collect_pb_trace_chunks( @@ -1118,10 +1121,10 @@ mod tests { } #[test] - fn test_collect_trace_chunks_v05() { + fn test_convert_trace_chunks_v04_to_v05() { let chunk = vec![create_test_no_alloc_span(123, 456, 789, 1, true)]; - let collection = collect_trace_chunks(vec![chunk], TraceEncoding::V05).unwrap(); + let collection = convert_trace_chunks_v04_to_v05(vec![chunk]).unwrap(); let (dict, traces) = match collection { TraceChunks::V05(payload) => payload, @@ -1192,27 +1195,6 @@ mod tests { ); } - #[test] - fn test_collect_trace_chunks_v04() { - let chunk = vec![create_test_no_alloc_span(123, 456, 789, 1, true)]; - - let collection = collect_trace_chunks(vec![chunk], TraceEncoding::V04).unwrap(); - - let traces = match collection { - TraceChunks::V04(traces) => traces, - _ => panic!("Unexpected type"), - }; - - assert_eq!(traces.len(), 1); - assert_eq!(traces[0].len(), 1); - let span = &traces[0][0]; - assert_eq!(span.trace_id, 123); - assert_eq!(span.span_id, 456); - assert_eq!(span.parent_id, 789); - assert_eq!(span.start, 1); - assert_eq!(span.error, 0); - } - #[test] fn test_rmp_serde_deserialize_meta_with_null_values() { // Create a JSON representation with null value in meta diff --git a/libdd-trace-utils/src/tracer_payload.rs b/libdd-trace-utils/src/tracer_payload.rs index 30752cabdf..2bd419ef10 100644 --- a/libdd-trace-utils/src/tracer_payload.rs +++ b/libdd-trace-utils/src/tracer_payload.rs @@ -3,8 +3,9 @@ use crate::span::v05::dict::SharedDict; use crate::span::{v04, v05, BytesData, SharedDictBytes, TraceData}; -use crate::trace_utils::collect_trace_chunks; +use crate::trace_utils::convert_trace_chunks_v04_to_v05; use crate::{msgpack_decoder, trace_utils::cmp_send_data_payloads}; +use anyhow::Ok; use libdd_trace_protobuf::pb; use std::cmp::Ordering; use std::iter::Iterator; @@ -19,6 +20,9 @@ pub enum TraceEncoding { V04, /// v0.5 encoding (TracerPayloadV05). V05, + /// V1 encoding. Input is decoded as v0.4 (same span shape) and re-encoded as V1 msgpack + /// when sent to the agent. + V1, } #[derive(Debug)] @@ -29,6 +33,7 @@ pub enum TraceChunks { V05((SharedDict, Vec>)), /// Collection of v0.4 spans to be serialized as a V1 msgpack payload. V1(Vec>>), + // V1(Vec>>), } impl TraceChunks { @@ -38,6 +43,7 @@ impl TraceChunks { TraceChunks::V05(traces) => TracerPayloadCollection::V05(traces), // V1 uses the same underlying span structure as V04. TraceChunks::V1(traces) => TracerPayloadCollection::V04(traces), + // TraceChunks::V1(traces) => TracerPayloadCollection::V1(traces), } } } @@ -62,6 +68,8 @@ pub enum TracerPayloadCollection { V04(Vec>), /// Collection of TraceChunkSpan with de-duplicated strings. V05((SharedDictBytes, Vec>)), + // /// V0.4-shaped spans that must be serialized as a V1 msgpack payload on send. + // V1(Vec>), } impl TracerPayloadCollection { @@ -92,6 +100,11 @@ impl TracerPayloadCollection { dest.append(src) } } + // TracerPayloadCollection::V1(dest) => { + // if let TracerPayloadCollection::V1(src) = other { + // dest.append(src) + // } + // } // TODO: Properly handle non-OK states to prevent possible panics (APMSP-18190). #[allow(clippy::unimplemented)] TracerPayloadCollection::V05(_) => unimplemented!("Append for V05 not implemented"), @@ -144,6 +157,7 @@ impl TracerPayloadCollection { } TracerPayloadCollection::V04(collection) => collection.len(), TracerPayloadCollection::V05((_, collection)) => collection.len(), + // TracerPayloadCollection::V1(collection) => collection.len(), } } } @@ -228,13 +242,16 @@ pub fn decode_to_trace_chunks( data: libdd_tinybytes::Bytes, encoding_type: TraceEncoding, ) -> Result<(TraceChunks, usize), anyhow::Error> { - let (data, size) = match encoding_type { - TraceEncoding::V04 => msgpack_decoder::v04::from_bytes(data), - TraceEncoding::V05 => msgpack_decoder::v05::from_bytes(data), + match encoding_type { + TraceEncoding::V04 | TraceEncoding::V1 => { + let (data, size) = msgpack_decoder::v04::from_bytes(data).map_err(|e| anyhow::format_err!("Error deserializing trace from request body: {e}"))?; + Ok((TraceChunks::V04(data), size)) + } + TraceEncoding::V05 => { + let (data, size) = msgpack_decoder::v05::from_bytes(data).map_err(|e| anyhow::format_err!("Error deserializing trace from request body: {e}"))?; + Ok((convert_trace_chunks_v04_to_v05(data)?, size)) + } } - .map_err(|e| anyhow::format_err!("Error deserializing trace from request body: {e}"))?; - - Ok((collect_trace_chunks(data, encoding_type)?, size)) } #[cfg(test)]