diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index 37517a6ae..a9bab75a7 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -504,7 +504,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "ordered_hash_map", "proptest", - "prost 0.13.5", + "prost 0.14.3", "rand 0.8.5", "regex", "reqwest", @@ -640,6 +640,18 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-hex" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531185e432bb31db1ecda541e9e7ab21468d4d844ad7505e0546a49b4945d49b" +dependencies = [ + "cfg-if", + "cpufeatures", + "proptest", + "serde_core", +] + [[package]] name = "const_format" version = "0.2.35" @@ -744,10 +756,10 @@ source = "git+https://github.com/DataDog/saluki/?rev=f863626dbfe3c59bb390985fa65 dependencies = [ "bytes", "prost 0.13.5", - "prost-types", + "prost-types 0.13.5", "protobuf", "protobuf-codegen", - "tonic", + "tonic 0.13.1", "tonic-build", ] @@ -2117,9 +2129,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" dependencies = [ "futures-core", "futures-sink", @@ -2131,17 +2143,19 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "base64 0.22.1", - "hex", + "const-hex", "opentelemetry", "opentelemetry_sdk", - "prost 0.13.5", + "prost 0.14.3", "serde", - "tonic", + "serde_json", + "tonic 0.14.5", + "tonic-prost", ] [[package]] @@ -2152,9 +2166,9 @@ checksum = "83d059a296a47436748557a353c5e6c5705b9470ef6c95cfc52c21a8814ddac2" [[package]] name = "opentelemetry_sdk" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f644aa9e5e31d11896e024305d7e3c98a88884d9f8919dbf37a9991bc47a4b" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" dependencies = [ "futures-channel", "futures-executor", @@ -2162,7 +2176,6 @@ dependencies = [ "opentelemetry", "percent-encoding", "rand 0.9.2", - "serde_json", "thiserror 2.0.18", ] @@ -2464,7 +2477,7 @@ dependencies = [ "petgraph 0.7.1", "prettyplease", "prost 0.13.5", - "prost-types", + "prost-types 0.13.5", "regex", "syn 2.0.115", "tempfile", @@ -2505,6 +2518,15 @@ dependencies = [ "prost 0.13.5", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost 0.14.3", +] + [[package]] name = "protobuf" version = "3.7.2" @@ -3587,6 +3609,27 @@ name = "tonic" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project", + "prost 0.13.5", + "tokio-stream", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" dependencies = [ "async-trait", "base64 0.22.1", @@ -3599,7 +3642,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost 0.13.5", + "sync_wrapper", "tokio", "tokio-stream", "tower", @@ -3617,20 +3660,31 @@ dependencies = [ "prettyplease", "proc-macro2", "prost-build", - "prost-types", + "prost-types 0.13.5", "quote", "syn 2.0.115", ] +[[package]] +name = "tonic-prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +dependencies = [ + "bytes", + "prost 0.14.3", + "tonic 0.14.5", +] + [[package]] name = "tonic-types" -version = "0.13.1" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07439468da24d5f211d3f3bd7b63665d8f45072804457e838a87414a478e2db8" +checksum = "2a875a902255423d34c1f20838ab374126db8eb41625b7947a1d54113b0b7399" dependencies = [ - "prost 0.13.5", - "prost-types", - "tonic", + "prost 0.14.3", + "prost-types 0.14.3", + "tonic 0.14.5", ] [[package]] diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 8b3f0f94a..e6611236e 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -44,13 +44,13 @@ rustls-pemfile = { version = "2.0", default-features = false, features = ["std"] rustls-pki-types = { version = "1.0", default-features = false } hyper-rustls = { version = "0.27.7", default-features = false } rand = { version = "0.8", default-features = false } -prost = { version = "0.13", default-features = false } -tonic-types = { version = "0.13", default-features = false } +prost = { version = "0.14", default-features = false } +tonic-types = { version = "0.14", default-features = false } zstd = { version = "0.13.3", default-features = false } futures = { version = "0.3.31", default-features = false } serde-aux = { version = "4.7", default-features = false } serde_html_form = { version = "0.2", default-features = false } -opentelemetry-proto = { version = "0.30.0", features = ["trace", "with-serde", "gen-tonic"] } +opentelemetry-proto = { version = "0.31.0", features = ["trace", "with-serde", "gen-tonic"] } opentelemetry-semantic-conventions = { version = "0.30", features = ["semconv_experimental"] } # Pinned to <0.8.3: version 0.8.3 upgraded to openssl-probe 0.2.x which scans all cert # directories and parses ~200 individual cert files on Lambda instead of loading a single diff --git a/bottlecap/LICENSE-3rdparty.csv b/bottlecap/LICENSE-3rdparty.csv index 9c6ff4580..34d05c5e9 100644 --- a/bottlecap/LICENSE-3rdparty.csv +++ b/bottlecap/LICENSE-3rdparty.csv @@ -12,6 +12,8 @@ aws-lc-sys,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC) AND Open axum,https://github.com/tokio-rs/axum,MIT,The axum Authors axum-core,https://github.com/tokio-rs/axum,MIT,The axum-core Authors base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,Marshall Pierce +bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner +bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers buf_redux,https://github.com/abonander/buf_redux,MIT OR Apache-2.0,Austin Bonander @@ -22,6 +24,7 @@ bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Se cc,https://github.com/rust-lang/cc-rs,MIT OR Apache-2.0,Alex Crichton cfg-if,https://github.com/rust-lang/cfg-if,MIT OR Apache-2.0,Alex Crichton chrono,https://github.com/chronotope/chrono,MIT OR Apache-2.0,The chrono Authors +const-hex,https://github.com/danipopes/const-hex,MIT OR Apache-2.0,DaniPopes <57450786+DaniPopes@users.noreply.github.com> const_format,https://github.com/rodrimati1992/const_format_crates,Zlib,rodrimati1992 const_format_proc_macros,https://github.com/rodrimati1992/const_format_crates,Zlib,rodrimati1992 cookie,https://github.com/SergioBenitez/cookie-rs,MIT OR Apache-2.0,"Sergio Benitez , Alex Crichton " @@ -144,6 +147,7 @@ powerfmt,https://github.com/jhpratt/powerfmt,MIT OR Apache-2.0,Jacob Pratt , Alex Crichton " proc-macro2-diagnostics,https://github.com/SergioBenitez/proc-macro2-diagnostics,MIT OR Apache-2.0,Sergio Benitez +proptest,https://github.com/proptest-rs/proptest,MIT OR Apache-2.0,Jason Lingle prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " prost-derive,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " prost-types,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " @@ -158,6 +162,7 @@ r-efi,https://github.com/r-efi/r-efi,MIT OR Apache-2.0 OR LGPL-2.1-or-later,The rand,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_chacha,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers, The CryptoCorrosion Contributors" rand_core,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" +rand_xorshift,https://github.com/rust-random/rngs,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" redox_syscall,https://gitlab.redox-os.org/redox-os/syscall,MIT,Jeremy Soller regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-automata,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " @@ -174,6 +179,7 @@ rustls-native-certs,https://github.com/rustls/rustls-native-certs,Apache-2.0 OR rustls-pemfile,https://github.com/rustls/pemfile,Apache-2.0 OR ISC OR MIT,The rustls-pemfile Authors rustls-pki-types,https://github.com/rustls/pki-types,MIT OR Apache-2.0,The rustls-pki-types Authors rustls-webpki,https://github.com/rustls/webpki,ISC,The rustls-webpki Authors +rusty-fork,https://github.com/altsysrq/rusty-fork,MIT OR Apache-2.0,Jason Lingle ryu,https://github.com/dtolnay/ryu,Apache-2.0 OR BSL-1.0,David Tolnay safemem,https://github.com/abonander/safemem,MIT OR Apache-2.0,Austin Bonander schannel,https://github.com/steffengy/schannel-rs,MIT,"Steven Fackler , Steffen Butzer " @@ -224,6 +230,7 @@ tokio-rustls,https://github.com/rustls/tokio-rustls,MIT OR Apache-2.0,The tokio- tokio-stream,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-util,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco +tonic-prost,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-types,https://github.com/hyperium/tonic,MIT,"Lucio Franco , Rafael Lemos " tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers @@ -237,6 +244,7 @@ tracing-subscriber,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman twoway,https://github.com/bluss/twoway,MIT OR Apache-2.0,bluss typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,"Paho Lurie-Gregg , Andre Bogus " +unarray,https://github.com/cameron1024/unarray,MIT OR Apache-2.0,The unarray Authors uncased,https://github.com/SergioBenitez/uncased,MIT OR Apache-2.0,Sergio Benitez unicase,https://github.com/seanmonstar/unicase,MIT OR Apache-2.0,Sean McArthur unicode-ident,https://github.com/dtolnay/unicode-ident,(MIT OR Apache-2.0) AND Unicode-3.0,David Tolnay @@ -248,6 +256,7 @@ ustr,https://github.com/anderslanglands/ustr,BSD-2-Clause-Patent,Anders Langland utf8_iter,https://github.com/hsivonen/utf8_iter,Apache-2.0 OR MIT,Henri Sivonen valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors value-bag,https://github.com/sval-rs/value-bag,Apache-2.0 OR MIT,Ashley Mannix +wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton want,https://github.com/seanmonstar/want,MIT,Sean McArthur wasi,https://github.com/bytecodealliance/wasi,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers wasip2,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasip2 Authors diff --git a/bottlecap/src/traces/trace_processor.rs b/bottlecap/src/traces/trace_processor.rs index b71dc9d0e..05e151c65 100644 --- a/bottlecap/src/traces/trace_processor.rs +++ b/bottlecap/src/traces/trace_processor.rs @@ -379,7 +379,7 @@ impl TraceProcessor for ServerlessTraceProcessor { // stats are still counted. SamplerPriority::None (-128) means no explicit priority // was set and the trace is kept; drop priorities are SamplerPriority::AutoDrop (0) // and UserDrop (-1, not represented in SamplerPriority). - if config.compute_trace_stats_on_extension + let body_size = if config.compute_trace_stats_on_extension && let TracerPayloadCollection::V07(ref mut tracer_payloads) = payload { for tp in tracer_payloads.iter_mut() { @@ -391,7 +391,18 @@ impl TraceProcessor for ServerlessTraceProcessor { if tracer_payloads.is_empty() { return (None, payloads_for_stats); } - } + + // Update body_size after dropping sampled-out traces. + // Use the protobuf-encoded size of the filtered payload so the + // TraceAggregator's 3.2 MB batch limit reflects only the data that + // will actually be sent to the backend. + tracer_payloads + .iter() + .map(prost::Message::encoded_len) + .sum() + } else { + body_size + }; let owned_header_tags = OwnedTracerHeaderTags::from(header_tags.clone()); @@ -1251,4 +1262,111 @@ mod tests { "stats must include all traces even when all are sampled out" ); } + + /// Verifies that `body_size` in the returned `SendDataBuilderInfo` reflects the + /// protobuf-encoded size of the filtered payload, not the original request body. + #[test] + fn test_process_traces_body_size_reflects_filtered_payload() { + use libdd_trace_obfuscation::obfuscation_config::ObfuscationConfig; + use prost::Message as _; + + let config = Arc::new(Config { + apm_dd_url: "https://trace.agent.datadoghq.com".to_string(), + compute_trace_stats_on_extension: true, + ..Config::default() + }); + let tags_provider = Arc::new(Provider::new( + config.clone(), + "lambda".to_string(), + &std::collections::HashMap::from([( + "function_arn".to_string(), + "test-arn".to_string(), + )]), + )); + let processor = ServerlessTraceProcessor { + obfuscation_config: Arc::new( + ObfuscationConfig::new().expect("Failed to create ObfuscationConfig"), + ), + }; + let header_tags = tracer_header_tags::TracerHeaderTags { + lang: "rust", + lang_version: "1.0", + lang_interpreter: "", + lang_vendor: "", + tracer_version: "1.0", + container_id: "", + client_computed_top_level: false, + client_computed_stats: false, + dropped_p0_traces: 0, + dropped_p0_spans: 0, + }; + + let make_span = |trace_id: u64, priority: f64| -> pb::Span { + let mut metrics = HashMap::new(); + metrics.insert("_sampling_priority_v1".to_string(), priority); + pb::Span { + trace_id, + span_id: trace_id, + parent_id: 0, + metrics, + service: "svc".to_string(), + name: "op".to_string(), + resource: "res".to_string(), + ..Default::default() + } + }; + + // 1 kept trace, 3 dropped traces; original body_size is intentionally large + let traces = vec![ + vec![make_span(1, 1.0)], + vec![make_span(2, -1.0)], + vec![make_span(3, -1.0)], + vec![make_span(4, -1.0)], + ]; + + let (payload_info, stats_collection) = + processor.process_traces(config, tags_provider, header_tags, traces, 999_999, None); + + let info = payload_info.expect("expected Some payload"); + + // The reported size must equal the sum of encoded_len() of the kept TracerPayloads. + // stats_collection has all 4 traces. Reconstruct the filtered payload (only trace_id=1 + // was kept with priority=1) and compute its encoded_len. + let TracerPayloadCollection::V07(ref all_payloads) = stats_collection else { + panic!("expected V07"); + }; + let expected_size: usize = all_payloads + .iter() + .filter_map(|tp| { + let kept_chunks: Vec = tp + .chunks + .iter() + .filter(|c| c.spans.iter().any(|s| s.trace_id == 1)) + .cloned() + .collect(); + if kept_chunks.is_empty() { + None + } else { + Some(pb::TracerPayload { + chunks: kept_chunks, + ..tp.clone() + }) + } + }) + .map(|tp| tp.encoded_len()) + .sum(); + + assert!( + expected_size > 0, + "expected_size must be non-zero for a non-empty payload" + ); + assert_eq!( + info.size, expected_size, + "body_size must equal protobuf encoded_len of the filtered payload" + ); + assert!( + info.size < 999_999, + "body_size must be smaller than the original unfiltered request size" + ); + } }