ref(allocator): Try to use jemalloc for etl-api and etl-replicator (#487)

Author: iambriccardo

Cargo.toml

@@ -71,7 +71,9 @@ serde_json = { version = "1.0.141", default-features = false }
 serde_yaml = { version = "0.9.34", default-features = false }
 sqlx = { version = "0.8.6", default-features = false }
 tempfile = { version = "3.23.0", default-features = false }
-thiserror = "2.0.12"
+thiserror = "2.0.12"    
+tikv-jemalloc-ctl = { version = "0.6.0", default-features = false, features = ["stats"] }
+tikv-jemallocator = { version = "0.6.1", default-features = false, features = ["background_threads_runtime_support", "unprefixed_malloc_on_supported_platforms"] }
 tokio = { version = "1.47.0", default-features = false }
 tokio-postgres = { git = "https://github.com/MaterializeInc/rust-postgres", default-features = false, rev = "c4b473b478b3adfbf8667d2fbe895d8423f1290b" }
 tokio-rustls = { version = "0.26.2", default-features = false }

etl-api/Dockerfile

@@ -20,7 +20,8 @@ RUN apt-get update && \
     pkg-config \
     libssl-dev \
     clang \
-    lld && \
+    lld \
+    make && \
     rm -rf /var/lib/apt/lists/*
 
 # Copy cargo config for build optimizations (lld linker, etc.)

etl-replicator/Cargo.toml

@@ -15,6 +15,7 @@ etl-telemetry = { workspace = true }
 
 anyhow = { workspace = true, features = ["std"] }
 configcat = { workspace = true }
+metrics = { workspace = true }
 reqwest = { workspace = true, features = ["rustls-tls", "json"] }
 secrecy = { workspace = true }
 sentry = { workspace = true }
@@ -28,3 +29,7 @@ sqlx = { workspace = true, features = [
 
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal"] }
 tracing = { workspace = true, default-features = true }
+
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+tikv-jemalloc-ctl = { workspace = true }
+tikv-jemallocator = { workspace = true }

etl-replicator/Dockerfile

@@ -15,14 +15,15 @@ ARG BUILDPLATFORM
 RUN echo "Native build on $BUILDPLATFORM for $TARGETPLATFORM"
 
 # Install build dependencies including lld linker for faster linking
-# TODO: remove protobuf-compiler once the upstream gcp-bigquery-client remove it from its deps
+# TODO: remove protobuf-compiler once the upstream gcp-bigquery-client removes it from its deps.
 RUN apt-get update && \
     apt-get install -y \
     pkg-config \
     libssl-dev \
     protobuf-compiler \
     clang \
-    lld && \
+    lld \
+    make && \
     rm -rf /var/lib/apt/lists/*
 
 # Copy cargo config for build optimizations (lld linker, etc.)
@@ -44,6 +45,10 @@ WORKDIR /app
 # Create non-root user (distroless already has nonroot user)
 USER nonroot:nonroot
 
+# jemalloc configuration is compiled into the binary (see main.rs).
+# Use MALLOC_CONF env var to override at runtime if needed.
+# Example: MALLOC_CONF="narenas:4,dirty_decay_ms:5000"
+
 # Copy binary; configuration must be provided at runtime via mounted volume or environment overrides
 COPY --from=builder /app/target/release/etl-replicator ./etl-replicator