feat(DI): add config to create probes through a file (#13747)

This pull request introduces support for loading probe definitions from
a file in JSON format, enabling dynamic instrumentation via external
configuration. The changes include updates to the
`DynamicInstrumentationConfig` class, the debugger's initialization
logic, and new tests to validate the functionality.

This can be configured via `DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE` and
used in combination with RC.

The JSON should be an array of probe objects in the same format as
received via the RC config object, for example:

```
[
        {
            "id": "12e4866b-c2d0-4948-baf8-bd98027cd457",
            "version": 0,
            "type": "LOG_PROBE",
            "language": "python",
            "where": {
                "sourceFile": "tests/submod/stuff.py",
                "lines": [36],
            },
            "tags": [],
            "template": "Hello new monitoring API",
            "captureSnapshot": True,
            "capture": {"maxReferenceDepth": 3},
            "evaluateAt": "EXIT",
        }
    ]
```


## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [ ] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Co-authored-by: Gabriele N. Tornetta <[email protected]>

Author: 2 people

ddtrace/debugging/_debugger.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from collections import deque
 from itertools import chain
+import json
 import linecache
 import os
 from pathlib import Path
@@ -37,6 +38,7 @@
 from ddtrace.debugging._probe.remoteconfig import ProbePollerEvent
 from ddtrace.debugging._probe.remoteconfig import ProbePollerEventType
 from ddtrace.debugging._probe.remoteconfig import ProbeRCAdapter
+from ddtrace.debugging._probe.remoteconfig import build_probe
 from ddtrace.debugging._probe.status import ProbeStatusLogger
 from ddtrace.debugging._signal.collector import SignalCollector
 from ddtrace.debugging._signal.model import Signal
@@ -270,6 +272,8 @@ def __init__(self, tracer: Optional[Tracer] = None) -> None:
             raise_on_exceed=False,
         )
 
+        self.probe_file = di_config.probe_file
+
         if di_config.enabled:
             # TODO: this is only temporary and will be reverted once the DD_REMOTE_CONFIGURATION_ENABLED variable
             #  has been removed
@@ -281,8 +285,28 @@ def __init__(self, tracer: Optional[Tracer] = None) -> None:
             di_callback = self.__rc_adapter__(None, self._on_configuration, status_logger=status_logger)
             remoteconfig_poller.register("LIVE_DEBUGGING", di_callback, restart_on_fork=True)
 
+            # Load local probes from the probe file.
+            self._load_local_config()
+
         log.debug("%s initialized (service name: %s)", self.__class__.__name__, service_name)
 
+    def _load_local_config(self) -> None:
+        if self.probe_file is None:
+            return
+
+        # This is intentionally an all or nothing approach. If one probe is malformed, none of the
+        # local probes will be installed, that way waiting for the success log guarantees installation.
+        try:
+            raw_probes = json.loads(self.probe_file.read_text())
+
+            probes = [build_probe(p) for p in raw_probes]
+
+            self._on_configuration(ProbePollerEvent.NEW_PROBES, probes)
+            log.info("Successfully loaded probes from file %s: %s", self.probe_file, [p.probe_id for p in probes])
+
+        except Exception as e:
+            log.error("Failed to load probes from file %s: %s", self.probe_file, e)
+
     def _dd_debugger_hook(self, probe: Probe) -> None:
         """Debugger probe hook.
 

ddtrace/settings/dynamic_instrumentation.py

@@ -3,6 +3,7 @@
 
 from ddtrace import config as ddconfig
 from ddtrace.internal import gitmetadata
+from ddtrace.internal.compat import Path
 from ddtrace.internal.constants import DEFAULT_SERVICE_NAME
 from ddtrace.internal.utils.config import get_application_name
 from ddtrace.settings._agent import config as agent_config
@@ -138,5 +139,13 @@ class DynamicInstrumentationConfig(DDConfig):
         help="List of identifiers to exclude from redaction",
     )
 
+    probe_file = DDConfig.v(
+        t.Optional[Path],
+        "probe_file",
+        default=None,
+        help_type="Path",
+        help="Path to a file containing probe definitions",
+    )
+
 
 config = DynamicInstrumentationConfig()

tests/debugging/test_debugger.py

@@ -1,7 +1,9 @@
 from collections import Counter
 from decimal import Decimal
+import json
 import os.path
 import sys
+import tempfile
 from threading import Thread
 
 import mock
@@ -24,6 +26,7 @@
 from ddtrace.debugging._signal.snapshot import _EMPTY_CAPTURED_CONTEXT
 from ddtrace.debugging._signal.tracing import SPAN_NAME
 from ddtrace.debugging._signal.utils import redacted_value
+from ddtrace.internal.compat import Path
 from ddtrace.internal.remoteconfig.worker import remoteconfig_poller
 from ddtrace.internal.service import ServiceStatus
 from ddtrace.internal.utils.formats import format_trace_id
@@ -1294,3 +1297,78 @@ def test_debugger_exception_conditional_function_probe():
     return_capture = snapshot_data["captures"]["return"]
     assert return_capture["throwable"]["message"] == "'Hello', 'world!', 42"
     assert return_capture["throwable"]["type"] == "Exception"
+
+
+def test_debugger_probe_file_configuration():
+    # Create sample probe configurations in the expected JSON format
+    probe_config = [
+        {
+            "id": "12e4866b-c2d0-4948-baf8-bd98027cd457",
+            "version": 0,
+            "type": "LOG_PROBE",
+            "language": "python",
+            "where": {
+                "sourceFile": "tests/submod/stuff.py",
+                "lines": [36],
+            },
+            "tags": [],
+            "template": "Hello new monitoring API",
+            "captureSnapshot": True,
+            "capture": {"maxReferenceDepth": 3},
+            "evaluateAt": "EXIT",
+        }
+    ]
+
+    # Create temporary probe file
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file:
+        json.dump(probe_config, temp_file)
+        # Capture the PATH to the file
+        temp_file_path = Path(temp_file.name)
+
+    try:
+        with debugger(probe_file=temp_file_path) as d:
+            from tests.submod.stuff import Stuff
+
+            Stuff().instancestuff()
+
+            assert len(d._probe_registry) == 1
+            assert d._probe_registry.get("12e4866b-c2d0-4948-baf8-bd98027cd457", False)
+    finally:
+        # Clean up temporary file
+        if os.path.exists(temp_file_path):
+            os.unlink(temp_file_path)
+
+
+def test_debugger_probe_malformed_file_configuration():
+    # Failing to parse the probe configuration will result in an error being logged.
+    malformed_probe_config = [
+        {
+            "version": 0,
+            "type": "LOG_PROBE",
+            "language": "python",
+            "where": {
+                "sourceFile": "tests/submod/stuff.py",
+                "lines": [36],
+            },
+            "tags": [],
+            "template": "Hello new monitoring API",
+            "captureSnapshot": True,
+            "capture": {"maxReferenceDepth": 3},
+            "evaluateAt": "EXIT",
+        }
+    ]
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file:
+        json.dump(malformed_probe_config, temp_file)
+        temp_file_path = temp_file.name
+
+    try:
+        with mock.patch("ddtrace.debugging._debugger.log") as mock_logger:
+            with debugger(probe_file=temp_file_path):
+                mock_logger.error.assert_called_once()
+                call_args = mock_logger.error.call_args[0]
+                assert "Failed to load probes from file" in call_args[0]
+                assert temp_file_path in call_args[1]
+    finally:
+        if os.path.exists(temp_file_path):
+            os.unlink(temp_file_path)

tests/telemetry/test_writer.py

@@ -366,6 +366,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python
         {"name": "DD_DYNAMIC_INSTRUMENTATION_ENABLED", "origin": "default", "value": False},
         {"name": "DD_DYNAMIC_INSTRUMENTATION_MAX_PAYLOAD_SIZE", "origin": "default", "value": 1048576},
         {"name": "DD_DYNAMIC_INSTRUMENTATION_METRICS_ENABLED", "origin": "default", "value": True},
+        {"name": "DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE", "origin": "default", "value": None},
         {"name": "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS", "origin": "default", "value": "set()"},
         {"name": "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES", "origin": "default", "value": "set()"},
         {"name": "DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS", "origin": "default", "value": "set()"},