chore(llmobs): make expected output optional (#14208)

- fix case where updating a record with expected_output to no
expected_output (null)
- update now only updates the specified fields and checks that it is a
valid update with valid fields
- extend timeouts for experiments calls to overcome read timeout
operations

pulling dataset with [empty
"expected_column"](https://app.datadoghq.com/llm/datasets/3926711e-10a7-4f4e-b29f-6de4ed1faca8?page=1)
fails before
before
<img width="1610" height="196" alt="image"
src="https://github.com/user-attachments/assets/c7d0f4d1-02ac-44d5-b86d-0804ca086f9f"
/>
after
<img width="1030" height="410" alt="image"
src="https://github.com/user-attachments/assets/6a188c8a-bd34-4aec-8d27-f57ea3e7377f"
/>


creating dataset
before
<img width="1574" height="237" alt="image"
src="https://github.com/user-attachments/assets/232df808-bbc0-494a-adc7-6fb6ee37022c"
/>
after
<img width="919" height="677" alt="image"
src="https://github.com/user-attachments/assets/3e13461a-ecad-4d52-b9f5-1435e197f961"
/>


append, update, and delete would result in errors on push
<img width="1571" height="180" alt="image"
src="https://github.com/user-attachments/assets/902e5640-82bd-4475-8659-76558c02a273"
/>

All those cases succeed after this change


A [large dataset
pull](https://app.datadoghq.com/llm/datasets/3926711e-10a7-4f4e-b29f-6de4ed1faca8?page=1)
has been tested to work with the extended timeout, we will bump the
timeout as needed, and possibly make it configurable in the future


## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

---------

Co-authored-by: Yun Kim <[email protected]>

Author: gary-huang

ddtrace/llmobs/_experiment.py

@@ -141,6 +141,11 @@ def push(self) -> None:
         self._updated_record_ids = []
 
     def update(self, index: int, record: DatasetRecordRaw) -> None:
+        if all(k not in record for k in ("input_data", "expected_output", "metadata")):
+            raise ValueError(
+                "invalid update, record should contain at least one of "
+                "input_data, expected_output, or metadata to update"
+            )
         record_id = self._records[index]["record_id"]
         self._updated_record_ids.append(record_id)
         self._records[index] = {**record, "record_id": record_id}

ddtrace/llmobs/_llmobs.py

@@ -611,7 +611,9 @@ def pull_dataset(cls, name: str) -> Dataset:
         return ds
 
     @classmethod
-    def create_dataset(cls, name: str, description: str, records: List[DatasetRecord] = []) -> Dataset:
+    def create_dataset(cls, name: str, description: str, records: Optional[List[DatasetRecord]] = None) -> Dataset:
+        if records is None:
+            records = []
         ds = cls._instance._dne_client.dataset_create(name, description)
         for r in records:
             ds.append(r)
@@ -625,11 +627,15 @@ def create_dataset_from_csv(
         csv_path: str,
         dataset_name: str,
         input_data_columns: List[str],
-        expected_output_columns: List[str],
-        metadata_columns: List[str] = [],
+        expected_output_columns: Optional[List[str]] = None,
+        metadata_columns: Optional[List[str]] = None,
         csv_delimiter: str = ",",
         description="",
     ) -> Dataset:
+        if expected_output_columns is None:
+            expected_output_columns = []
+        if metadata_columns is None:
+            metadata_columns = []
         ds = cls._instance._dne_client.dataset_create(dataset_name, description)
 
         # Store the original field size limit to restore it later

ddtrace/llmobs/_writer.py

@@ -297,6 +297,7 @@ class LLMObsExperimentsClient(BaseLLMObsWriter):
     EVP_SUBDOMAIN_HEADER_VALUE = EXP_SUBDOMAIN_NAME
     AGENTLESS_BASE_URL = AGENTLESS_EXP_BASE_URL
     ENDPOINT = ""
+    TIMEOUT = 5.0
 
     def request(self, method: str, path: str, body: JSONType = None) -> Response:
         headers = {
@@ -308,7 +309,7 @@ def request(self, method: str, path: str, body: JSONType = None) -> Response:
             headers[EVP_SUBDOMAIN_HEADER_NAME] = self.EVP_SUBDOMAIN_HEADER_VALUE
 
         encoded_body = json.dumps(body).encode("utf-8") if body else b""
-        conn = get_connection(self._intake)
+        conn = get_connection(url=self._intake, timeout=self.TIMEOUT)
         try:
             url = self._intake + self._endpoint + path
             logger.debug("requesting %s", url)
@@ -353,30 +354,40 @@ def dataset_create(self, name: str, description: str) -> Dataset:
         curr_version = response_data["data"]["attributes"]["current_version"]
         return Dataset(name, dataset_id, [], description, curr_version, _dne_client=self)
 
+    @staticmethod
+    def _get_record_json(record: Union[DatasetRecord, DatasetRecordRaw], is_update: bool) -> JSONType:
+        # for now, if a user wants to "erase" the value of expected_output, they are expected to
+        # set expected_output to None, and we serialize that as empty string to indicate this to BE
+        expected_output: JSONType = None
+        if "expected_output" in record:
+            expected_output = "" if record["expected_output"] is None else record["expected_output"]
+
+        # for now, if a user wants to "erase" the value of metadata, they are expected to
+        # set metadata to None, and we serialize that as an empty map to indicate this to BE
+        metadata: JSONType = None
+        if "metadata" in record:
+            metadata = {} if record["metadata"] is None else record["metadata"]
+
+        rj: JSONType = {
+            "input": cast(Dict[str, JSONType], record.get("input_data")),
+            "expected_output": expected_output,
+            "metadata": metadata,
+        }
+
+        if is_update:
+            rj["id"] = record["record_id"]  # type: ignore
+
+        return rj
+
     def dataset_batch_update(
         self,
         dataset_id: str,
         insert_records: List[DatasetRecordRaw],
         update_records: List[DatasetRecord],
         delete_record_ids: List[str],
     ) -> Tuple[int, List[str]]:
-        irs: JSONType = [
-            {
-                "input": cast(Dict[str, JSONType], r["input_data"]),
-                "expected_output": r["expected_output"],
-                "metadata": r.get("metadata", {}),
-            }
-            for r in insert_records
-        ]
-        urs: JSONType = [
-            {
-                "input": cast(Dict[str, JSONType], r["input_data"]),
-                "expected_output": r["expected_output"],
-                "metadata": r.get("metadata", {}),
-                "id": r["record_id"],
-            }
-            for r in update_records
-        ]
+        irs: JSONType = [self._get_record_json(r, False) for r in insert_records]
+        urs: JSONType = [self._get_record_json(r, True) for r in update_records]
         path = f"/api/unstable/llm-obs/v1/datasets/{dataset_id}/batch_update"
         body: JSONType = {
             "data": {
@@ -428,7 +439,7 @@ def dataset_get_with_records(self, name: str) -> Dataset:
                 {
                     "record_id": record["id"],
                     "input_data": attrs["input"],
-                    "expected_output": attrs["expected_output"],
+                    "expected_output": attrs.get("expected_output"),
                     "metadata": attrs.get("metadata", {}),
                 }
             )

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_2c079a59-06d7-4592-adcc-8f4124c0ea3b_batch_update_post_7d371dba.yaml

@@ -0,0 +1,47 @@
+interactions:
+- request:
+    body: '{"data": {"type": "datasets", "id": "2c079a59-06d7-4592-adcc-8f4124c0ea3b",
+      "attributes": {"insert_records": [], "update_records": [{"input": "A", "expected_output":
+      null, "metadata": null, "id": "371eb8d2-f5f3-44e6-a8bb-64169a656a7b"}], "delete_records":
+      []}}}'
+    headers:
+      Accept:
+      - '*/*'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Accept-Encoding
+      : - identity
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '261'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Content-Type
+      : - application/json
+      User-Agent:
+      - python-requests/2.32.3
+    method: POST
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/2c079a59-06d7-4592-adcc-8f4124c0ea3b/batch_update
+  response:
+    body:
+      string: '{"data":[{"id":"5d36809c-3741-45a7-8f32-67cf24d29e85","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-06T20:51:41.035546097Z","dataset_id":"2c079a59-06d7-4592-adcc-8f4124c0ea3b","expected_output":{"answer":"Paris"},"input":"A","metadata":{"difficulty":"easy"},"updated_at":"2025-08-06T20:51:41.035546171Z","version":2}}]}'
+    headers:
+      content-length:
+      - '382'
+      content-security-policy:
+      - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
+      content-type:
+      - application/vnd.api+json
+      date:
+      - Wed, 06 Aug 2025 20:51:41 GMT
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      vary:
+      - Accept-Encoding
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - SAMEORIGIN
+    status:
+      code: 200
+      message: OK
+version: 1

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_2c079a59-06d7-4592-adcc-8f4124c0ea3b_batch_update_post_f237e453.yaml

@@ -0,0 +1,48 @@
+interactions:
+- request:
+    body: '{"data": {"type": "datasets", "id": "2c079a59-06d7-4592-adcc-8f4124c0ea3b",
+      "attributes": {"insert_records": [{"input": {"prompt": "What is the capital
+      of France?"}, "expected_output": {"answer": "Paris"}, "metadata": {"difficulty":
+      "easy"}}], "update_records": [], "delete_records": []}}}'
+    headers:
+      Accept:
+      - '*/*'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Accept-Encoding
+      : - identity
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '289'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Content-Type
+      : - application/json
+      User-Agent:
+      - python-requests/2.32.3
+    method: POST
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/2c079a59-06d7-4592-adcc-8f4124c0ea3b/batch_update
+  response:
+    body:
+      string: '{"data":[{"id":"371eb8d2-f5f3-44e6-a8bb-64169a656a7b","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-06T20:51:38.394638367Z","dataset_id":"2c079a59-06d7-4592-adcc-8f4124c0ea3b","expected_output":{"answer":"Paris"},"input":{"prompt":"What
+        is the capital of France?"},"metadata":{"difficulty":"easy"},"updated_at":"2025-08-06T20:51:38.394638367Z","version":1}}]}'
+    headers:
+      content-length:
+      - '422'
+      content-security-policy:
+      - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
+      content-type:
+      - application/vnd.api+json
+      date:
+      - Wed, 06 Aug 2025 20:51:38 GMT
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      vary:
+      - Accept-Encoding
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - SAMEORIGIN
+    status:
+      code: 200
+      message: OK
+version: 1

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_2c079a59-06d7-4592-adcc-8f4124c0ea3b_records_get_906563da.yaml

@@ -0,0 +1,45 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Accept-Encoding
+      : - identity
+      Connection:
+      - keep-alive
+      ? !!python/object/apply:multidict._multidict.istr
+      - Content-Length
+      : - '0'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Content-Type
+      : - application/json
+      User-Agent:
+      - python-requests/2.32.3
+    method: GET
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/2c079a59-06d7-4592-adcc-8f4124c0ea3b/records
+  response:
+    body:
+      string: '{"data":[{"id":"5d36809c-3741-45a7-8f32-67cf24d29e85","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-06T20:51:41.035546Z","dataset_id":"2c079a59-06d7-4592-adcc-8f4124c0ea3b","expected_output":{"answer":"Paris"},"input":"A","metadata":{"difficulty":"easy"},"updated_at":"2025-08-06T20:51:41.035546Z"}}],"meta":{"after":""}}'
+    headers:
+      content-length:
+      - '384'
+      content-security-policy:
+      - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
+      content-type:
+      - application/vnd.api+json
+      date:
+      - Wed, 06 Aug 2025 20:51:51 GMT
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      vary:
+      - Accept-Encoding
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - SAMEORIGIN
+    status:
+      code: 200
+      message: OK
+version: 1

…eef395f8_batch_update_post_3195521f.yaml …95290c88_batch_update_post_9280615a.yamltests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_6b9d0a65-a8a0-47ef-9deb-6fdbeef395f8_batch_update_post_3195521f.yaml renamed to tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_2facf581-08b9-40c8-8377-28f895290c88_batch_update_post_9280615a.yaml tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_6b9d0a65-a8a0-47ef-9deb-6fdbeef395f8_batch_update_post_3195521f.yaml renamed to tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_2facf581-08b9-40c8-8377-28f895290c88_batch_update_post_9280615a.yaml

@@ -1,39 +1,39 @@
 interactions:
 - request:
-    body: '{"data": {"type": "datasets", "id": "6b9d0a65-a8a0-47ef-9deb-6fdbeef395f8",
+    body: '{"data": {"type": "datasets", "id": "2facf581-08b9-40c8-8377-28f895290c88",
       "attributes": {"insert_records": [{"input": {"prompt": "What is the capital
-      of France?"}, "expected_output": {"answer": "Paris"}, "metadata": {}}], "update_records":
+      of France?"}, "expected_output": {"answer": "Paris"}, "metadata": null}], "update_records":
       [], "delete_records": []}}}'
     headers:
       Accept:
       - '*/*'
-      ? !!python/object/new:multidict._multidict.istr
+      ? !!python/object/apply:multidict._multidict.istr
       - Accept-Encoding
       : - identity
       Connection:
       - keep-alive
       Content-Length:
-      - '269'
-      ? !!python/object/new:multidict._multidict.istr
+      - '271'
+      ? !!python/object/apply:multidict._multidict.istr
       - Content-Type
       : - application/json
       User-Agent:
       - python-requests/2.32.3
     method: POST
-    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/6b9d0a65-a8a0-47ef-9deb-6fdbeef395f8/batch_update
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/2facf581-08b9-40c8-8377-28f895290c88/batch_update
   response:
     body:
-      string: '{"data":[{"id":"d2d91cb9-c26a-45ec-84f1-68e6e2abc415","type":"datasets","attributes":{"author":{"id":"0dd9d379-c1a3-11ed-b4e0-566658a732f8"},"created_at":"2025-07-16T20:06:44.561248129Z","dataset_id":"6b9d0a65-a8a0-47ef-9deb-6fdbeef395f8","expected_output":{"answer":"Paris"},"input":{"prompt":"What
-        is the capital of France?"},"metadata":{},"updated_at":"2025-07-16T20:06:44.561248129Z","version":1}}]}'
+      string: '{"data":[{"id":"6eea94f8-d0d9-439b-813c-d6601c683828","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-06T20:51:29.335441534Z","dataset_id":"2facf581-08b9-40c8-8377-28f895290c88","expected_output":{"answer":"Paris"},"input":{"prompt":"What
+        is the capital of France?"},"updated_at":"2025-08-06T20:51:29.335441534Z","version":1}}]}'
     headers:
       content-length:
-      - '403'
+      - '389'
       content-security-policy:
       - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
       content-type:
       - application/vnd.api+json
       date:
-      - Wed, 16 Jul 2025 20:06:44 GMT
+      - Wed, 06 Aug 2025 20:51:29 GMT
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       vary:

…c38e8946_batch_update_post_0532546a.yaml …8e0eff66_batch_update_post_03f126d3.yamltests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_ce11ec07-4c99-49c8-a471-acddc38e8946_batch_update_post_0532546a.yaml renamed to tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_3249d56a-701e-45e1-a317-790c8e0eff66_batch_update_post_03f126d3.yaml tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_ce11ec07-4c99-49c8-a471-acddc38e8946_batch_update_post_0532546a.yaml renamed to tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_3249d56a-701e-45e1-a317-790c8e0eff66_batch_update_post_03f126d3.yaml

@@ -1,25 +1,25 @@
 interactions:
 - request:
-    body: '{"data": {"type": "datasets", "id": "ce11ec07-4c99-49c8-a471-acddc38e8946",
+    body: '{"data": {"type": "datasets", "id": "3249d56a-701e-45e1-a317-790c8e0eff66",
       "attributes": {"insert_records": [], "update_records": [], "delete_records":
-      ["1dd96d4f-e2ba-4e20-b36f-9fd03131d655"]}}}'
+      ["4b02d88b-1458-4b88-8322-3ec086113dd0"]}}}'
     headers:
       Accept:
       - '*/*'
-      ? !!python/object/new:multidict._multidict.istr
+      ? !!python/object/apply:multidict._multidict.istr
       - Accept-Encoding
       : - identity
       Connection:
       - keep-alive
       Content-Length:
       - '196'
-      ? !!python/object/new:multidict._multidict.istr
+      ? !!python/object/apply:multidict._multidict.istr
       - Content-Type
       : - application/json
       User-Agent:
       - python-requests/2.32.3
     method: POST
-    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/ce11ec07-4c99-49c8-a471-acddc38e8946/batch_update
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/3249d56a-701e-45e1-a317-790c8e0eff66/batch_update
   response:
     body:
       string: '{"data":[]}'
@@ -31,7 +31,7 @@ interactions:
       content-type:
       - application/vnd.api+json
       date:
-      - Thu, 17 Jul 2025 05:00:20 GMT
+      - Wed, 06 Aug 2025 20:52:13 GMT
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       vary: