feat(evaluations): Set JSON as standard for output to pass to judge

packages/client/src/launchdarkly_ai_server/evaluations/runner.py

@@ -12,6 +12,7 @@
 from datetime import UTC, datetime
 from typing import Any, Literal, cast
 
+from ..content import tool_result_string
 from ..judge_scoring import (
     FORMATTING_INSTRUCTIONS,
     build_message_history,
@@ -822,7 +823,7 @@ def _judge_variables(
         variables.update(
             {
                 "input": row_result.get("input") or "",
-                "response_to_evaluate": output if output is not None else "",
+                "response_to_evaluate": tool_result_string(output),
                 "message_history": build_message_history(
                     user_input=row_result.get("input"),
                     trajectory=trajectory,
@@ -949,7 +950,7 @@ async def _run_ld_judge_for_result(
         try:
             result = await execution.handler(
                 dict(resolved.config),
-                row.get("output"),
+                variables["response_to_evaluate"],
                 {},
                 {
                     **variables,

packages/client/src/launchdarkly_ai_server/judge_scoring.py

@@ -15,6 +15,7 @@
 from math import isfinite
 from typing import Any
 
+from .content import tool_result_string
 from .utils import parse_json_with_possible_fences
 
 FORMATTING_INSTRUCTIONS = "\n".join(
@@ -45,10 +46,14 @@ def build_message_history(
     the AI Library's default templates read the JSON shape from
     ``{{message_history}}``, and one that stopped being told it would return
     prose and fail every result as invalid output.
+
+    A non-string output is shown as JSON, by the rule tool results follow, so a
+    rubric reads the same text in every SDK.
     """
+    output_text = tool_result_string(output) if output else None
     return "\n\n".join(
         str(part)
-        for part in (user_input, trajectory, output, FORMATTING_INSTRUCTIONS)
+        for part in (user_input, trajectory, output_text, FORMATTING_INSTRUCTIONS)
         if part
     )
 

packages/client/tests/test_evaluations_run.py

@@ -2266,6 +2266,61 @@ async def handler(
     assert result.passed is True
 
 
+@pytest.mark.asyncio
+async def test_structured_output_reaches_the_judge_as_json(
+    monkeypatch: pytest.MonkeyPatch,
+    stub_sdk_client: MagicMock,
+) -> None:
+    transport = SequencedTransport(
+        [
+            response(200, {"id": "dataset-id", "name": "golden"}),
+            response(
+                200,
+                dataset_page([{"rowIndex": 0, "input": "Question"}], total=1),
+            ),
+            response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}),
+            response(
+                201,
+                {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"},
+            ),
+            response(
+                200,
+                {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}},
+            ),
+        ]
+    )
+    accuracy_judge_variation(monkeypatch)
+    evals = init_evaluations(
+        project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport
+    )
+    judged: dict[str, str] = {}
+
+    async def handler(
+        config: dict[str, Any],
+        user_input: str | None,
+        tool_handlers: dict[str, Callable[..., Any]],
+        variables: dict[str, Any],
+    ) -> dict[str, Any]:
+        if "Judge" in prompt_text(config):
+            assert user_input == variables["response_to_evaluate"]
+            judged["response"] = variables["response_to_evaluate"]
+            judged["history"] = variables["message_history"]
+            return {"output": '{"score": 1, "reasoning": "ok"}'}
+        return {"output": {"answer": "original", "tags": ["a"], "final": True}}
+
+    await evals.run(
+        key="support-qa",
+        dataset="golden",
+        handler=tagged(handler),
+        generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"},
+        criteria=[Judge(key="$ld:ai:judge:accuracy")],
+    )
+
+    rendered = '{"answer": "original", "tags": ["a"], "final": true}'
+    assert judged["response"] == rendered
+    assert rendered in judged["history"]
+
+
 @pytest.mark.asyncio
 async def test_duplicate_criteria_rejected_before_any_request() -> None:
     transport = SequencedTransport([])

packages/client/tests/test_judge_scoring.py

@@ -5,6 +5,8 @@
 import pytest
 
 from launchdarkly_ai_server.judge_scoring import (
+    FORMATTING_INSTRUCTIONS,
+    build_message_history,
     parse_judge_response,
     typesafe_judge_entries,
 )
@@ -81,3 +83,19 @@ def test_typesafe_entries_keep_label_order() -> None:
             "reason": "",
         },
     ]
+
+
+def test_message_history_renders_a_structured_output_as_json() -> None:
+    history = build_message_history(
+        user_input="q", output={"answer": "oui", "ok": True, "note": None}
+    )
+    assert history == "\n\n".join(
+        ["q", '{"answer": "oui", "ok": true, "note": null}', FORMATTING_INSTRUCTIONS]
+    )
+
+
+def test_message_history_keeps_non_ascii_and_skips_an_empty_output() -> None:
+    assert '{"city": "Zürich"}' in build_message_history(output={"city": "Zürich"})
+    assert build_message_history(user_input="q", output={}) == "\n\n".join(
+        ["q", FORMATTING_INSTRUCTIONS]
+    )