diff --git a/src/agents/models/openai_chatcompletions.py b/src/agents/models/openai_chatcompletions.py index 83d2618262..3993a7a559 100644 --- a/src/agents/models/openai_chatcompletions.py +++ b/src/agents/models/openai_chatcompletions.py @@ -53,6 +53,11 @@ if TYPE_CHECKING: from ..model_settings import ModelSettings +_UNSUPPORTED_NONSTREAMING_CHOICES_MESSAGE = ( + "Chat Completions with multiple choices or nonzero choice indexes is not fully " + "supported; only one primary choice can be processed." +) + class OpenAIChatCompletionsModel(Model): _OFFICIAL_OPENAI_SUPPORTED_INPUT_CONTENT_TYPES = frozenset( @@ -75,6 +80,7 @@ def __init__( self._has_warned_unsupported_prompt = False self._has_warned_unsupported_conversation_state = False self._has_warned_unsupported_reasoning_settings = False + self._has_warned_unsupported_choice = False def _non_null_or_omit(self, value: Any) -> Any: return value if value is not None else omit @@ -129,6 +135,22 @@ def _handle_unsupported_reasoning_settings(self, model_settings: ModelSettings) ) self._has_warned_unsupported_reasoning_settings = True + def _handle_unsupported_choices(self, choices: list[Choice]) -> None: + unsupported_choice_indexes = [choice.index for choice in choices if choice.index != 0] + if len(choices) <= 1 and not unsupported_choice_indexes: + return + + if self._strict_feature_validation: + raise UserError(_UNSUPPORTED_NONSTREAMING_CHOICES_MESSAGE) + + if not self._has_warned_unsupported_choice: + logger.warning( + "%s Using the first returned choice; enable strict feature validation to " + "raise an error instead.", + _UNSUPPORTED_NONSTREAMING_CHOICES_MESSAGE, + ) + self._has_warned_unsupported_choice = True + def get_retry_advice(self, request: ModelRetryAdviceRequest) -> ModelRetryAdvice | None: return get_openai_retry_advice(request) @@ -312,6 +334,8 @@ async def get_response( "output_tokens_details": usage.output_tokens_details.model_dump(), } + self._handle_unsupported_choices(response.choices) + # Some providers signal a filtered non-streaming completion only through # finish_reason="content_filter" and an otherwise empty message. Preserve # that terminal signal as a refusal instead of returning an empty output. diff --git a/tests/models/test_openai_chatcompletions.py b/tests/models/test_openai_chatcompletions.py index c3d196b595..d5395b385f 100644 --- a/tests/models/test_openai_chatcompletions.py +++ b/tests/models/test_openai_chatcompletions.py @@ -24,6 +24,7 @@ TopLogprob, ) from openai.types.completion_usage import ( + CompletionTokensDetails, CompletionUsage, PromptTokensDetails, ) @@ -74,6 +75,23 @@ def _minimal_chat_completion(content: str = "ok") -> ChatCompletion: ) +def _chat_completion_with_choice_indexes(*indexes: int) -> ChatCompletion: + return ChatCompletion( + id="resp-id", + created=0, + model="fake", + object="chat.completion", + choices=[ + Choice( + index=index, + finish_reason="stop", + message=ChatCompletionMessage(role="assistant", content=f"choice-{index}"), + ) + for index in indexes + ], + ) + + async def _run_chat_completions_model_with_custom_base_url( model_settings: ModelSettings | dict[str, Any] | None = None, tools: list[Tool] | None = None, @@ -198,6 +216,181 @@ async def patched_fetch_response(self, *args, **kwargs): assert resp.raw_usage is None +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("choice_indexes", [(0, 1), (1,)]) +async def test_get_response_rejects_unsupported_choices_in_strict_mode( + monkeypatch: pytest.MonkeyPatch, + choice_indexes: tuple[int, ...], +) -> None: + async def patched_fetch_response(self, *args, **kwargs): + return _chat_completion_with_choice_indexes(*choice_indexes) + + monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response) + model = OpenAIProvider( + use_responses=False, + strict_feature_validation=True, + ).get_model("gpt-4") + + expected_message = ( + "Chat Completions with multiple choices or nonzero choice indexes is not fully " + "supported; only one primary choice can be processed." + ) + with pytest.raises(UserError) as exc_info: + await model.get_response( + system_instructions=None, + input="", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + assert str(exc_info.value) == expected_message + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_get_response_accepts_single_primary_choice_in_strict_mode( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + async def patched_fetch_response(self, *args, **kwargs): + return _chat_completion_with_choice_indexes(0) + + monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response) + model = OpenAIProvider( + use_responses=False, + strict_feature_validation=True, + ).get_model("gpt-4") + caplog.set_level(logging.WARNING, logger="openai.agents") + + response = await model.get_response( + system_instructions=None, + input="", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + + assert isinstance(response.output[0], ResponseOutputMessage) + assert isinstance(response.output[0].content[0], ResponseOutputText) + assert response.output[0].content[0].text == "choice-0" + assert not any( + "multiple choices or nonzero choice indexes" in record.getMessage() + for record in caplog.records + ) + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_get_response_traces_usage_before_rejecting_unsupported_choices( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + chat = _chat_completion_with_choice_indexes(0, 1) + chat.usage = CompletionUsage( + completion_tokens=3, + prompt_tokens=7, + total_tokens=10, + prompt_tokens_details=PromptTokensDetails(cached_tokens=2), + completion_tokens_details=CompletionTokensDetails(reasoning_tokens=1), + ) + + async def patched_fetch_response(self, *args, **kwargs): + return chat + + monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response) + model = OpenAIProvider( + use_responses=False, + strict_feature_validation=True, + ).get_model("gpt-4") + caplog.set_level(logging.DEBUG, logger="openai.agents") + + with trace(workflow_name="unsupported-choices-usage"): + with pytest.raises(UserError, match="multiple choices or nonzero choice indexes"): + await model.get_response( + system_instructions=None, + input="", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.ENABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + + generation = next(span for span in fetch_ordered_spans() if span.span_data.type == "generation") + assert generation.span_data.usage is not None + assert generation.span_data.usage["requests"] == 1 + assert generation.span_data.usage["input_tokens"] == 7 + assert generation.span_data.usage["output_tokens"] == 3 + assert generation.span_data.usage["total_tokens"] == 10 + assert generation.span_data.usage["input_tokens_details"]["cached_tokens"] == 2 + assert generation.span_data.usage["output_tokens_details"]["reasoning_tokens"] == 1 + assert generation.span_data.output is None + exported_span = generation.export() + assert exported_span is not None + assert exported_span["error"] is not None + assert any( + message == "Received model response" or "LLM resp:" in message + for message in (record.getMessage() for record in caplog.records) + ) + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("choice_indexes", [(0, 1), (1,)]) +async def test_get_response_warns_once_and_uses_first_choice( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + choice_indexes: tuple[int, ...], +) -> None: + async def patched_fetch_response(self, *args, **kwargs): + return _chat_completion_with_choice_indexes(*choice_indexes) + + monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response) + model = OpenAIProvider(use_responses=False).get_model("gpt-4") + caplog.set_level(logging.WARNING, logger="openai.agents") + + responses = [ + await model.get_response( + system_instructions=None, + input="", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + for _ in range(2) + ] + + for response in responses: + assert isinstance(response.output[0], ResponseOutputMessage) + assert isinstance(response.output[0].content[0], ResponseOutputText) + assert response.output[0].content[0].text == f"choice-{choice_indexes[0]}" + warnings = [ + record + for record in caplog.records + if "multiple choices or nonzero choice indexes" in record.getMessage() + ] + assert len(warnings) == 1 + + @pytest.mark.allow_call_model_methods @pytest.mark.asyncio @pytest.mark.parametrize(