Add logprobs to ModelSettings

src/agents/extensions/models/litellm_model.py

@@ -313,6 +313,7 @@ async def _fetch_response(
  stream=stream,
  stream_options=stream_options,
  reasoning_effort=reasoning_effort,
+ top_logprobs=model_settings.top_logprobs,
  extra_headers={**HEADERS, **(model_settings.extra_headers or {})},
  api_key=self.api_key,
  base_url=self.base_url,

src/agents/model_settings.py

@@ -55,6 +55,7 @@ class MCPToolChoice:
 ToolChoice: TypeAlias = Union[Literal["auto", "required", "none"], str, MCPToolChoice, None]
 
 
+
 @dataclass
 class ModelSettings:
  """Settings to use when calling an LLM.
@@ -116,6 +117,10 @@ class ModelSettings:
  """Additional output data to include in the model response.
  [include parameter](https://platform.openai.com/docs/api-reference/responses/create#responses-create-include)"""
 
+ top_logprobs: int | None = None
+ """Number of top tokens to return logprobs for. Setting this will
+ automatically include ``"message.output_text.logprobs"`` in the response."""
+
  extra_query: Query | None = None
  """Additional query fields to provide with the request.
  Defaults to None if not provided."""

src/agents/models/openai_chatcompletions.py

@@ -286,6 +286,7 @@ async def _fetch_response(
  stream_options=self._non_null_or_not_given(stream_options),
  store=self._non_null_or_not_given(store),
  reasoning_effort=self._non_null_or_not_given(reasoning_effort),
+ top_logprobs=self._non_null_or_not_given(model_settings.top_logprobs),
  extra_headers={**HEADERS, **(model_settings.extra_headers or {})},
  extra_query=model_settings.extra_query,
  extra_body=model_settings.extra_body,

src/agents/models/openai_responses.py

@@ -3,7 +3,7 @@
 import json
 from collections.abc import AsyncIterator
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Literal, overload
+from typing import TYPE_CHECKING, Any, Literal, cast, overload
 
 from openai import NOT_GIVEN, APIStatusError, AsyncOpenAI, AsyncStream, NotGiven
 from openai.types import ChatModel
@@ -247,9 +247,12 @@ async def _fetch_response(
  converted_tools = Converter.convert_tools(tools, handoffs)
  response_format = Converter.get_response_format(output_schema)
 
- include: list[ResponseIncludable] = converted_tools.includes
+ include_set: set[str] = set(converted_tools.includes)
  if model_settings.response_include is not None:
- include = list({*include, *model_settings.response_include})
+ include_set.update(model_settings.response_include)
+ if model_settings.top_logprobs is not None:
+ include_set.add("message.output_text.logprobs")
+ include = cast(list[ResponseIncludable], list(include_set))
 
  if _debug.DONT_LOG_MODEL_DATA:
  logger.debug("Calling LLM")
@@ -264,6 +267,10 @@ async def _fetch_response(
  f"Previous response id: {previous_response_id}\n"
  )
 
+ extra_args = dict(model_settings.extra_args or {})
+ if model_settings.top_logprobs is not None:
+ extra_args["top_logprobs"] = model_settings.top_logprobs
+
  return await self._client.responses.create(
  previous_response_id=self._non_null_or_not_given(previous_response_id),
  instructions=self._non_null_or_not_given(system_instructions),
@@ -286,7 +293,7 @@ async def _fetch_response(
  store=self._non_null_or_not_given(model_settings.store),
  reasoning=self._non_null_or_not_given(model_settings.reasoning),
  metadata=self._non_null_or_not_given(model_settings.metadata),
- **(model_settings.extra_args or {}),
+ **extra_args,
  )
 
  def _get_client(self) -> AsyncOpenAI:

tests/model_settings/test_serialization.py

@@ -58,6 +58,7 @@ def test_all_fields_serialization() -> None:
  store=False,
  include_usage=False,
  response_include=["reasoning.encrypted_content"],
+ top_logprobs=1,
  extra_query={"foo": "bar"},
  extra_body={"foo": "bar"},
  extra_headers={"foo": "bar"},
@@ -164,6 +165,7 @@ def test_pydantic_serialization() -> None:
  metadata={"foo": "bar"},
  store=False,
  include_usage=False,
+ top_logprobs=1,
  extra_query={"foo": "bar"},
  extra_body={"foo": "bar"},
  extra_headers={"foo": "bar"},

tests/test_logprobs.py

@@ -0,0 +1,50 @@
+import pytest
+from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
+
+from agents import ModelSettings, ModelTracing, OpenAIResponsesModel
+
+
+class DummyResponses:
+ async def create(self, **kwargs):
+ self.kwargs = kwargs
+
+ class DummyResponse:
+ id = "dummy"
+ output = []
+ usage = type(
+ "Usage",
+ (),
+ {
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "total_tokens": 0,
+ "input_tokens_details": InputTokensDetails(cached_tokens=0),
+ "output_tokens_details": OutputTokensDetails(reasoning_tokens=0),
+ },
+ )()
+
+ return DummyResponse()
+
+
+class DummyClient:
+ def __init__(self):
+ self.responses = DummyResponses()
+
+
+@pytest.mark.allow_call_model_methods
+@pytest.mark.asyncio
+async def test_top_logprobs_param_passed():
+ client = DummyClient()
+ model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore
+ await model.get_response(
+ system_instructions=None,
+ input="hi",
+ model_settings=ModelSettings(top_logprobs=2),
+ tools=[],
+ output_schema=None,
+ handoffs=[],
+ tracing=ModelTracing.DISABLED,
+ previous_response_id=None,
+ )
+ assert client.responses.kwargs["top_logprobs"] == 2
+ assert "message.output_text.logprobs" in client.responses.kwargs["include"]