Support passing prompt_label to langfuse (#11018)

* fix: add prompt label support to prompt management hook * feat: support 'prompt_label' parameter for langfuse prompt management Closes #9003 (reply in thread) * fix(litellm_logging.py): deep copy optional params to avoid mutation while logging * fix(log-consistent-optional-param-values-across-providers): ensures params can be used for finetuning from providers * fix: fix linting error * test: update test * test: update langfuse tests * fix(litellm_logging.py): avoid deepcopying optional params might contain thread object

Author: krrishdholakia

litellm/integrations/anthropic_cache_control_hook.py

@@ -28,6 +28,7 @@ def get_chat_completion_prompt(
  prompt_id: Optional[str],
  prompt_variables: Optional[dict],
  dynamic_callback_params: StandardCallbackDynamicParams,
+ prompt_label: Optional[str] = None,
  ) -> Tuple[str, List[AllMessageValues], dict]:
  """
  Apply cache control directives based on specified injection points.
@@ -79,10 +80,10 @@ def _process_message_injection(
  # Case 1: Target by specific index
  if targetted_index is not None:
  if 0 <= targetted_index < len(messages):
- messages[targetted_index] = (
- AnthropicCacheControlHook._safe_insert_cache_control_in_message(
-  messages[targetted_index], control
- )
+ messages[
+ targetted_index
+ ] = AnthropicCacheControlHook._safe_insert_cache_control_in_message(
+ messages[targetted_index], control
  )
  # Case 2: Target by role
  elif targetted_role is not None:

litellm/integrations/custom_logger.py

@@ -87,6 +87,7 @@ async def async_get_chat_completion_prompt(
  dynamic_callback_params: StandardCallbackDynamicParams,
  litellm_logging_obj: LiteLLMLoggingObj,
  tools: Optional[List[Dict]] = None,
+ prompt_label: Optional[str] = None,
  ) -> Tuple[str, List[AllMessageValues], dict]:
  """
  Returns:
@@ -104,6 +105,7 @@ def get_chat_completion_prompt(
  prompt_id: Optional[str],
  prompt_variables: Optional[dict],
  dynamic_callback_params: StandardCallbackDynamicParams,
+ prompt_label: Optional[str] = None,
  ) -> Tuple[str, List[AllMessageValues], dict]:
  """
  Returns:

litellm/integrations/custom_prompt_management.py

@@ -18,6 +18,7 @@ def get_chat_completion_prompt(
  prompt_id: Optional[str],
  prompt_variables: Optional[dict],
  dynamic_callback_params: StandardCallbackDynamicParams,
+ prompt_label: Optional[str] = None,
  ) -> Tuple[str, List[AllMessageValues], dict]:
  """
  Returns:
@@ -43,6 +44,7 @@ def _compile_prompt_helper(
  prompt_id: str,
  prompt_variables: Optional[dict],
  dynamic_callback_params: StandardCallbackDynamicParams,
+ prompt_label: Optional[str] = None,
  ) -> PromptManagementClient:
  raise NotImplementedError(
  "Custom prompt management does not support compile prompt helper"

litellm/integrations/humanloop.py

@@ -155,11 +155,8 @@ def get_chat_completion_prompt(
  prompt_id: Optional[str],
  prompt_variables: Optional[dict],
  dynamic_callback_params: StandardCallbackDynamicParams,
- ) -> Tuple[
- str,
- List[AllMessageValues],
- dict,
- ]:
+ prompt_label: Optional[str] = None,
+ ) -> Tuple[str, List[AllMessageValues], dict,]:
  humanloop_api_key = dynamic_callback_params.get(
  "humanloop_api_key"
  ) or get_secret_str("HUMANLOOP_API_KEY")

litellm/integrations/langfuse/langfuse_prompt_management.py

@@ -130,9 +130,12 @@ def integration_name(self):
  return "langfuse"
 
  def _get_prompt_from_id(
- self, langfuse_prompt_id: str, langfuse_client: LangfuseClass
+ self,
+ langfuse_prompt_id: str,
+ langfuse_client: LangfuseClass,
+ prompt_label: Optional[str] = None,
  ) -> PROMPT_CLIENT:
- return langfuse_client.get_prompt(langfuse_prompt_id)
+ return langfuse_client.get_prompt(langfuse_prompt_id, label=prompt_label)
 
  def _compile_prompt(
  self,
@@ -176,18 +179,16 @@ async def async_get_chat_completion_prompt(
  dynamic_callback_params: StandardCallbackDynamicParams,
  litellm_logging_obj: LiteLLMLoggingObj,
  tools: Optional[List[Dict]] = None,
- ) -> Tuple[
- str,
- List[AllMessageValues],
- dict,
- ]:
+ prompt_label: Optional[str] = None,
+ ) -> Tuple[str, List[AllMessageValues], dict,]:
  return self.get_chat_completion_prompt(
  model,
  messages,
  non_default_params,
  prompt_id,
  prompt_variables,
  dynamic_callback_params,
+ prompt_label=prompt_label,
  )
 
  def should_run_prompt_management(
@@ -211,6 +212,7 @@ def _compile_prompt_helper(
  prompt_id: str,
  prompt_variables: Optional[dict],
  dynamic_callback_params: StandardCallbackDynamicParams,
+ prompt_label: Optional[str] = None,
  ) -> PromptManagementClient:
  langfuse_client = langfuse_client_init(
  langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"),
@@ -219,7 +221,9 @@ def _compile_prompt_helper(
  langfuse_host=dynamic_callback_params.get("langfuse_host"),
  )
  langfuse_prompt_client = self._get_prompt_from_id(
- langfuse_prompt_id=prompt_id, langfuse_client=langfuse_client
+ langfuse_prompt_id=prompt_id,
+ langfuse_client=langfuse_client,
+ prompt_label=prompt_label,
  )
 
  ## SET PROMPT

litellm/integrations/prompt_management_base.py

@@ -33,6 +33,7 @@ def _compile_prompt_helper(
  prompt_id: str,
  prompt_variables: Optional[dict],
  dynamic_callback_params: StandardCallbackDynamicParams,
+ prompt_label: Optional[str] = None,
  ) -> PromptManagementClient:
  pass
 
@@ -49,11 +50,13 @@ def compile_prompt(
  prompt_variables: Optional[dict],
  client_messages: List[AllMessageValues],
  dynamic_callback_params: StandardCallbackDynamicParams,
+ prompt_label: Optional[str] = None,
  ) -> PromptManagementClient:
  compiled_prompt_client = self._compile_prompt_helper(
  prompt_id=prompt_id,
  prompt_variables=prompt_variables,
  dynamic_callback_params=dynamic_callback_params,
+ prompt_label=prompt_label,
  )
 
  try:
@@ -82,6 +85,7 @@ def get_chat_completion_prompt(
  prompt_id: Optional[str],
  prompt_variables: Optional[dict],
  dynamic_callback_params: StandardCallbackDynamicParams,
+ prompt_label: Optional[str] = None,
  ) -> Tuple[str, List[AllMessageValues], dict]:
  if prompt_id is None:
  raise ValueError("prompt_id is required for Prompt Management Base class")
@@ -95,6 +99,7 @@ def get_chat_completion_prompt(
  prompt_variables=prompt_variables,
  client_messages=messages,
  dynamic_callback_params=dynamic_callback_params,
+ prompt_label=prompt_label,
  )
 
  completed_messages = prompt_template["completed_messages"] or messages

litellm/integrations/vector_stores/bedrock_vector_store.py

@@ -75,6 +75,7 @@ async def async_get_chat_completion_prompt(
  dynamic_callback_params: StandardCallbackDynamicParams,
  litellm_logging_obj: LiteLLMLoggingObj,
  tools: Optional[List[Dict]] = None,
+ prompt_label: Optional[str] = None,
  ) -> Tuple[str, List[AllMessageValues], dict]:
  """
  Retrieves the context from the Bedrock Knowledge Base and appends it to the messages.
@@ -99,10 +100,11 @@ async def async_get_chat_completion_prompt(
  f"Bedrock Knowledge Base Response: {bedrock_kb_response}"
  )
 
- context_message, context_string = (
- self.get_chat_completion_message_from_bedrock_kb_response(
- bedrock_kb_response
- )
+ (
+ context_message,
+ context_string,
+ ) = self.get_chat_completion_message_from_bedrock_kb_response(
+ bedrock_kb_response
  )
  if context_message is not None:
  messages.append(context_message)
@@ -126,9 +128,9 @@ async def async_get_chat_completion_prompt(
  )
  )
 
- litellm_logging_obj.model_call_details["vector_store_request_metadata"] = (
- vector_store_request_metadata
- )
+ litellm_logging_obj.model_call_details[
+ "vector_store_request_metadata"
+ ] = vector_store_request_metadata
 
  return model, messages, non_default_params
 
@@ -140,9 +142,9 @@ def transform_bedrock_kb_response_to_vector_store_search_response(
  """
  Transform a BedrockKBResponse to a VectorStoreSearchResponse
  """
- retrieval_results: Optional[List[BedrockKBRetrievalResult]] = (
- bedrock_kb_response.get("retrievalResults", None)
- )
+ retrieval_results: Optional[
+ List[BedrockKBRetrievalResult]
+ ] = bedrock_kb_response.get("retrievalResults", None)
  vector_store_search_response: VectorStoreSearchResponse = (
  VectorStoreSearchResponse(search_query=query, data=[])
  )

litellm/litellm_core_utils/litellm_logging.py

@@ -539,6 +539,7 @@ def get_chat_completion_prompt(
  prompt_id: Optional[str],
  prompt_variables: Optional[dict],
  prompt_management_logger: Optional[CustomLogger] = None,
+ prompt_label: Optional[str] = None,
  ) -> Tuple[str, List[AllMessageValues], dict]:
  custom_logger = (
  prompt_management_logger
@@ -559,6 +560,7 @@ def get_chat_completion_prompt(
  prompt_id=prompt_id,
  prompt_variables=prompt_variables,
  dynamic_callback_params=self.standard_callback_dynamic_params,
+ prompt_label=prompt_label,
  )
  self.messages = messages
  return model, messages, non_default_params
@@ -572,6 +574,7 @@ async def async_get_chat_completion_prompt(
  prompt_variables: Optional[dict],
  prompt_management_logger: Optional[CustomLogger] = None,
  tools: Optional[List[Dict]] = None,
+ prompt_label: Optional[str] = None,
  ) -> Tuple[str, List[AllMessageValues], dict]:
  custom_logger = (
  prompt_management_logger
@@ -594,6 +597,7 @@ async def async_get_chat_completion_prompt(
  dynamic_callback_params=self.standard_callback_dynamic_params,
  litellm_logging_obj=self,
  tools=tools,
+ prompt_label=prompt_label,
  )
  self.messages = messages
  return model, messages, non_default_params

litellm/main.py

@@ -97,6 +97,7 @@
  get_optional_params_image_gen,
  get_optional_params_transcription,
  get_secret,
+ get_standard_openai_params,
  mock_completion_streaming_obj,
  read_config_args,
  supports_httpx_timeout,
@@ -428,6 +429,7 @@ async def acompletion(
  prompt_id=kwargs.get("prompt_id", None),
  prompt_variables=kwargs.get("prompt_variables", None),
  tools=tools,
+ prompt_label=kwargs.get("prompt_label", None),
  )
 
  #########################################################
@@ -983,6 +985,7 @@ def completion( # type: ignore # noqa: PLR0915
  assistant_continue_message=assistant_continue_message,
  )
  ######## end of unpacking kwargs ###########
+ standard_openai_params = get_standard_openai_params(params=args)
  non_default_params = get_non_default_completion_params(kwargs=kwargs)
  litellm_params = {} # used to prevent unbound var errors
  ## PROMPT MANAGEMENT HOOKS ##
@@ -1001,6 +1004,7 @@ def completion( # type: ignore # noqa: PLR0915
  non_default_params=non_default_params,
  prompt_id=prompt_id,
  prompt_variables=prompt_variables,
+ prompt_label=kwargs.get("prompt_label", None),
  )
 
  try:
@@ -1234,10 +1238,13 @@ def completion( # type: ignore # noqa: PLR0915
  max_retries=max_retries,
  timeout=timeout,
  )
- logging.update_environment_variables(
+ cast(LiteLLMLoggingObj, logging).update_environment_variables(
  model=model,
  user=user,
- optional_params=optional_params,
+ optional_params={
+ **standard_openai_params,
+ **non_default_params,
+ }, # [IMPORTANT] - using standard_openai_params ensures consistent params logged to langfuse for finetuning / eval datasets.
  litellm_params=litellm_params,
  custom_llm_provider=custom_llm_provider,
  )

litellm/proxy/_new_secret_config.yaml

@@ -1,8 +1,8 @@
 model_list:
- - model_name: "gemini-2.0-flash"
+ - model_name: "gemini-2.0-flash-gemini"
  litellm_params:
- model: gemini/gemini-2.0-flash-live-001
- - model_name: "gpt-4.1-openai"
+ model: gemini/gemini-2.0-flash
+ - model_name: "gpt-4o-mini-openai"
  litellm_params:
  model: gpt-4.1-mini-2025-04-14
  api_key: os.environ/OPENAI_API_KEY
@@ -71,6 +71,16 @@ model_list:
  model: mistral/*
  api_key: os.environ/MISTRAL_API_KEY
  access_groups: ["beta-models"]
+ - model_name: my-langfuse-model
+ litellm_params:
+ model: langfuse/gpt-3.5-turbo
+ prompt_id: "jokes"
+ prompt_label: "latest"
+ api_key: os.environ/OPENAI_API_KEY
 
 litellm_settings:
- cache: true
+ callbacks: ["langfuse"]
+
+general_settings:
+ store_model_in_db: true
+ store_prompts_in_spend_logs: true