Add AutoGen logger for generating report and integrating with OCI monitoring.

ads/llm/autogen/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/llm/autogen/constants.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class Events:
+ KEY = "event_name"
+
+ EXCEPTION = "exception"
+ LLM_CALL = "llm_call"
+ TOOL_CALL = "tool_call"
+ NEW_AGENT = "new_agent"
+ NEW_CLIENT = "new_client"
+ RECEIVED_MESSAGE = "received_message"
+ SESSION_START = "logging_session_start"
+ SESSION_STOP = "logging_session_stop"

ads/llm/autogen/reports/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/llm/autogen/reports/base.py

@@ -0,0 +1,67 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+import json
+import logging
+import os
+
+from jinja2 import Environment, FileSystemLoader
+
+logger = logging.getLogger(__name__)
+
+
+class BaseReport:
+ """Base class containing utilities for generating reports."""
+
+ @staticmethod
+ def format_json_string(s) -> str:
+ """Formats the JSON string in markdown."""
+ return f"```json\n{json.dumps(json.loads(s), indent=2)}\n```"
+
+ @staticmethod
+ def _parse_date_time(datetime_string: str):
+ """Parses a datetime string in the logs into date and time.
+ Keeps only the seconds in the time.
+ """
+ date_str, time_str = datetime_string.split(" ", 1)
+ time_str = time_str.split(".", 1)[0]
+ return date_str, time_str
+
+ @staticmethod
+ def _preview_message(message: str, max_length=30) -> str:
+ """Shows the beginning part of a string message."""
+ # Return the entire string if it is less than the max_length
+ if len(message) <= max_length:
+ return message
+ # Go backward until we find the first whitespace
+ idx = 30
+ while not message[idx].isspace() and idx > 0:
+ idx -= 1
+ # If we found a whitespace
+ if idx > 0:
+ return message[:idx] + "..."
+ # If we didn't find a whitespace
+ return message[:30] + "..."
+
+ @classmethod
+ def _render_template(cls, template_path, **kwargs) -> str:
+ """Render Jinja template with kwargs."""
+ template_dir = os.path.join(os.path.dirname(__file__), "templates")
+ environment = Environment(
+ loader=FileSystemLoader(template_dir), autoescape=True
+ )
+ template = environment.get_template(template_path)
+ try:
+ html = template.render(**kwargs)
+ except Exception:
+ logger.error(
+ "Unable to render template %s with data:\n%s",
+ template_path,
+ str(kwargs),
+ )
+ return cls._render_template(
+ template_path=template_path,
+ sender=kwargs.get("sender", "N/A"),
+ content="TEMPLATE RENDER ERROR",
+ timestamp=kwargs.get("timestamp", ""),
+ )
+ return html

ads/llm/autogen/reports/data.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+"""Contains the data structure for logging and reporting."""
+import copy
+import json
+from dataclasses import asdict, dataclass, field
+from typing import Optional, Union
+
+from ads.llm.autogen.constants import Events
+
+
+@dataclass
+class LogData:
+ """Base class for the data field of LogRecord."""
+
+ def to_dict(self):
+ """Convert the log data to dictionary."""
+ return asdict(self)
+
+
+@dataclass
+class LogRecord:
+ """Represents a log record.
+
+ The `data` field is for pre-defined structured data, which should be an instance of LogData.
+ The `kwargs` field is for freeform key value pairs.
+ """
+
+ session_id: str
+ thread_id: int
+ timestamp: str
+ event_name: str
+ source_id: Optional[int] = None
+ source_name: Optional[str] = None
+ # Structured data for specific type of logs
+ data: Optional[LogData] = None
+ # Freeform data
+ kwargs: dict = field(default_factory=dict)
+
+ def to_dict(self):
+ """Convert the log record to dictionary."""
+ return asdict(self)
+
+ def to_string(self):
+ """Serialize the log record to JSON string."""
+ return json.dumps(self.to_dict(), default=str)
+
+ @classmethod
+ def from_dict(cls, data: dict) -> "LogRecord":
+ """Initializes a LogRecord object from dictionary."""
+ event_mapping = {
+ Events.NEW_AGENT: AgentData,
+ Events.TOOL_CALL: ToolCallData,
+ Events.LLM_CALL: LLMCompletionData,
+ }
+ if Events.KEY not in data:
+ raise KeyError("event_name not found in data.")
+
+ data = copy.deepcopy(data)
+
+ event_name = data["event_name"]
+ if event_name in event_mapping and data.get("data"):
+ data["data"] = event_mapping[event_name](**data.pop("data"))
+
+ return cls(**data)
+
+
+@dataclass
+class AgentData(LogData):
+ """Represents agent log Data."""
+
+ agent_name: str
+ agent_class: str
+ agent_module: Optional[str] = None
+ is_manager: Optional[bool] = None
+
+
+@dataclass
+class LLMCompletionData(LogData):
+ """Represents LLM completion log data."""
+
+ invocation_id: str
+ request: dict
+ response: dict
+ start_time: str
+ end_time: str
+ cost: Optional[float] = None
+ is_cached: Optional[bool] = None
+
+
+@dataclass
+class ToolCallData(LogData):
+ """Represents tool call log data."""
+
+ tool_name: str
+ start_time: str
+ end_time: str
+ agent_name: str
+ agent_class: str
+ agent_module: Optional[str] = None
+ input_args: dict = field(default_factory=dict)
+ returns: Optional[Union[str, list, dict, tuple]] = None