feat: add AI module for LLM interaction and a heuristic for checking code–docstring consistency

pyproject.toml

@@ -39,6 +39,7 @@ dependencies = [
  "cryptography >=44.0.0,<45.0.0",
  "semgrep == 1.113.0",
  "email-validator >=2.2.0,<3.0.0",
+ "pydantic >= 2.11.5,<2.12.0",
 ]
 keywords = []
 # https://pypi.org/classifiers/

src/macaron/ai/README.md

@@ -0,0 +1,50 @@
+# Macaron AI Module
+
+This module provides the foundation for interacting with Large Language Models (LLMs) in a provider-agnostic way. It includes an abstract client definition, provider-specific client implementations, a client factory, and utility functions for processing responses.
+
+## Module Components
+
+- **ai_client.py**
+ Defines the abstract [`AIClient`](./clients/base.py) class. This class handles the initialization of LLM configuration from the defaults and serves as the base for all specific AI client implementations.
+
+- **openai_client.py**
+ Implements the [`OpenAiClient`](./clients/openai_client.py) class, a concrete subclass of [`AIClient`](./ai_client.py). This client interacts with OpenAI-like APIs by sending requests using HTTP and processing the responses. It also validates and structures responses using the tools provided.
+
+- **ai_factory.py**
+ Contains the [`AIClientFactory`](./clients/base.py) class, which is responsible for reading provider configuration from the defaults and creating the correct AI client instance.
+
+- **ai_tools.py**
+ Offers utility functions such as `structure_response` to assist with parsing and validating the JSON response returned by an LLM. These functions ensure that responses conform to a given Pydantic model for easier downstream processing.
+
+## Usage
+
+1. **Configuration:**
+ The module reads the LLM configuration from the application defaults (using the `defaults` module). Make sure that the `llm` section in your configuration includes valid settings such as `enabled`, `api_key`, `api_endpoint`, `model`, and `context_window`.
+
+2. **Creating a Client:**
+ Use the [`AIClientFactory`](./clients/ai_factory.py) to create an AI client instance. The factory checks the configured provider and returns a client (e.g., an instance of [`OpenAiClient`](./clients/openai_client.py)) that can be used to invoke the LLM.
+
+ Example:
+ ```py
+ from macaron.ai.clients.ai_factory import AIClientFactory
+
+ factory = AIClientFactory()
+ client = factory.create_client(system_prompt="You are a helpful assistant.")
+ response = client.invoke("Hello, how can you assist me?")
+ print(response)
+ ```
+
+3. **Response Processing:**
+ When a structured response is required, pass a Pydantic model class to the `invoke` method. The [`ai_tools.py`](./ai_tools.py) module takes care of parsing and validating the response to ensure it meets the expected structure.
+
+## Logging and Error Handling
+
+- The module uses Python's logging framework to report important events, such as token usage and warnings when prompts exceed the allowed context window.
+- Configuration errors (e.g., missing API key or endpoint) are handled by raising descriptive exceptions, such as those defined in the [`ConfigurationError`](../errors.py).
+
+## Extensibility
+
+The design of the AI module is provider-agnostic. To add support for additional LLM providers:
+- Implement a new client by subclassing [`AIClient`](./clients/base.py).
+- Add the new client to the [`PROVIDER_MAPPING`](./clients/ai_factory.py).
+- Update the configuration defaults accordingly.

src/macaron/ai/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

src/macaron/ai/ai_tools.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""This module provides utility functions for Large Language Model (LLM)."""
+import json
+import logging
+import re
+from typing import Any
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+def extract_json(response_text: str) -> Any:
+ """
+ Parse the response from the LLM.
+
+ If raw JSON parsing fails, attempts to extract a JSON object from text.
+
+ Parameters
+ ----------
+ response_text: str
+ The response text from the LLM.
+
+ Returns
+ -------
+ dict[str, Any] | None
+ The structured JSON object.
+ """
+ try:
+ data = json.loads(response_text)
+ except json.JSONDecodeError:
+ logger.debug("Full JSON parse failed; trying to extract JSON from text.")
+ # If the response is not a valid JSON, try to extract a JSON object from the text.
+ match = re.search(r"\{.*\}", response_text, re.DOTALL)
+ if not match:
+ return None
+ try:
+ data = json.loads(match.group(0))
+ except json.JSONDecodeError as e:
+ logger.debug("Failed to parse extracted JSON: %s", e)
+ return None
+
+ return data

src/macaron/ai/clients/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""This module provides a mapping of AI client providers to their respective client classes."""
+
+from macaron.ai.clients.base import AIClient
+from macaron.ai.clients.openai_client import OpenAiClient
+
+PROVIDER_MAPPING: dict[str, type[AIClient]] = {"openai": OpenAiClient}

src/macaron/ai/clients/ai_factory.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""This module defines the AIClientFactory class for creating AI clients based on provider configuration."""
+
+import logging
+
+from macaron.ai.clients import PROVIDER_MAPPING
+from macaron.ai.clients.base import AIClient
+from macaron.config.defaults import defaults
+from macaron.errors import ConfigurationError
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class AIClientFactory:
+ """Factory to create AI clients based on provider configuration."""
+
+ def __init__(self) -> None:
+ """
+ Initialize the AI client.
+
+ The LLM configuration is read from defaults.
+ """
+ self.params = self._load_defaults()
+
+ def _load_defaults(self) -> dict | None:
+ section_name = "llm"
+ default_values = {
+ "enabled": False,
+ "provider": "",
+ "api_key": "",
+ "api_endpoint": "",
+ "model": "",
+ }
+
+ if defaults.has_section(section_name):
+ section = defaults[section_name]
+ default_values["enabled"] = section.getboolean("enabled", default_values["enabled"])
+ for key, default_value in default_values.items():
+ if isinstance(default_value, str):
+ default_values[key] = str(section.get(key, default_value)).strip().lower()
+
+ if default_values["enabled"]:
+ for key, value in default_values.items():
+ if not value:
+ raise ConfigurationError(
+ f"AI client configuration '{key}' is required but not set in the defaults."
+ )
+
+ return default_values
+
+ def create_client(self, system_prompt: str) -> AIClient | None:
+ """Create an AI client based on the configured provider."""
+ if not self.params or not self.params["enabled"]:
+ return None
+
+ client_class = PROVIDER_MAPPING.get(self.params["provider"])
+ if client_class is None:
+ logger.error("Provider '%s' is not supported.", self.params["provider"])
+ return None
+ return client_class(system_prompt, self.params)

src/macaron/ai/clients/base.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""This module defines the abstract AIClient class for implementing AI clients."""
+
+from abc import ABC, abstractmethod
+
+
+class AIClient(ABC):
+ """This abstract class is used to implement ai clients."""
+
+ def __init__(self, system_prompt: str, params: dict) -> None:
+ """
+ Initialize the AI client.
+
+ The LLM configuration is read from defaults.
+ """
+ self.system_prompt = system_prompt
+ self.params = params
+
+ @abstractmethod
+ def invoke(
+ self,
+ user_prompt: str,
+ temperature: float = 0.2,
+ response_format: dict | None = None,
+ ) -> dict:
+ """
+ Invoke the LLM and optionally validate its response.
+
+ Parameters
+ ----------
+ user_prompt: str
+ The user prompt to send to the LLM.
+ temperature: float
+ The temperature for the LLM response.
+ response_format: dict | None
+ The json schema to validate the response against.
+
+ Returns
+ -------
+ dict
+ The validated schema if `response_format` is provided,
+ or the raw string response if not.
+ """

src/macaron/ai/clients/openai_client.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""This module provides a client for interacting with a Large Language Model (LLM) that is Openai like."""
+
+import logging
+from typing import Any, TypeVar
+
+from pydantic import BaseModel
+
+from macaron.ai.ai_tools import extract_json
+from macaron.ai.clients.base import AIClient
+from macaron.errors import ConfigurationError, HeuristicAnalyzerValueError
+from macaron.util import send_post_http_raw
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class OpenAiClient(AIClient):
+ """A client for interacting with a Large Language Model that is OpenAI API like."""
+
+ def invoke(
+ self,
+ user_prompt: str,
+ temperature: float = 0.2,
+ response_format: dict | None = None,
+ max_tokens: int = 4000,
+ seed: int = 42,
+ timeout: int = 30,
+ ) -> Any:
+ """
+ Invoke the LLM and optionally validate its response.
+
+ Parameters
+ ----------
+ user_prompt: str
+ The user prompt to send to the LLM.
+ temperature: float
+ The temperature for the LLM response.
+ response_format: dict
+ The json schema to validate the response against. If provided, the response will be parsed and validated.
+ max_tokens: int
+ The maximum number of tokens for the LLM response.
+ timeout: int
+ The timeout for the HTTP request in seconds.
+
+ Returns
+ -------
+ Optional[T | str]
+ The validated Pydantic model instance if `structured_output` is provided,
+ or the raw string response if not.
+
+ Raises
+ ------
+ HeuristicAnalyzerValueError
+ If there is an error in parsing or validating the response.
+ """
+ if not self.params["enabled"]:
+ raise ConfigurationError("AI client is not enabled. Please check your configuration.")
+
+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.params['api_key']}"}
+ payload = {
+ "model": self.params["model"],
+ "messages": [{"role": "system", "content": self.system_prompt}, {"role": "user", "content": user_prompt}],
+ "response_format": response_format,
+ "temperature": temperature,
+ "seed": seed,
+ "max_tokens": max_tokens,
+ }
+
+ try:
+ response = send_post_http_raw(
+ url=self.params["api_endpoint"], json_data=payload, headers=headers, timeout=timeout
+ )
+ if not response:
+ raise HeuristicAnalyzerValueError("No response received from the LLM.")
+ response_json = response.json()
+ usage = response_json.get("usage", {})
+
+ if usage:
+ usage_str = ", ".join(f"{key} = {value}" for key, value in usage.items())
+ logger.info("LLM call token usage: %s", usage_str)
+
+ message_content = response_json["choices"][0]["message"]["content"]
+ return extract_json(message_content)
+
+ except Exception as e:
+ logger.error("Error during LLM invocation: %s", e)
+ raise HeuristicAnalyzerValueError(f"Failed to get or validate LLM response: {e}") from e

src/macaron/ai/prompts/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

src/macaron/ai/schemas/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

src/macaron/config/defaults.ini

@@ -609,6 +609,9 @@ popular_packages_path =
 # A boolean value that determines whether to check the deliverability of the email address.
 check_deliverability = True
 
+# The threshold for a package's description score to be considered secure.
+score_threshold = 70
+
 # ==== The following sections are for source code analysis using Semgrep ====
 # rulesets: a reference to a 'ruleset' in this section refers to a Semgrep .yaml file containing one or more rules.
 # rules: a reference to a 'rule' in this section refers to an individual rule ID, specified by the '- id:' field in
@@ -632,3 +635,18 @@ custom_semgrep_rules_path =
 # .yaml prefix. Note, this will be ignored if a path to custom semgrep rules is not provided. This list may not contain
 # duplicated elements, meaning that ruleset names must be unique.
 disabled_custom_rulesets =
+
+[llm]
+# The LLM configuration for Macaron.
+# If enabled, the LLM will be used to analyze the results and provide insights.
+enabled = False
+# The provider for the LLM service.
+# Supported providers :
+# - openai: OpenAI's GPT models.
+provider =
+# The API key for the LLM service.
+api_key =
+# The API endpoint for the LLM service.
+api_endpoint =
+# The model to use for the LLM service.
+model =

src/macaron/malware_analyzer/pypi_heuristics/heuristics.py

@@ -49,6 +49,12 @@ class Heuristics(str, Enum):
  #: Indicates that the package has a similar structure to other packages maintained by the same user.
  SIMILAR_PROJECTS = "similar_projects"
 
+ #: Indicates that the package contains some code that doesn't match the docstrings.
+ MATCHING_DOCSTRINGS = "matching_docstrings"
+
+ #: Indicates that the package description is inconsistent.
+ INCONSISTENT_DESCRIPTION = "inconsistent_description"
+
 
 class HeuristicResult(str, Enum):
  """Result type indicating the outcome of a heuristic."""