feat: implement prometheus exporter (#406)

Co-authored-by: mickael <contact@mickael-caudrelier.fr>

Author: NicolasIRAGNE

.gitignore

@@ -10,6 +10,7 @@ Thumbs.db
 
 # Python virtual-envs & tooling
 .venv*/
+venv/
 .python-version
 __pycache__/
 *.egg-info/
@@ -38,3 +39,6 @@ tmp/
 # Project-specific files
 history.txt
 digest.txt
+
+# Environment variables
+.env

.pre-commit-config.yaml

@@ -117,6 +117,7 @@ repos:
  'fastapi[standard]>=0.109.1',
  httpx,
  pathspec>=0.12.1,
+ prometheus-client,
  pydantic,
  pytest-asyncio,
  pytest-mock,
@@ -138,6 +139,7 @@ repos:
  'fastapi[standard]>=0.109.1',
  httpx,
  pathspec>=0.12.1,
+ prometheus-client,
  pydantic,
  pytest-asyncio,
  pytest-mock,

Dockerfile

@@ -43,4 +43,5 @@ RUN set -eux; \
 USER appuser
 
 EXPOSE 8000
+EXPOSE 9090
 CMD ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000"]

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
  "tiktoken>=0.7.0", # Support for o200k_base encoding
  "typing_extensions>= 4.0.0; python_version < '3.10'",
  "uvicorn>=0.11.7", # Minimum safe release (https://osv.dev/vulnerability/PYSEC-2020-150)
+ "prometheus-client",
 ]
 
 license = {file = "LICENSE"}

requirements.txt

@@ -2,6 +2,7 @@ click>=8.0.0
 fastapi[standard]>=0.109.1 # Vulnerable to https://osv.dev/vulnerability/PYSEC-2024-38
 httpx
 pathspec>=0.12.1
+prometheus-client
 pydantic
 python-dotenv
 slowapi

src/server/main.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import threading
 from pathlib import Path
 
 from dotenv import load_dotenv
@@ -12,6 +13,7 @@
 from slowapi.errors import RateLimitExceeded
 from starlette.middleware.trustedhost import TrustedHostMiddleware
 
+from server.metrics_server import start_metrics_server
 from server.routers import dynamic, index, ingest
 from server.server_config import templates
 from server.server_utils import lifespan, limiter, rate_limit_exception_handler
@@ -26,6 +28,17 @@
 # Register the custom exception handler for rate limits
 app.add_exception_handler(RateLimitExceeded, rate_limit_exception_handler)
 
+# Start metrics server in a separate thread if enabled
+if os.getenv("GITINGEST_METRICS_ENABLED", "false").lower() == "true":
+ metrics_host = os.getenv("GITINGEST_METRICS_HOST", "127.0.0.1")
+ metrics_port = int(os.getenv("GITINGEST_METRICS_PORT", "9090"))
+ metrics_thread = threading.Thread(
+ target=start_metrics_server,
+ args=(metrics_host, metrics_port),
+ daemon=True,
+ )
+ metrics_thread.start()
+
 
 # Mount static files dynamically to serve CSS, JS, and other static assets
 static_dir = Path(__file__).parent.parent / "static"

src/server/metrics_server.py

@@ -0,0 +1,57 @@
+"""Prometheus metrics server running on a separate port."""
+
+import logging
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+from prometheus_client import REGISTRY, generate_latest
+
+# Create a logger for this module
+logger = logging.getLogger(__name__)
+
+# Create a separate FastAPI app for metrics
+metrics_app = FastAPI(
+ title="Gitingest Metrics",
+ description="Prometheus metrics for Gitingest",
+ docs_url=None,
+ redoc_url=None,
+)
+
+
+@metrics_app.get("/metrics")
+async def metrics() -> HTMLResponse:
+ """Serve Prometheus metrics without authentication.
+
+ This endpoint is only accessible from the local network.
+
+ Returns
+ -------
+ HTMLResponse
+ Prometheus metrics in text format
+
+ """
+ return HTMLResponse(
+ content=generate_latest(REGISTRY),
+ status_code=200,
+ media_type="text/plain",
+ )
+
+
+def start_metrics_server(host: str = "127.0.0.1", port: int = 9090) -> None:
+ """Start the metrics server on a separate port.
+
+ Parameters
+ ----------
+ host : str
+ The host to bind to (default: 127.0.0.1 for local network only)
+ port : int
+ The port to bind to (default: 9090)
+
+ Returns
+ -------
+ None
+
+ """
+ logger.info("Starting metrics server on %s:%s", host, port)
+ uvicorn.run(metrics_app, host=host, port=port)

src/server/routers/ingest.py

@@ -2,13 +2,16 @@
 
 from fastapi import APIRouter, HTTPException, Request, status
 from fastapi.responses import FileResponse, JSONResponse
+from prometheus_client import Counter
 
 from gitingest.config import TMP_BASE_PATH
 from server.models import IngestRequest
 from server.routers_utils import COMMON_INGEST_RESPONSES, _perform_ingestion
 from server.server_config import MAX_DISPLAY_SIZE
 from server.server_utils import limiter
 
+ingest_counter = Counter("gitingest_ingest_total", "Number of ingests", ["status", "url"])
+
 router = APIRouter()
 
 
@@ -33,13 +36,16 @@ async def api_ingest(
  - **JSONResponse**: Success response with ingestion results or error response with appropriate HTTP status code
 
  """
- return await _perform_ingestion(
+ response = await _perform_ingestion(
  input_text=ingest_request.input_text,
  max_file_size=ingest_request.max_file_size,
  pattern_type=ingest_request.pattern_type,
  pattern=ingest_request.pattern,
  token=ingest_request.token,
  )
+ # limit URL to 255 characters
+ ingest_counter.labels(status=response.status_code, url=ingest_request.input_text[:255]).inc()
+ return response
 
 
 @router.get("/api/{user}/{repository}", responses=COMMON_INGEST_RESPONSES)
@@ -72,13 +78,16 @@ async def api_ingest_get(
  **Returns**
  - **JSONResponse**: Success response with ingestion results or error response with appropriate HTTP status code
  """
- return await _perform_ingestion(
+ response = await _perform_ingestion(
  input_text=f"{user}/{repository}",
  max_file_size=max_file_size,
  pattern_type=pattern_type,
  pattern=pattern,
  token=token or None,
  )
+ # limit URL to 255 characters
+ ingest_counter.labels(status=response.status_code, url=f"{user}/{repository}"[:255]).inc()
+ return response
 
 
 @router.get("/api/download/file/{ingest_id}", response_class=FileResponse)

src/server/server_config.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+from pathlib import Path
+
 from fastapi.templating import Jinja2Templates
 
 MAX_DISPLAY_SIZE: int = 300_000
@@ -19,4 +21,7 @@
  {"name": "ApiAnalytics", "url": "https://github.com/tom-draper/api-analytics"},
 ]
 
-templates = Jinja2Templates(directory="server/templates")
+
+# Use absolute path to templates directory
+templates_dir = Path(__file__).parent / "templates"
+templates = Jinja2Templates(directory=templates_dir)