Added Adaboost algorithm

machine_learning/adaboost.py

@@ -1,17 +1,37 @@
+"""
+AdaBoost implementation for binary classification using decision stumps.
+
+Reference: https://en.wikipedia.org/wiki/AdaBoost
+
+>>> import numpy as np
+>>> X = np.array([[0, 0], [1, 1], [1, 0], [0, 1]])
+>>> y = np.array([0, 1, 1, 0])
+>>> clf = AdaBoost(n_estimators=5)
+>>> clf.fit(X, y)
+>>> clf.predict(np.array([[0, 0], [1, 1]]))
+array([0, 1])
+"""
 
 import numpy as np
+from typing import Any, Dict, List
+
 
-# AdaBoost implementation for binary classification
-# Uses decision stumps (one-level trees) as weak learners
 class AdaBoost:
- def __init__(self, n_estimators=50):
- # Number of boosting rounds
- self.n_estimators = n_estimators
- self.alphas = [] # Weights for each weak learner
- self.models = [] # List of weak learners (stumps)
+ def __init__(self, n_estimators: int = 50) -> None:
+ """Initialize AdaBoost classifier.
+ Args:
+ n_estimators: Number of boosting rounds.
+ """
+ self.n_estimators: int = n_estimators
+ self.alphas: List[float] = [] # Weights for each weak learner
+ self.models: List[Dict[str, Any]] = [] # List of weak learners (stumps)
 
- def fit(self, X, y):
- # X: (n_samples, n_features), y: (n_samples,) with labels 0 or 1
+ def fit(self, X: np.ndarray, y: np.ndarray) -> None:
+ """Fit AdaBoost model.
+ Args:
+ X: (n_samples, n_features) feature matrix
+ y: (n_samples,) labels (0 or 1)
+ """
  n_samples, n_features = X.shape
  w = np.ones(n_samples) / n_samples # Initialize sample weights
  self.models = []
@@ -20,8 +40,8 @@ def fit(self, X, y):
  for _ in range(self.n_estimators):
  # Train a decision stump with weighted samples
  stump = self._build_stump(X, y_, w)
- pred = stump['pred']
- err = stump['error']
+ pred = stump["pred"]
+ err = stump["error"]
  # Compute alpha (learner weight)
  alpha = 0.5 * np.log((1 - err) / (err + 1e-10))
  # Update sample weights
@@ -30,20 +50,35 @@ def fit(self, X, y):
  self.models.append(stump)
  self.alphas.append(alpha)
 
- def predict(self, X):
- # Aggregate predictions from all weak learners
+ def predict(self, X: np.ndarray) -> np.ndarray:
+ """Predict class labels for samples in X.
+ Args:
+ X: (n_samples, n_features) feature matrix
+ Returns:
+ (n_samples,) predicted labels (0 or 1)
+ >>> import numpy as np
+ >>> X = np.array([[0, 0], [1, 1], [1, 0], [0, 1]])
+ >>> y = np.array([0, 1, 1, 0])
+ >>> clf = AdaBoost(n_estimators=5)
+ >>> clf.fit(X, y)
+ >>> clf.predict(np.array([[0, 0], [1, 1]]))
+ array([0, 1])
+ """
  clf_preds = np.zeros(X.shape[0])
  for alpha, stump in zip(self.alphas, self.models):
- pred = self._stump_predict(X, stump['feature'], stump['threshold'], stump['polarity'])
+ pred = self._stump_predict(
+ X, stump["feature"], stump["threshold"], stump["polarity"]
+ )
  clf_preds += alpha * pred
- # Return final prediction (majority vote)
  return np.where(clf_preds >= 0, 1, 0)
 
- def _build_stump(self, X, y, w):
- # Find the best decision stump for current weights
+ def _build_stump(
+ self, X: np.ndarray, y: np.ndarray, w: np.ndarray
+ ) -> Dict[str, Any]:
+ """Find the best decision stump for current weights."""
  n_samples, n_features = X.shape
- min_error = float('inf')
- best_stump = {}
+ min_error = float("inf")
+ best_stump: Dict[str, Any] = {}
  for feature in range(n_features):
  thresholds = np.unique(X[:, feature])
  for threshold in thresholds:
@@ -53,16 +88,18 @@ def _build_stump(self, X, y, w):
  if error < min_error:
  min_error = error
  best_stump = {
- 'feature': feature,
- 'threshold': threshold,
- 'polarity': polarity,
- 'error': error,
- 'pred': pred.copy()
+ "feature": feature,
+ "threshold": threshold,
+ "polarity": polarity,
+ "error": error,
+ "pred": pred.copy(),
  }
  return best_stump
 
- def _stump_predict(self, X, feature, threshold, polarity):
- # Predict using a single decision stump
+ def _stump_predict(
+ self, X: np.ndarray, feature: int, threshold: float, polarity: int
+ ) -> np.ndarray:
+ """Predict using a single decision stump."""
  pred = np.ones(X.shape[0])
  if polarity == 1:
  pred[X[:, feature] < threshold] = -1