Added Adaboost algorithm

Author: ai-nerdy

machine_learning/adaboost.py

@@ -4,10 +4,10 @@
 Reference: https://en.wikipedia.org/wiki/AdaBoost
 
 >>> import numpy as np
->>> X = np.array([[0, 0], [1, 1], [1, 0], [0, 1]])
->>> y = np.array([0, 1, 1, 0])
+>>> features = np.array([[0, 0], [1, 1], [1, 0], [0, 1]])
+>>> labels = np.array([0, 1, 1, 0])
 >>> clf = AdaBoost(n_estimators=5)
->>> clf.fit(X, y)
+>>> clf.fit(features, labels)
 >>> clf.predict(np.array([[0, 0], [1, 1]]))
 array([0, 1])
 """
@@ -26,65 +26,65 @@ def __init__(self, n_estimators: int = 50) -> None:
  self.alphas: List[float] = [] # Weights for each weak learner
  self.models: List[Dict[str, Any]] = [] # List of weak learners (stumps)
 
- def fit(self, X: np.ndarray, y: np.ndarray) -> None:
+ def fit(self, feature_matrix: np.ndarray, target: np.ndarray) -> None:
  """Fit AdaBoost model.
  Args:
- X: (n_samples, n_features) feature matrix
- y: (n_samples,) labels (0 or 1)
+ feature_matrix: (n_samples, n_features) feature matrix
+ target: (n_samples,) labels (0 or 1)
  """
- n_samples, n_features = X.shape
- w = np.ones(n_samples) / n_samples # Initialize sample weights
+ n_samples, n_features = feature_matrix.shape
+ sample_weights = np.ones(n_samples) / n_samples # Initialize sample weights
  self.models = []
  self.alphas = []
- y_ = np.where(y == 0, -1, 1) # Convert labels to -1, 1
+ y_signed = np.where(target == 0, -1, 1) # Convert labels to -1, 1
  for _ in range(self.n_estimators):
  # Train a decision stump with weighted samples
- stump = self._build_stump(X, y_, w)
+ stump = self._build_stump(feature_matrix, y_signed, sample_weights)
  pred = stump["pred"]
  err = stump["error"]
  # Compute alpha (learner weight)
  alpha = 0.5 * np.log((1 - err) / (err + 1e-10))
  # Update sample weights
- w *= np.exp(-alpha * y_ * pred)
- w /= np.sum(w)
+ sample_weights *= np.exp(-alpha * y_signed * pred)
+ sample_weights /= np.sum(sample_weights)
  self.models.append(stump)
  self.alphas.append(alpha)
 
- def predict(self, X: np.ndarray) -> np.ndarray:
- """Predict class labels for samples in X.
+ def predict(self, feature_matrix: np.ndarray) -> np.ndarray:
+ """Predict class labels for samples in feature_matrix.
  Args:
- X: (n_samples, n_features) feature matrix
+ feature_matrix: (n_samples, n_features) feature matrix
  Returns:
  (n_samples,) predicted labels (0 or 1)
  >>> import numpy as np
- >>> X = np.array([[0, 0], [1, 1], [1, 0], [0, 1]])
- >>> y = np.array([0, 1, 1, 0])
+ >>> features = np.array([[0, 0], [1, 1], [1, 0], [0, 1]])
+ >>> labels = np.array([0, 1, 1, 0])
  >>> clf = AdaBoost(n_estimators=5)
- >>> clf.fit(X, y)
+ >>> clf.fit(features, labels)
  >>> clf.predict(np.array([[0, 0], [1, 1]]))
  array([0, 1])
  """
- clf_preds = np.zeros(X.shape[0])
+ clf_preds = np.zeros(feature_matrix.shape[0])
  for alpha, stump in zip(self.alphas, self.models):
  pred = self._stump_predict(
- X, stump["feature"], stump["threshold"], stump["polarity"]
+ feature_matrix, stump["feature"], stump["threshold"], stump["polarity"]
  )
  clf_preds += alpha * pred
  return np.where(clf_preds >= 0, 1, 0)
 
  def _build_stump(
- self, X: np.ndarray, y: np.ndarray, w: np.ndarray
+ self, feature_matrix: np.ndarray, target_signed: np.ndarray, sample_weights: np.ndarray
  ) -> Dict[str, Any]:
  """Find the best decision stump for current weights."""
- n_samples, n_features = X.shape
+ n_samples, n_features = feature_matrix.shape
  min_error = float("inf")
  best_stump: Dict[str, Any] = {}
  for feature in range(n_features):
- thresholds = np.unique(X[:, feature])
+ thresholds = np.unique(feature_matrix[:, feature])
  for threshold in thresholds:
  for polarity in [1, -1]:
- pred = self._stump_predict(X, feature, threshold, polarity)
- error = np.sum(w * (pred != y))
+ pred = self._stump_predict(feature_matrix, feature, threshold, polarity)
+ error = np.sum(sample_weights * (pred != target_signed))
  if error < min_error:
  min_error = error
  best_stump = {
@@ -97,12 +97,12 @@ def _build_stump(
  return best_stump
 
  def _stump_predict(
- self, X: np.ndarray, feature: int, threshold: float, polarity: int
+ self, feature_matrix: np.ndarray, feature: int, threshold: float, polarity: int
  ) -> np.ndarray:
  """Predict using a single decision stump."""
- pred = np.ones(X.shape[0])
+ pred = np.ones(feature_matrix.shape[0])
  if polarity == 1:
- pred[X[:, feature] < threshold] = -1
+ pred[feature_matrix[:, feature] < threshold] = -1
  else:
- pred[X[:, feature] > threshold] = -1
+ pred[feature_matrix[:, feature] > threshold] = -1
  return pred