Added ensemble method example code

Author: TannerGilbert

Ensemble_Methods/code/averaging.py

@@ -0,0 +1,24 @@
+# based on https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard
+# and https://www.kaggle.com/eikedehling/trying-out-stacking-approaches
+from sklearn.base import BaseEstimator, TransformerMixin, clone, RegressorMixin
+import numpy as np
+
+
+class AveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
+ def __init__(self, models):
+ self.models = models
+
+ def fit(self, X, y):
+ self.models_ = [clone(x) for x in self.models]
+
+ # Train cloned base models
+ for model in self.models_:
+ model.fit(X, y)
+
+ return self
+
+ def predict(self, X):
+ predictions = np.column_stack([
+ model.predict(X) for model in self.models_
+ ])
+ return np.mean(predictions, axis=1)

Ensemble_Methods/code/bagging.py

@@ -0,0 +1,48 @@
+from sklearn.base import BaseEstimator, TransformerMixin, clone, RegressorMixin
+import numpy as np
+
+
+class BaggingModels(BaseEstimator, RegressorMixin, TransformerMixin):
+ def __init__(self, models, task_type='classification'):
+ self.models = models
+ self.task_type = task_type
+
+ def fit(self, X, y):
+ self.models_ = [clone(x) for x in self.models]
+
+ for model in self.models_:
+ X_tmp, y_tmp = self.subsample(X, y)
+ model.fit(X_tmp, y_tmp)
+
+ return self
+
+ # Create a random subsample from the dataset with replacement
+ @staticmethod
+ def subsample(X, y, ratio=1.0):
+ X_new, y_new = list(), list()
+ n_sample = round(len(X) * ratio)
+ while len(X_new) < n_sample:
+ index = np.random.randint(len(X))
+ X_new.append(X[index])
+ y_new.append(y[index])
+ return X_new, y_new
+
+ def predict(self, X):
+ predictions_array = np.column_stack([
+ model.predict(X) for model in self.models_
+ ])
+ if self.task_type == 'classification':
+ return np.array([np.argmax(np.bincount(predictions)) for predictions in predictions_array])
+ else:
+ return np.mean(predictions_array, axis=1)
+
+ def predict_proba(self, X):
+ if self.task_type == 'classification':
+ predictions = []
+ for x in X:
+ prediction = np.row_stack([
+ model.predict_proba([x]) for model in self.models_
+ ])
+ predictions.append(np.mean(prediction, axis=0))
+ return np.array(predictions)
+ return None

Ensemble_Methods/code/blending.py

@@ -0,0 +1,48 @@
+from sklearn.base import BaseEstimator, TransformerMixin, clone, RegressorMixin
+from sklearn.model_selection import train_test_split
+import numpy as np
+
+
+class BlendingModels(BaseEstimator, RegressorMixin, TransformerMixin):
+ def __init__(self, base_models, meta_model, holdout_pct=0.2, use_features_in_secondary=False):
+ self.base_models = base_models
+ self.meta_model = meta_model
+ self.holdout_pct = holdout_pct
+ self.use_features_in_secondary = use_features_in_secondary
+
+ def fit(self, X, y):
+ """Fit all the models on the given dataset"""
+ self.base_models_ = [clone(x) for x in self.base_models]
+ self.meta_model_ = clone(self.meta_model)
+
+ X_train, X_holdout, y_train, y_holdout = train_test_split(X, y, test_size=self.holdout_pct)
+
+ holdout_predictions = np.zeros((X_holdout.shape[0], len(self.base_models)))
+ for i, model in enumerate(self.base_models_):
+ model.fit(X_train, y_train)
+ y_pred = model.predict(X_holdout)
+ holdout_predictions[:, i] = y_pred
+ if self.use_features_in_secondary:
+ self.meta_model_.fit(np.hstack((X_holdout, holdout_predictions)), y_holdout)
+ else:
+ self.meta_model_.fit(holdout_predictions, y_holdout)
+
+ return self
+
+ def predict(self, X):
+ meta_features = np.column_stack([
+ model.predict(X) for model in self.base_models_
+ ])
+ if self.use_features_in_secondary:
+ return self.meta_model_.predict(np.hstack((X, meta_features)))
+ else:
+ return self.meta_model_.predict(meta_features)
+
+ def predict_proba(self, X):
+ meta_features = np.column_stack([
+ model.predict(X) for model in self.base_models_
+ ])
+ if self.use_features_in_secondary:
+ return self.meta_model_.predict_proba(np.hstack((X, meta_features)))
+ else:
+ return self.meta_model_.predict_proba(meta_features)

Ensemble_Methods/code/majority_vote.py

@@ -0,0 +1,22 @@
+from sklearn.base import BaseEstimator, TransformerMixin, clone, ClassifierMixin
+import numpy as np
+
+
+class MajorityVote(BaseEstimator, ClassifierMixin, TransformerMixin):
+ def __init__(self, models):
+ self.models = models
+
+ def fit(self, X, y):
+ self.models_ = [clone(x) for x in self.models]
+
+ # Train cloned base models
+ for model in self.models_:
+ model.fit(X, y)
+
+ return self
+
+ def predict(self, X):
+ predictions_array = np.column_stack([
+ model.predict(X) for model in self.models_
+ ])
+ return np.array([np.argmax(np.bincount(predictions)) for predictions in predictions_array])

Ensemble_Methods/code/stacking.py

@@ -0,0 +1,61 @@
+# based on https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard
+# and https://www.kaggle.com/eikedehling/trying-out-stacking-approaches
+from sklearn.base import BaseEstimator, TransformerMixin, clone, RegressorMixin
+from sklearn.model_selection import KFold
+import numpy as np
+
+
+class StackingModels(BaseEstimator, RegressorMixin, TransformerMixin):
+ def __init__(self, base_models, meta_model, n_folds=5, task_type='classification', use_features_in_secondary=False):
+ self.base_models = base_models
+ self.meta_model = meta_model
+ self.n_folds = n_folds
+ self.task_type = task_type
+ self.use_features_in_secondary = use_features_in_secondary
+
+ def fit(self, X, y):
+ """Fit all the models on the given dataset"""
+ self.base_models_ = [list() for _ in self.base_models]
+ self.meta_model_ = clone(self.meta_model)
+ kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
+
+ # Train cloned base models and create out-of-fold predictions
+ out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))
+ for i, model in enumerate(self.base_models):
+ for train_index, holdout_index in kfold.split(X, y):
+ instance = clone(model)
+ self.base_models_[i].append(instance)
+ instance.fit(X[train_index], y[train_index])
+ y_pred = instance.predict(X[holdout_index])
+ out_of_fold_predictions[holdout_index, i] = y_pred
+
+ if self.use_features_in_secondary:
+ self.meta_model_.fit(np.hstack((X, out_of_fold_predictions)), y)
+ else:
+ self.meta_model_.fit(out_of_fold_predictions, y)
+
+ return self
+
+ def predict(self, X):
+ if self.task_type == 'classification':
+ meta_features = np.column_stack([[np.argmax(np.bincount(predictions)) for predictions in
+ np.column_stack([model.predict(X) for model in base_models])]
+ for base_models in self.base_models_])
+ else:
+ meta_features = np.column_stack([
+ np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)
+ for base_models in self.base_models_])
+ if self.use_features_in_secondary:
+ return self.meta_model_.predict(np.hstack((X, meta_features)))
+ else:
+ return self.meta_model_.predict(meta_features)
+
+ def predict_proba(self, X):
+ if self.task_type == 'classification':
+ meta_features = np.column_stack([[np.argmax(np.bincount(predictions)) for predictions in
+ np.column_stack([model.predict(X) for model in base_models])]
+ for base_models in self.base_models_])
+ if self.use_features_in_secondary:
+ return self.meta_model_.predict_proba(np.hstack((X, meta_features)))
+ else:
+ return self.meta_model_.predict_proba(meta_features)

Ensemble_Methods/code/stacking_retrained.py

@@ -0,0 +1,54 @@
+# based on https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard
+# and https://www.kaggle.com/eikedehling/trying-out-stacking-approaches
+from sklearn.base import BaseEstimator, TransformerMixin, clone, RegressorMixin
+from sklearn.model_selection import KFold
+import numpy as np
+
+
+class StackingModelsRetrained(BaseEstimator, RegressorMixin, TransformerMixin):
+ def __init__(self, base_models, meta_model, n_folds=5, use_features_in_secondary=False):
+ self.base_models = base_models
+ self.meta_model = meta_model
+ self.n_folds = n_folds
+ self.use_features_in_secondary = use_features_in_secondary
+
+ def fit(self, X, y):
+ """Fit all the models on the given dataset"""
+ self.base_models_ = [clone(x) for x in self.base_models]
+ self.meta_model_ = clone(self.meta_model)
+ kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
+
+ # Train cloned base models and create out-of-fold predictions
+ out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))
+ for i, model in enumerate(self.base_models):
+ for train_index, holdout_index in kfold.split(X, y):
+ instance = clone(model)
+ instance.fit(X[train_index], y[train_index])
+ y_pred = instance.predict(X[holdout_index])
+ out_of_fold_predictions[holdout_index, i] = y_pred
+
+ if self.use_features_in_secondary:
+ self.meta_model_.fit(np.hstack((X, out_of_fold_predictions)), y)
+ else:
+ self.meta_model_.fit(out_of_fold_predictions, y)
+
+ for model in self.base_models_:
+ model.fit(X, y)
+
+ return self
+
+ def predict(self, X):
+ meta_features = np.column_stack([
+ base_model.predict(X) for base_model in self.base_models_])
+ if self.use_features_in_secondary:
+ return self.meta_model_.predict(np.hstack((X, meta_features)))
+ else:
+ return self.meta_model_.predict(meta_features)
+
+ def predict_proba(self, X):
+ meta_features = np.column_stack([
+ base_model.predict(X) for base_model in self.base_models_])
+ if self.use_features_in_secondary:
+ return self.meta_model_.predict_proba(np.hstack((X, meta_features)))
+ else:
+ return self.meta_model_.predict_proba(meta_features)

Ensemble_Methods/code/weighted_average.py

@@ -0,0 +1,26 @@
+# based on https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard
+# and https://www.kaggle.com/eikedehling/trying-out-stacking-approaches
+from sklearn.base import BaseEstimator, TransformerMixin, clone, RegressorMixin
+import numpy as np
+
+
+class WeightedAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
+ def __init__(self, models, weights):
+ self.models = models
+ self.weights = weights
+ assert sum(self.weights) == 1
+
+ def fit(self, X, y):
+ self.models_ = [clone(x) for x in self.models]
+
+ # Train cloned base models
+ for model in self.models_:
+ model.fit(X, y)
+
+ return self
+
+ def predict(self, X):
+ predictions = np.column_stack([
+ model.predict(X) for model in self.models_
+ ])
+ return np.sum(predictions * self.weights, axis=1)