FIX compute the median of std dev for each class to over-sample in SMOTENC (#1015)

Author: glemaitre

doc/over_sampling.rst

@@ -203,9 +203,9 @@ or relying on `dtype` inference if the columns are using the
  >>> print(sorted(Counter(y_resampled).items()))
  [(0, 30), (1, 30)]
  >>> print(X_resampled[-5:])
- [['A' 0.52... 2]
+ [['A' 0.19... 2]
  ['B' -0.36... 2]
- ['B' 0.93... 2]
+ ['B' 0.87... 2]
  ['B' 0.37... 2]
  ['B' 0.33... 2]]
 

doc/whats_new/v0.11.rst

@@ -14,6 +14,15 @@ Bug fixes
  they are plugged into an Euclidean distance computation.
  :pr:`1014` by :user:`Guillaume Lemaitre <glemaitre>`.
 
+- Fix a bug in :class:`~imblearn.over_sampling.SMOTENC` where the median of standard
+ deviation of the continuous features was only computed on the minority class. Now,
+ we are computing this statistic for each class that is up-sampled.
+ :pr:`1015` by :user:`Guillaume Lemaitre <glemaitre>`.
+
+- Fix a bug in :class:`~imblearn.over_sampling.SMOTENC` such that the case where
+ the median of standard deviation of the continuous features is null is handled
+ in the multiclass case as well.
+ :pr:`1015` by :user:`Guillaume Lemaitre <glemaitre>`.
 
 Version 0.11.0
 ==============

imblearn/over_sampling/_smote/base.py

@@ -9,7 +9,6 @@
 import math
 import numbers
 import warnings
-from collections import Counter
 
 import numpy as np
 from scipy import sparse
@@ -23,7 +22,6 @@
  check_random_state,
 )
 from sklearn.utils.sparsefuncs_fast import (
- csc_mean_variance_axis0,
  csr_mean_variance_axis0,
 )
 from sklearn.utils.validation import _num_features
@@ -116,11 +114,11 @@ def _make_samples(
  rows = np.floor_divide(samples_indices, nn_num.shape[1])
  cols = np.mod(samples_indices, nn_num.shape[1])
 
- X_new = self._generate_samples(X, nn_data, nn_num, rows, cols, steps)
+ X_new = self._generate_samples(X, nn_data, nn_num, rows, cols, steps, y_type)
  y_new = np.full(n_samples, fill_value=y_type, dtype=y_dtype)
  return X_new, y_new
 
- def _generate_samples(self, X, nn_data, nn_num, rows, cols, steps):
+ def _generate_samples(self, X, nn_data, nn_num, rows, cols, steps, y_type=None):
  r"""Generate a synthetic sample.
 
  The rule for the generation is:
@@ -155,6 +153,9 @@ def _generate_samples(self, X, nn_data, nn_num, rows, cols, steps):
  steps : ndarray of shape (n_samples,), dtype=float
  Step sizes for new samples.
 
+ y_type : None
+ Unused parameter. Only for compatibility reason with SMOTE-NC.
+
  Returns
  -------
  X_new : {ndarray, sparse matrix} of shape (n_samples, n_features)
@@ -465,8 +466,9 @@ class SMOTENC(SMOTE):
  continuous_features_ : ndarray of shape (n_cont_features,), dtype=np.int64
  Indices of the continuous features.
 
- median_std_ : float
- Median of the standard deviation of the continuous features.
+ median_std_ : dict of int -> float
+ Median of the standard deviation of the continuous features for each
+ class to be over-sampled.
 
  n_features_ : int
  Number of features observed at `fit`.
@@ -627,23 +629,8 @@ def _fit_resample(self, X, y):
  self._validate_column_types(X)
  self._validate_estimator()
 
- # compute the median of the standard deviation of the minority class
- target_stats = Counter(y)
- class_minority = min(target_stats, key=target_stats.get)
-
  X_continuous = _safe_indexing(X, self.continuous_features_, axis=1)
  X_continuous = check_array(X_continuous, accept_sparse=["csr", "csc"])
- X_minority = _safe_indexing(X_continuous, np.flatnonzero(y == class_minority))
-
- if sparse.issparse(X):
- if X.format == "csr":
- _, var = csr_mean_variance_axis0(X_minority)
- else:
- _, var = csc_mean_variance_axis0(X_minority)
- else:
- var = X_minority.var(axis=0)
- self.median_std_ = np.median(np.sqrt(var))
-
  X_categorical = _safe_indexing(X, self.categorical_features_, axis=1)
  if X_continuous.dtype.name != "object":
  dtype_ohe = X_continuous.dtype
@@ -664,28 +651,54 @@ def _fit_resample(self, X, y):
  if not sparse.issparse(X_ohe):
  X_ohe = sparse.csr_matrix(X_ohe, dtype=dtype_ohe)
 
- # we can replace the 1 entries of the categorical features with the
- # median of the standard deviation. It will ensure that whenever
- # distance is computed between 2 samples, the difference will be equal
- # to the median of the standard deviation as in the original paper.
-
- # In the edge case where the median of the std is equal to 0, the 1s
- # entries will be also nullified. In this case, we store the original
- # categorical encoding which will be later used for inverting the OHE
- if math.isclose(self.median_std_, 0):
- self._X_categorical_minority_encoded = _safe_indexing(
- X_ohe.toarray(), np.flatnonzero(y == class_minority)
+ X_encoded = sparse.hstack((X_continuous, X_ohe), format="csr", dtype=dtype_ohe)
+ X_resampled = [X_encoded.copy()]
+ y_resampled = [y.copy()]
+
+ # SMOTE resampling starts here
+ self.median_std_ = {}
+ for class_sample, n_samples in self.sampling_strategy_.items():
+ if n_samples == 0:
+ continue
+ target_class_indices = np.flatnonzero(y == class_sample)
+ X_class = _safe_indexing(X_encoded, target_class_indices)
+
+ _, var = csr_mean_variance_axis0(
+ X_class[:, : self.continuous_features_.size]
  )
+ self.median_std_[class_sample] = np.median(np.sqrt(var))
+
+ # In the edge case where the median of the std is equal to 0, the 1s
+ # entries will be also nullified. In this case, we store the original
+ # categorical encoding which will be later used for inverting the OHE
+ if math.isclose(self.median_std_[class_sample], 0):
+ # This variable will be used when generating data
+ self._X_categorical_minority_encoded = X_class[
+ :, self.continuous_features_.size :
+ ].toarray()
+
+ # we can replace the 1 entries of the categorical features with the
+ # median of the standard deviation. It will ensure that whenever
+ # distance is computed between 2 samples, the difference will be equal
+ # to the median of the standard deviation as in the original paper.
+ X_class_categorical = X_class[:, self.continuous_features_.size :]
+ # With one-hot encoding, the median will be repeated twice. We need
+ # to divide by sqrt(2) such that we only have one median value
+ # contributing to the Euclidean distance
+ X_class_categorical.data[:] = self.median_std_[class_sample] / np.sqrt(2)
+ X_class[:, self.continuous_features_.size :] = X_class_categorical
 
- # With one-hot encoding, the median will be repeated twice. We need to divide
- # by sqrt(2) such that we only have one median value contributing to the
- # Euclidean distance
- X_ohe.data = (
- np.ones_like(X_ohe.data, dtype=X_ohe.dtype) * self.median_std_ / np.sqrt(2)
- )
- X_encoded = sparse.hstack((X_continuous, X_ohe), format="csr")
+  self.nn_k_.fit(X_class)
+  nns = self.nn_k_.kneighbors(X_class, return_distance=False)[:, 1:]
+  X_new, y_new = self._make_samples(
+  X_class, y.dtype, class_sample, X_class, nns, n_samples, 1.0
+ )
+  X_resampled.append(X_new)
+  y_resampled.append(y_new)
 
- X_resampled, y_resampled = super()._fit_resample(X_encoded, y)
+ X_resampled = sparse.vstack(X_resampled, format=X_encoded.format)
+ y_resampled = np.hstack(y_resampled)
+ # SMOTE resampling ends here
 
  # reverse the encoding of the categorical features
  X_res_cat = X_resampled[:, self.continuous_features_.size :]
@@ -723,7 +736,7 @@ def _fit_resample(self, X, y):
 
  return X_resampled, y_resampled
 
- def _generate_samples(self, X, nn_data, nn_num, rows, cols, steps):
+ def _generate_samples(self, X, nn_data, nn_num, rows, cols, steps, y_type):
  """Generate a synthetic sample with an additional steps for the
  categorical features.
 
@@ -741,7 +754,7 @@ def _generate_samples(self, X, nn_data, nn_num, rows, cols, steps):
 
  # In the case that the median std was equal to zeros, we have to
  # create non-null entry based on the encoded of OHE
- if math.isclose(self.median_std_, 0):
+ if math.isclose(self.median_std_[y_type], 0):
  nn_data[
  :, self.continuous_features_.size :
  ] = self._X_categorical_minority_encoded

imblearn/over_sampling/_smote/tests/test_smote_nc.py

@@ -130,6 +130,8 @@ def test_smotenc(data):
  assert set(X[:, cat_idx]) == set(X_resampled[:, cat_idx])
  assert X[:, cat_idx].dtype == X_resampled[:, cat_idx].dtype
 
+ assert isinstance(smote.median_std_, dict)
+
 
 # part of the common test which apply to SMOTE-NC even if it is not default
 # constructible
@@ -193,6 +195,7 @@ def test_smotenc_pandas():
  X_res, y_res = smote.fit_resample(X, y)
  assert_array_equal(X_res_pd.to_numpy(), X_res)
  assert_allclose(y_res_pd, y_res)
+ assert set(smote.median_std_.keys()) == {0, 1}
 
 
 def test_smotenc_preserve_dtype():
@@ -234,20 +237,36 @@ def test_smote_nc_with_null_median_std():
  [
  [1, 2, 1, "A"],
  [2, 1, 2, "A"],
+ [2, 1, 2, "A"],
  [1, 2, 3, "B"],
  [1, 2, 4, "C"],
  [1, 2, 5, "C"],
+ [1, 2, 4, "C"],
+ [1, 2, 4, "C"],
+ [1, 2, 4, "C"],
  ],
  dtype="object",
  )
  labels = np.array(
- ["class_1", "class_1", "class_1", "class_2", "class_2"], dtype=object
+ [
+ "class_1",
+ "class_1",
+ "class_1",
+ "class_1",
+ "class_2",
+ "class_2",
+ "class_3",
+ "class_3",
+ "class_3",
+ ],
+ dtype=object,
  )
  smote = SMOTENC(categorical_features=[3], k_neighbors=1, random_state=0)
  X_res, y_res = smote.fit_resample(data, labels)
  # check that the categorical feature is not random but correspond to the
  # categories seen in the minority class samples
- assert X_res[-1, -1] == "C"
+ assert_array_equal(X_res[-3:, -1], np.array(["C", "C", "C"], dtype=object))
+ assert smote.median_std_ == {"class_2": 0.0, "class_3": 0.0}
 
 
 def test_smotenc_categorical_encoder():