[FIX] Passing checks (#298)

* Initial fix for all tests passing locally py=3.8 * fix bug in tests * fix bug in test for data * debugging error in dummy forward pass * debug try -2 * catch runtime error in ci * catch runtime error in ci * add better debug test setup * debug some more * run this test only * remove sum backward * remove inplace in inception block * undo silly change * Enable all tests * fix flake * fix bug in test setup * remove anamoly detection * minor changes to comments * Apply suggestions from code review Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Address comments from Shuhei * revert change leading to bug * fix flake * change comment position in feature validator * Add documentation for _is_datasets_consistent * address comments from arlind * case when all nans in test Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com>

Author: ravinkohli

autoPyTorch/api/base_task.py

@@ -1386,7 +1386,7 @@ def fit_ensemble(
  Args:
  optimize_metric (str): name of the metric that is used to
  evaluate a pipeline. if not specified, value passed to search will be used
- precision (int), (default=32): Numeric precision used when loading
+ precision (Optional[int]): Numeric precision used when loading
  ensemble data. Can be either 16, 32 or 64.
  ensemble_nbest (Optional[int]):
  only consider the ensemble_nbest models to build the ensemble.
@@ -1429,6 +1429,7 @@ def fit_ensemble(
  "Please call the `search()` method of {} prior to "
  "fit_ensemble().".format(self.__class__.__name__))
 
+ precision = precision if precision is not None else self.precision
  if precision not in [16, 32, 64]:
  raise ValueError("precision must be one of 16, 32, 64 but got {}".format(precision))
 
@@ -1479,7 +1480,7 @@ def fit_ensemble(
  manager = self._init_ensemble_builder(
  time_left_for_ensembles=time_left_for_ensemble,
  optimize_metric=self.opt_metric if optimize_metric is None else optimize_metric,
- precision=self.precision if precision is None else precision,
+ precision=precision,
  ensemble_size=ensemble_size,
  ensemble_nbest=ensemble_nbest,
  )

autoPyTorch/data/tabular_feature_validator.py

@@ -1,5 +1,5 @@
 import functools
-from typing import Dict, List, Optional, Tuple, cast
+from typing import Dict, List, Optional, Tuple, Union, cast
 
 import numpy as np
 
@@ -100,6 +100,7 @@ def _comparator(cmp1: str, cmp2: str) -> int:
  if cmp1 not in choices or cmp2 not in choices:
  raise ValueError('The comparator for the column order only accepts {}, '
  'but got {} and {}'.format(choices, cmp1, cmp2))
+
  idx1, idx2 = choices.index(cmp1), choices.index(cmp2)
  return idx1 - idx2
 
@@ -246,13 +247,12 @@ def transform(
  # having a value for a categorical column.
  # We need to convert the column in test data to
  # object otherwise the test column is interpreted as float
- if len(self.categorical_columns) > 0:
- categorical_columns = self.column_transformer.transformers_[0][-1]
- for column in categorical_columns:
- if X[column].isna().all():
- X[column] = X[column].astype('object')
-
  if self.column_transformer is not None:
+ if len(self.categorical_columns) > 0:
+ categorical_columns = self.column_transformer.transformers_[0][-1]
+ for column in categorical_columns:
+ if X[column].isna().all():
+ X[column] = X[column].astype('object')
  X = self.column_transformer.transform(X)
 
  # Sparse related transformations
@@ -337,16 +337,10 @@ def _check_data(
 
  dtypes = [dtype.name for dtype in X.dtypes]
 
- dtypes_diff = [s_dtype != dtype for s_dtype, dtype in zip(self.dtypes, dtypes)]
+ diff_cols = X.columns[[s_dtype != dtype for s_dtype, dtype in zip(self.dtypes, dtypes)]]
  if len(self.dtypes) == 0:
  self.dtypes = dtypes
- elif (
- any(dtypes_diff) # the dtypes of some columns are different in train and test dataset
- and self.all_nan_columns is not None # Ignore all_nan_columns is None
- and len(set(X.columns[dtypes_diff]).difference(self.all_nan_columns)) != 0
- ):
- # The dtypes can be different if and only if the column belongs
- # to all_nan_columns as these columns would be imputed.
+ elif not self._is_datasets_consistent(diff_cols, X):
  raise ValueError("The dtype of the features must not be changed after fit(), but"
  " the dtypes of some columns are different between training ({}) and"
  " test ({}) datasets.".format(self.dtypes, dtypes))
@@ -508,6 +502,33 @@ def infer_objects(self, X: pd.DataFrame) -> pd.DataFrame:
 
  return X
 
+ def _is_datasets_consistent(self, diff_cols: List[Union[int, str]], X: pd.DataFrame) -> bool:
+ """
+ Check the consistency of dtypes between training and test datasets.
+ The dtypes can be different if the column belongs to `self.all_nan_columns`
+ (list of column names with all nans in training data) or if the column is
+ all nan as these columns would be imputed.
+
+ Args:
+ diff_cols (List[bool]):
+ The column labels that have different dtypes.
+ X (pd.DataFrame):
+ A validation or test dataset to be compared with the training dataset
+ Returns:
+ _ (bool): Whether the training and test datasets are consistent.
+ """
+ if self.all_nan_columns is None:
+ if len(diff_cols) == 0:
+ return True
+ else:
+ return all(X[diff_cols].isna().all())
+
+ # dtype is different ==> the column in at least either of train or test datasets must be all NaN
+ # inconsistent <==> dtype is different and the col in both train and test is not all NaN
+ inconsistent_cols = list(set(diff_cols) - self.all_nan_columns)
+
+ return len(inconsistent_cols) == 0 or all(X[inconsistent_cols].isna().all())
+
 
 def has_object_columns(
  feature_types: pd.Series,

autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/NoEncoder.py

@@ -39,7 +39,7 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]:
  Returns:
  (Dict[str, Any]): the updated 'X' dictionary
  """
- X.update({'encoder': self.preprocessor})
+ # X.update({'encoder': self.preprocessor})
  return X
 
  @staticmethod

autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/NoScaler.py

@@ -42,7 +42,7 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]:
  Returns:
  np.ndarray: Transformed features
  """
- X.update({'scaler': self.preprocessor})
+ # X.update({'scaler': self.preprocessor})
  return X
 
  @staticmethod

autoPyTorch/pipeline/components/setup/network_backbone/InceptionTimeBackbone.py

@@ -78,7 +78,7 @@ def __init__(self, n_res_inputs: int, n_outputs: int):
  def forward(self, x: torch.Tensor, res: torch.Tensor) -> torch.Tensor:
  shortcut = self.shortcut(res)
  shortcut = self.bn(shortcut)
- x += shortcut
+ x = x + shortcut
  return torch.relu(x)
 
 

autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py

@@ -21,7 +21,7 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator:
 
  self.embedding = self.build_embedding(
  num_input_features=num_input_features,
- num_numerical_features=num_numerical_columns)
+ num_numerical_features=num_numerical_columns) # type: ignore[arg-type]
  return self
 
  def transform(self, X: Dict[str, Any]) -> Dict[str, Any]:

autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py

@@ -109,11 +109,7 @@ def train_step(self, data: np.ndarray, targets: np.ndarray) -> Tuple[float, torc
  loss = loss_func(self.criterion, original_outputs, adversarial_outputs)
  loss.backward()
  self.optimizer.step()
- if self.scheduler:
- if 'ReduceLROnPlateau' in self.scheduler.__class__.__name__:
- self.scheduler.step(loss)
- else:
- self.scheduler.step()
+
  # only passing the original outputs since we do not care about
  # the adversarial performance.
  return loss.item(), original_outputs

autoPyTorch/pipeline/components/training/trainer/base_trainer_choice.py

@@ -280,6 +280,7 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom
  y=y,
  **kwargs
  )
+
  # Add snapshots to base network to enable
  # predicting with snapshot ensemble
  self.choice: autoPyTorchComponent = cast(autoPyTorchComponent, self.choice)

examples/tabular/40_advanced/example_custom_configuration_space.py

@@ -59,7 +59,7 @@ def get_search_space_updates():
  value_range=['shake-shake'],
  default_value='shake-shake')
  updates.append(node_name='network_backbone',
- hyperparameter='ResNetBackbone:shake_shake_method',
+ hyperparameter='ResNetBackbone:shake_shake_update_func',
  value_range=['M3'],
  default_value='M3'
  )

test/test_data/test_feature_validator.py

@@ -204,7 +204,6 @@ def test_featurevalidator_supported_types(input_data_featuretest):
  assert sparse.issparse(transformed_X)
  else:
  assert isinstance(transformed_X, np.ndarray)
- assert np.shape(input_data_featuretest) == np.shape(transformed_X)
  assert np.issubdtype(transformed_X.dtype, np.number)
  assert validator._is_fitted
 
@@ -237,9 +236,10 @@ def test_featurevalidator_categorical_nan(input_data_featuretest):
  validator.fit(input_data_featuretest)
  transformed_X = validator.transform(input_data_featuretest)
  assert any(pd.isna(input_data_featuretest))
- assert any((-1 in categories) or ('-1' in categories) or ('Missing!' in categories) for categories in
- validator.encoder.named_transformers_['encoder'].categories_)
- assert np.shape(input_data_featuretest) == np.shape(transformed_X)
+ categories_ = validator.column_transformer.\
+ named_transformers_['categorical_pipeline'].named_steps['onehotencoder'].categories_
+ assert any(('0' in categories) or (0 in categories) or ('missing_value' in categories) for categories in
+ categories_)
  assert np.issubdtype(transformed_X.dtype, np.number)
  assert validator._is_fitted
  assert isinstance(transformed_X, np.ndarray)
@@ -292,7 +292,6 @@ def test_featurevalidator_fitontypeA_transformtypeB(input_data_featuretest):
  else:
  raise ValueError(type(input_data_featuretest))
  transformed_X = validator.transform(complementary_type)
- assert np.shape(input_data_featuretest) == np.shape(transformed_X)
  assert np.issubdtype(transformed_X.dtype, np.number)
  assert validator._is_fitted
 
@@ -436,36 +435,29 @@ def test_features_unsupported_calls_are_raised():
  expected
  """
  validator = TabularFeatureValidator()
- with pytest.raises(ValueError, match=r"AutoPyTorch does not support time"):
+ with pytest.raises(TypeError, match=r".*?Convert the time information to a numerical value"):
  validator.fit(
  pd.DataFrame({'datetime': [pd.Timestamp('20180310')]})
  )
+ validator = TabularFeatureValidator()
  with pytest.raises(ValueError, match=r"AutoPyTorch only supports.*yet, the provided input"):
  validator.fit({'input1': 1, 'input2': 2})
- with pytest.raises(ValueError, match=r"has unsupported dtype string"):
+ validator = TabularFeatureValidator()
+ with pytest.raises(TypeError, match=r".*?but input column A has an invalid type `string`.*"):
  validator.fit(pd.DataFrame([{'A': 1, 'B': 2}], dtype='string'))
+ validator = TabularFeatureValidator()
  with pytest.raises(ValueError, match=r"The feature dimensionality of the train and test"):
  validator.fit(X_train=np.array([[1, 2, 3], [4, 5, 6]]),
  X_test=np.array([[1, 2, 3, 4], [4, 5, 6, 7]]),
  )
+ validator = TabularFeatureValidator()
  with pytest.raises(ValueError, match=r"Cannot call transform on a validator that is not fit"):
  validator.transform(np.array([[1, 2, 3], [4, 5, 6]]))
 
 
 @pytest.mark.parametrize(
  'input_data_featuretest',
  (
- 'numpy_numericalonly_nonan',
- 'numpy_numericalonly_nan',
- 'pandas_numericalonly_nonan',
- 'pandas_numericalonly_nan',
- 'list_numericalonly_nonan',
- 'list_numericalonly_nan',
- # Category in numpy is handled via feat_type
- 'numpy_categoricalonly_nonan',
- 'numpy_mixed_nonan',
- 'numpy_categoricalonly_nan',
- 'numpy_mixed_nan',
  'sparse_bsr_nonan',
  'sparse_bsr_nan',
  'sparse_coo_nonan',
@@ -483,14 +475,14 @@ def test_features_unsupported_calls_are_raised():
  ),
  indirect=True
 )
-def test_no_encoder_created(input_data_featuretest):
+def test_no_column_transformer_created(input_data_featuretest):
  """
  Makes sure that for numerical only features, no encoder is created
  """
  validator = TabularFeatureValidator()
  validator.fit(input_data_featuretest)
  validator.transform(input_data_featuretest)
- assert validator.encoder is None
+ assert validator.column_transformer is None
 
 
 @pytest.mark.parametrize(
@@ -501,18 +493,18 @@ def test_no_encoder_created(input_data_featuretest):
  ),
  indirect=True
 )
-def test_encoder_created(input_data_featuretest):
+def test_column_transformer_created(input_data_featuretest):
  """
- This test ensures an encoder is created if categorical data is provided
+ This test ensures an column transformer is created if categorical data is provided
  """
  validator = TabularFeatureValidator()
  validator.fit(input_data_featuretest)
  transformed_X = validator.transform(input_data_featuretest)
- assert validator.encoder is not None
+ assert validator.column_transformer is not None
 
  # Make sure that the encoded features are actually encoded. Categorical columns are at
  # the start after transformation. In our fixtures, this is also honored prior encode
- enc_columns, feature_types = validator._get_columns_to_encode(input_data_featuretest)
+ cat_columns, _, feature_types = validator._get_columns_info(input_data_featuretest)
 
  # At least one categorical
  assert 'categorical' in validator.feat_type
@@ -521,20 +513,13 @@ def test_encoder_created(input_data_featuretest):
  if np.any([pd.api.types.is_numeric_dtype(input_data_featuretest[col]
  ) for col in input_data_featuretest.columns]):
  assert 'numerical' in validator.feat_type
- for i, feat_type in enumerate(feature_types):
- if 'numerical' in feat_type:
- np.testing.assert_array_equal(
- transformed_X[:, i],
- input_data_featuretest[input_data_featuretest.columns[i]].to_numpy()
- )
- elif 'categorical' in feat_type:
- np.testing.assert_array_equal(
- transformed_X[:, i],
- # Expect always 0, 1... because we use a ordinal encoder
- np.array([0, 1])
- )
- else:
- raise ValueError(feat_type)
+ # we expect this input to be the fixture 'pandas_mixed_nan'
+ np.testing.assert_array_equal(transformed_X, np.array([[1., 0., -1.], [0., 1., 1.]]))
+ else:
+ np.testing.assert_array_equal(transformed_X, np.array([[1., 0., 1., 0.], [0., 1., 0., 1.]]))
+
+ if not all([feat_type in ['numerical', 'categorical'] for feat_type in feature_types]):
+ raise ValueError("Expected only numerical and categorical feature types")
 
 
 def test_no_new_category_after_fit():
@@ -566,13 +551,12 @@ def test_unknown_encode_value():
  x['c'].cat.add_categories(['NA'], inplace=True)
  x.loc[0, 'c'] = 'NA' # unknown value
  x_t = validator.transform(x)
- # The first row should have a -1 as we added a new categorical there
- expected_row = [-1, -41, -3, -987.2]
+ # The first row should have a 0, 0 as we added a
+ # new categorical there and one hot encoder marks
+ # it as all zeros for the transformed column
+ expected_row = [0.0, 0.0, -0.5584294383572701, 0.5000000000000004, -1.5136598016833485]
  assert expected_row == x_t[0].tolist()
 
- # Notice how there is only one column 'c' to encode
- assert validator.categories == [list(range(2)) for i in range(1)]
-
 
 # Actual checks for the features
 @pytest.mark.parametrize(
@@ -624,19 +608,20 @@ def test_feature_validator_new_data_after_fit(
  assert sparse.issparse(transformed_X)
  else:
  assert isinstance(transformed_X, np.ndarray)
- assert np.shape(X_test) == np.shape(transformed_X)
 
  # And then check proper error messages
  if train_data_type == 'pandas':
  old_dtypes = copy.deepcopy(validator.dtypes)
  validator.dtypes = ['dummy' for dtype in X_train.dtypes]
- with pytest.raises(ValueError, match=r"Changing the dtype of the features after fit"):
+ with pytest.raises(ValueError,
+ match=r"The dtype of the features must not be changed after fit"):
  transformed_X = validator.transform(X_test)
  validator.dtypes = old_dtypes
  if test_data_type == 'pandas':
  columns = X_test.columns.tolist()
  X_test = X_test[reversed(columns)]
- with pytest.raises(ValueError, match=r"Changing the column order of the features"):
+ with pytest.raises(ValueError,
+ match=r"The column order of the features must not be changed after fit"):
  transformed_X = validator.transform(X_test)