Automlx Metrics Updates (#1233)

Author: prasankh

.github/workflows/run-forecast-explainer-tests.yml

@@ -54,6 +54,6 @@ jobs:
  $CONDA/bin/conda init
  source /home/runner/.bashrc
  pip install -r test-requirements-operators.txt
- pip install "oracle-automlx[forecasting]>=25.1.1"
+ pip install "oracle-automlx[forecasting]>=25.3.0"
  pip install pandas>=2.2.0
  python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast/test_explainers.py

.github/workflows/run-forecast-unit-tests.yml

@@ -56,6 +56,6 @@ jobs:
  $CONDA/bin/conda init
  source /home/runner/.bashrc
  pip install -r test-requirements-operators.txt
- pip install "oracle-automlx[forecasting]>=25.1.1"
+ pip install "oracle-automlx[forecasting]>=25.3.0"
  pip install pandas>=2.2.0
  python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast --ignore=tests/operators/forecast/test_explainers.py

ads/opctl/operator/lowcode/anomaly/model/automlx.py

@@ -24,8 +24,8 @@ class AutoMLXOperatorModel(AnomalyOperatorBaseModel):
  @runtime_dependency(
  module="automlx",
  err_msg=(
- "Please run `pip3 install oracle-automlx>=23.4.1` and "
- "`pip3 install oracle-automlx[classic]>=23.4.1` "
+ "Please run `pip3 install oracle-automlx>=25.3.0` and "
+ "`pip3 install oracle-automlx[classic]>=25.3.0` "
  "to install the required dependencies for automlx."
  ),
  )

ads/opctl/operator/lowcode/forecast/const.py

@@ -76,7 +76,7 @@ class ForecastOutputColumns(ExtendedEnum):
 
 AUTOMLX_METRIC_MAP = {
  "smape": "neg_sym_mean_abs_percent_error",
- "mape": "neg_sym_mean_abs_percent_error",
+ "mape": "neg_mean_abs_percent_error",
  "mase": "neg_mean_abs_scaled_error",
  "mae": "neg_mean_absolute_error",
  "mse": "neg_mean_squared_error",

ads/opctl/operator/lowcode/forecast/model/automlx.py

@@ -28,7 +28,9 @@
 
 logging.getLogger("report_creator").setLevel(logging.WARNING)
 AUTOMLX_N_ALGOS_TUNED = 4
-AUTOMLX_DEFAULT_SCORE_METRIC = "neg_sym_mean_abs_percent_error"
+AUTOMLX_DEFAULT_SCORE_METRIC = ['neg_sym_mean_abs_percent_error',
+ 'neg_mean_abs_percent_error',
+ 'neg_root_mean_squared_error']
 
 
 class AutoMLXOperatorModel(ForecastOperatorBaseModel):
@@ -45,10 +47,13 @@ def set_kwargs(self):
  model_kwargs_cleaned["n_algos_tuned"] = model_kwargs_cleaned.get(
  "n_algos_tuned", AUTOMLX_N_ALGOS_TUNED
  )
- model_kwargs_cleaned["score_metric"] = AUTOMLX_METRIC_MAP.get(
- self.spec.metric,
- model_kwargs_cleaned.get("score_metric", AUTOMLX_DEFAULT_SCORE_METRIC),
- )
+ metric_to_optimize = AUTOMLX_METRIC_MAP.get(self.spec.metric)
+ model_kwargs_cleaned["score_metric"] = AUTOMLX_DEFAULT_SCORE_METRIC
+ # The first score metric in the list will be the one for which the pipeline optimizes
+ if metric_to_optimize is not None:
+ model_kwargs_cleaned["score_metric"].remove(metric_to_optimize)
+ model_kwargs_cleaned["score_metric"].insert(0, metric_to_optimize)
+
  model_kwargs_cleaned.pop("task", None)
  time_budget = model_kwargs_cleaned.pop("time_budget", -1)
  model_kwargs_cleaned["preprocessing"] = (
@@ -70,7 +75,7 @@ def preprocess(self, data, series_id): # TODO: re-use self.le for explanations
  @runtime_dependency(
  module="automlx",
  err_msg=(
- "Please run `pip3 install oracle-automlx[forecasting]>=25.1.1` "
+ "Please run `pip3 install oracle-automlx[forecasting]>=25.3.0` "
  "to install the required dependencies for automlx."
  ),
  )
@@ -163,7 +168,7 @@ def _build_model(self) -> pd.DataFrame:
  self.models[s_id] = {}
  self.models[s_id]["model"] = model
  self.models[s_id]["le"] = self.le[s_id]
- self.models[s_id]["score"] = self.get_validation_score_and_metric(model)
+ self.models[s_id]["score"] = self.get_all_metrics(model)
 
  # In case of Naive model, model.forecast function call does not return confidence intervals.
  if f"{target}_ci_upper" not in summary_frame:
@@ -518,26 +523,27 @@ def explain_model(self):
  )
  logger.debug(f"Full Traceback: {traceback.format_exc()}")
 
- def get_validation_score_and_metric(self, model):
+ def get_all_metrics(self, model):
  trials = model.completed_trials_summary_
  model_params = model.selected_model_params_
  if len(trials) > 0:
- score_col = [col for col in trials.columns if "Score" in col][0]
- validation_score = trials[trials.Hyperparameters == model_params][
- score_col
+ all_metrics = trials[trials.Hyperparameters == model_params][
+ "All Metrics"
  ].iloc[0]
  else:
- validation_score = 0
- return -1 * validation_score
+ all_metrics = {}
+ reverse_map = {v: k for k, v in AUTOMLX_METRIC_MAP.items()}
+ all_metrics = {reverse_map[key]: -1 * value for key, value in all_metrics.items() if key in reverse_map}
+ return all_metrics
 
  def generate_train_metrics(self) -> pd.DataFrame:
  """
- Generate Training Metrics when fitted data is not available.
+ Generate Training Metrics for Automlx
  """
  total_metrics = pd.DataFrame()
  for s_id in self.forecast_output.list_series_ids():
  try:
- metrics = {self.spec.metric.upper(): self.models[s_id]["score"]}
+ metrics = self.models[s_id]["score"]
  metrics_df = pd.DataFrame.from_dict(
  metrics, orient="index", columns=[s_id]
  )