Run operator within "ads opctl run"

ads/opctl/backend/ads_ml_job.py

@@ -93,8 +93,11 @@ def init(
  `None` otherwise.
  """
 
- conda_slug = kwargs.get(
- "conda_slug", self.config["execution"].get("conda_slug", "conda_slug")
+ conda_slug = (
+ kwargs.get(
+ "conda_slug", self.config["execution"].get("conda_slug", "conda_slug")
+ )
+ or ""
  ).lower()
 
  # if conda slug contains '/' then the assumption is that it is a custom conda pack

ads/opctl/backend/local.py

@@ -880,7 +880,7 @@ def __init__(
 
  self.operator_info = operator_info
 
- def _run_with_python(self) -> int:
+ def _run_with_python(self, **kwargs: Dict) -> int:
  """Runs the operator within a local python environment.
 
  Returns
@@ -908,7 +908,7 @@ def _run_with_python(self) -> int:
  else:
  return 0
 
- def _run_with_container(self) -> int:
+ def _run_with_container(self, **kwargs: Dict) -> int:
  """Runs the operator within a container.
 
  Returns
@@ -964,6 +964,12 @@ def run(self, **kwargs: Dict) -> Dict:
  if not self.operator_info:
  self.operator_info = OperatorLoader.from_uri(self.operator_type).load()
 
+ if self.config.get("dry_run"):
+ logger.info(
+ "The dry run option is not supported for "
+ "the local backends and will be ignored."
+ )
+
  # run operator with provided runtime
  exit_code = self._RUNTIME_RUN_MAP.get(runtime_type, lambda: None)(**kwargs)
 

ads/opctl/cli.py

@@ -262,12 +262,6 @@ def _add_options(func):
 
 @commands.command()
 @add_options(_options)
-@click.option(
- "--backend-config",
- help="path to the backend config YAML file",
- required=False,
- default=None,
-)
 @click.option("--image", "-i", help="image name", required=False, default=None)
 @click.option("--conda-slug", help="slug name", required=False, default=None)
 @click.option(

ads/opctl/cmds.py

@@ -17,10 +17,7 @@
 from ads.common.extended_enum import ExtendedEnumMeta
 from ads.common.oci_datascience import DSCNotebookSession
 from ads.opctl.backend.ads_dataflow import DataFlowBackend
-from ads.opctl.backend.ads_ml_job import (
- MLJobBackend,
- MLJobDistributedBackend,
-)
+from ads.opctl.backend.ads_ml_job import MLJobBackend, MLJobDistributedBackend
 from ads.opctl.backend.ads_ml_pipeline import PipelineBackend
 from ads.opctl.backend.ads_model_deployment import ModelDeploymentBackend
 from ads.opctl.backend.local import (
@@ -58,6 +55,9 @@
  update_ini,
  verify_and_publish_image,
 )
+from ads.opctl.operator.common.backend_factory import (
+ BackendFactory as OperatorBackendFactory,
+)
 from ads.opctl.utils import get_service_pack_prefix, is_in_notebook_session
 
 
@@ -177,6 +177,19 @@ def run(config: Dict, **kwargs) -> Dict:
  """
  if config:
  p = ConfigProcessor(config).step(ConfigMerger, **kwargs)
+ try:
+ return OperatorBackendFactory.backend(
+ config=p,
+ backend=p.config["execution"]["backend"],
+ **{
+ key: value
+ for key, value in kwargs.items()
+ if key not in ("backend", "config")
+ },
+ ).run(**kwargs)
+ except RuntimeError:
+ pass
+
  if (
  p.config["kind"] != BACKEND_NAME.LOCAL.value
  and p.config["kind"] != "distributed"
@@ -371,7 +384,7 @@ def cancel(**kwargs) -> None:
  ----------
  kwargs: dict
  keyword argument, stores command line args
- 
+
  Returns
  -------
  None
@@ -383,9 +396,7 @@ def cancel(**kwargs) -> None:
  or DataScienceResourceRun.PIPELINE_RUN in kwargs["ocid"]
  ):
  kwargs["run_id"] = kwargs.pop("ocid")
- elif (
- DataScienceResource.JOB in kwargs["ocid"]
- ):
+ elif DataScienceResource.JOB in kwargs["ocid"]:
  kwargs["id"] = kwargs.pop("ocid")
  else:
  raise ValueError(f"{kwargs['ocid']} is invalid or not supported.")

ads/opctl/operator/__init__.py

@@ -4,7 +4,21 @@
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-__operators__ = ["forecast"]
+import os
+
+
+def __registered_operators():
+ """Gets the list of registered operators."""
+
+ target_dir = os.path.join(os.path.dirname(__file__), "lowcode")
+ return [
+ f
+ for f in os.listdir(target_dir)
+ if os.path.isdir(os.path.join(target_dir, f)) and not f.startswith("__")
+ ]
+
+
+__operators__ = __registered_operators()
 
 
 class OperatorNotFoundError(Exception):

ads/opctl/operator/cli.py

@@ -13,7 +13,7 @@
 from ads.common import auth as authutil
 from ads.common.auth import AuthType
 from ads.common.object_storage_details import ObjectStorageDetails
-from ads.opctl.constants import BACKEND_NAME
+from ads.opctl.constants import BACKEND_NAME, RUNTIME_TYPE
 from ads.opctl.decorator.common import click_options, with_auth
 from ads.opctl.utils import suppress_traceback
 
@@ -247,18 +247,24 @@ def publish_conda(debug: bool, **kwargs: Dict[str, Any]) -> None:
 @commands.command()
 @click_options(DEBUG_OPTION + ADS_CONFIG_OPTION + AUTH_TYPE_OPTION)
 @click.option(
- "--file", "-f", help="The path to resource YAML file.", required=True, default=None
+ "--file",
+ "-f",
+ help="The path to the operator's specification YAML file.",
+ required=True,
+ default=None,
 )
 @click.option(
  "--backend",
  "-b",
  help=(
  "Backend name or the path to the operator's backend config YAML file. "
- f"Example 1: `ads operator run -f operator.yaml -b {BACKEND_NAME.LOCAL.value}` "
- "Supported backends: "
- f"{[BACKEND_NAME.JOB.value,BACKEND_NAME.DATAFLOW.value,BACKEND_NAME.LOCAL.value,]} "
- "Example 2: `ads operator run -f operator.yaml -b backend.yaml` "
- "Use the `ads operator init` command to generate operator's configs. "
+ f"\n\nExample 1:\n\n`ads operator run -f operator.yaml -b {BACKEND_NAME.LOCAL.value}`\n\n"
+ "Supported backend names: "
+ f"{(BACKEND_NAME.JOB.value,BACKEND_NAME.JOB.value + '.' + RUNTIME_TYPE.CONTAINER.value,BACKEND_NAME.DATAFLOW.value,BACKEND_NAME.LOCAL.value,BACKEND_NAME.LOCAL.value + '.'+ RUNTIME_TYPE.CONTAINER.value,)}. "
+ "However some operators may support only a subset of these backends."
+ "\n\nExample 2:\n\n`ads operator run -f operator.yaml -b backend.yaml`\n\n"
+ "Use the `ads operator init --help` command to generate the operator's specification "
+ "and all required backend configs. Generating configs is optional and fully automated. "
  ),
  required=False,
  default=None,