[ODSC-48942]opctl create/publish error on M1 and M2 (#381)

Author: mrDzurb

THIRD_PARTY_LICENSES.txt

@@ -241,6 +241,12 @@ psutil
 * Source code: https://github.com/giampaolo/psutil
 * Project home: https://github.com/giampaolo/psutil
 
+py-cpuinfo
+* Copyright (c) 2014-2022 Matthew Brennan Jones <matthew.brennan.jones@gmail.com>
+* License: The MIT License (MIT)
+* Source code: https://github.com/workhorsy/py-cpuinfo
+* Project home: https://github.com/workhorsy/py-cpuinfo
+
 pyspark
 * Copyright 2014 and onwards The Apache Software Foundation.
 * License: Apache-2.0 LICENSE

ads/opctl/conda/cmds.py

@@ -638,7 +638,10 @@ def _publish(
  )
  except Exception:
  raise RuntimeError(f"Could not pack environment {conda_slug}.")
+ NOT_ALLOWED_CHARS = "@#$%^&*/"
 
+ if any(chr in conda_slug for chr in NOT_ALLOWED_CHARS):
+ raise ValueError(f"Invalid conda_slug. Found {NOT_ALLOWED_CHARS} in slug name. Please use a different slug name.")
  pack_file = os.path.join(pack_folder_path, f"{conda_slug}.tar.gz")
  if not os.path.exists(pack_file):
  raise RuntimeError(f"Pack {pack_file} was not created.")

ads/opctl/conda/pack.py

@@ -26,15 +26,16 @@ def main(pack_folder_path, manifest_file=None):
  )
  with open(manifest_path) as f:
  env = yaml.safe_load(f.read())
-
+ 
  with tempfile.TemporaryDirectory() as td:
  process = subprocess.Popen(
  ["conda", "env", "export", "--prefix", pack_folder_path],
  stdout=subprocess.PIPE,
  stderr=subprocess.PIPE,
  )
  stdout, stderr = process.communicate()
- if stderr:
+ 
+ if process.returncode and stderr:
  print(stderr)
  raise Exception(
  f"Error export environment information from {pack_folder_path}"

ads/opctl/docker/Dockerfile.job

@@ -100,4 +100,4 @@ RUN rm -rf /tmp/*
 
 RUN mkdir -p /etc/datascience/operators
 
-USER datascience
+USER datascience

ads/opctl/utils.py

@@ -17,6 +17,7 @@
 from subprocess import Popen, PIPE, STDOUT
 from typing import Union, List, Tuple, Dict
 import yaml
+import re
 
 import ads
 from ads.common.oci_client import OCIClientFactory
@@ -160,7 +161,14 @@ def build_image(
  )
  proc = _build_custom_operator_image(gpu, source_folder, dst_image)
  else:
- image, dockerfile, target = _get_image_name_dockerfile_target(image_type, gpu)
+ # https://stackoverflow.com/questions/66842004/get-the-processor-type-using-python-for-apple-m1-processor-gives-me-an-intel-pro
+ import cpuinfo
+ # Just get the manufacturer of the processors
+ manufacturer = cpuinfo.get_cpu_info().get('brand_raw')
+ arch = 'arm' if re.search("apple m\d ", manufacturer, re.IGNORECASE) else 'other'
+ print(f"The local machine's platform is {arch}.")
+ image, dockerfile, target = _get_image_name_dockerfile_target(image_type, gpu, arch)
+ print(f"dockerfile used is {dockerfile}")
  command = [
  "docker",
  "build",
@@ -186,14 +194,15 @@ def build_image(
  raise RuntimeError("Docker build failed.")
 
 
-def _get_image_name_dockerfile_target(type: str, gpu: bool) -> str:
+def _get_image_name_dockerfile_target(type: str, gpu: bool, arch: str) -> str:
  look_up = {
- ("job-local", False): (ML_JOB_IMAGE, "Dockerfile.job", None),
- ("job-local", True): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None),
- ("ads-ops-base", False): (OPS_IMAGE_BASE, "Dockerfile", "base"),
- ("ads-ops-base", True): (OPS_IMAGE_GPU_BASE, "Dockerfile.gpu", "base"),
+ ("job-local", False, "arm"): (ML_JOB_IMAGE, "Dockerfile.job.arm", None),
+ ("job-local", False, "other"): (ML_JOB_IMAGE, "Dockerfile.job", None),
+ ("job-local", True, "other"): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None),
+ ("ads-ops-base", False, "other"): (OPS_IMAGE_BASE, "Dockerfile", "base"),
+ ("ads-ops-base", True, "other"): (OPS_IMAGE_GPU_BASE, "Dockerfile.gpu", "base"),
  }
- return look_up[(type, gpu)]
+ return look_up[(type, gpu, arch)]
 
 
 @runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)

pyproject.toml

@@ -123,6 +123,7 @@ opctl = [
  "nbconvert",
  "nbformat",
  "oci-cli",
+ "py-cpuinfo",
 ]
 optuna = [
  "optuna==2.9.0",