feat: Add initial Model Builder SDK samples (#265)

* Add inital batch of MBSDK samples and sample tests * Drop unneeded prints * Update to use constants, address reviewers

Author: vinnysenthil

samples/model-builder/conftest.py

@@ -0,0 +1,113 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from unittest import mock
+from unittest.mock import MagicMock, patch
+
+from google.cloud import aiplatform
+
+
+@pytest.fixture
+def mock_sdk_init():
+ with patch.object(aiplatform, "init") as mock:
+ yield mock
+
+
+# ----------------------------------------------------------------------------
+# Dataset Fixtures
+# ----------------------------------------------------------------------------
+
+
+@pytest.fixture
+def mock_dataset():
+ mock = MagicMock(aiplatform.datasets.Dataset)
+ yield mock
+
+
+@pytest.fixture
+def mock_new_dataset(mock_dataset):
+ with patch.object(aiplatform.datasets.Dataset, "__new__") as mock_new_dataset:
+ mock_new_dataset.return_value = mock_dataset
+ yield mock_new_dataset
+
+
+@pytest.fixture
+def mock_init_dataset(mock_new_dataset):
+ with patch.object(aiplatform.datasets.Dataset, "__init__") as mock_init_dataset:
+ mock_init_dataset.return_value = None
+ yield mock_init_dataset
+
+
+@pytest.fixture
+def mock_create_dataset():
+ with patch.object(aiplatform.datasets.Dataset, "create") as mock:
+ mock.return_value = MagicMock(aiplatform.Dataset)
+ yield mock
+
+
+@pytest.fixture
+def mock_create_image_dataset():
+ with patch.object(aiplatform.datasets.ImageDataset, "create") as mock:
+ mock.return_value = MagicMock(aiplatform.Dataset)
+ yield mock
+
+
+# ----------------------------------------------------------------------------
+# TrainingJob Fixtures
+# ----------------------------------------------------------------------------
+
+
+@pytest.fixture
+def mock_init_automl_image_training_job():
+ with patch.object(
+ aiplatform.training_jobs.AutoMLImageTrainingJob, "__init__"
+ ) as mock:
+ mock.return_value = None
+ yield mock
+
+
+@pytest.fixture
+def mock_run_automl_image_training_job():
+ with patch.object(aiplatform.training_jobs.AutoMLImageTrainingJob, "run") as mock:
+ yield mock
+
+
+# ----------------------------------------------------------------------------
+# Model Fixtures
+# ----------------------------------------------------------------------------
+
+
+@pytest.fixture
+def mock_init_model():
+ with patch.object(aiplatform.models.Model, "__init__") as mock:
+ mock.return_value = None
+ yield mock
+
+
+@pytest.fixture
+def mock_batch_predict_model():
+ with patch.object(aiplatform.models.Model, "batch_predict") as mock:
+ yield mock
+
+
+# ----------------------------------------------------------------------------
+# Job Fixtures
+# ----------------------------------------------------------------------------
+
+
+@pytest.fixture
+def mock_create_batch_prediction_job():
+ with patch.object(aiplatform.jobs.BatchPredictionJob, "create") as mock:
+ yield mock

samples/model-builder/create_and_import_dataset_image_sample.py

@@ -0,0 +1,43 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Union
+
+from google.cloud import aiplatform
+
+# [START aiplatform_sdk_create_and_import_dataset_image_sample]
+def create_and_import_dataset_image_sample(
+ project: str,
+ location: str,
+ display_name: str,
+ src_uris: Union[str, List[str]],
+ sync: bool = True,
+):
+ aiplatform.init(project=project, location=location)
+
+ ds = aiplatform.ImageDataset.create(
+ display_name=display_name,
+ gcs_source=src_uris,
+ import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,
+ sync=sync,
+ )
+
+ ds.wait()
+
+ print(ds.display_name)
+ print(ds.resource_name)
+ return ds
+
+
+# [END aiplatform_sdk_create_and_import_dataset_image_sample]

samples/model-builder/create_and_import_dataset_image_sample_test.py

@@ -0,0 +1,41 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import test_constants as constants
+import create_and_import_dataset_image_sample
+
+from google.cloud.aiplatform import schema
+
+
+def test_create_and_import_dataset_image_sample(
+ mock_sdk_init, mock_create_image_dataset
+):
+
+ create_and_import_dataset_image_sample.create_and_import_dataset_image_sample(
+ project=constants.PROJECT,
+ location=constants.LOCATION,
+ src_uris=constants.GCS_SOURCES,
+ display_name=constants.DISPLAY_NAME,
+ )
+
+ mock_sdk_init.assert_called_once_with(
+ project=constants.PROJECT, location=constants.LOCATION
+ )
+ mock_create_image_dataset.assert_called_once_with(
+ display_name=constants.DISPLAY_NAME,
+ gcs_source=constants.GCS_SOURCES,
+ import_schema_uri=schema.dataset.ioformat.image.single_label_classification,
+ sync=True,
+ )

samples/model-builder/create_batch_prediction_job_sample.py

@@ -0,0 +1,48 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Sequence, Union
+
+from google.cloud import aiplatform
+
+# [START aiplatform_sdk_create_batch_prediction_job_sample]
+def create_batch_prediction_job_sample(
+ project: str,
+ location: str,
+ model_resource_name: str,
+ job_display_name: str,
+ gcs_source: Union[str, Sequence[str]],
+ gcs_destination: str,
+ sync: bool = True,
+):
+ aiplatform.init(project=project, location=location)
+
+ my_model = aiplatform.Model(model_resource_name)
+
+ batch_prediction_job = my_model.batch_predict(
+ job_display_name=job_display_name,
+ gcs_source=gcs_source,
+ gcs_destination_prefix=gcs_destination,
+ sync=sync,
+ )
+
+ batch_prediction_job.wait()
+
+ print(batch_prediction_job.display_name)
+ print(batch_prediction_job.resource_name)
+ print(batch_prediction_job.state)
+ return batch_prediction_job
+
+
+# [END aiplatform_sdk_create_batch_prediction_job_sample]

samples/model-builder/create_batch_prediction_job_sample_test.py

@@ -0,0 +1,42 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import test_constants as constants
+import create_batch_prediction_job_sample
+
+
+def test_create_batch_prediction_job_sample(
+ mock_sdk_init, mock_init_model, mock_batch_predict_model
+):
+
+ create_batch_prediction_job_sample.create_batch_prediction_job_sample(
+ project=constants.PROJECT,
+ location=constants.LOCATION,
+ model_resource_name=constants.MODEL_NAME,
+ job_display_name=constants.DISPLAY_NAME,
+ gcs_source=constants.GCS_SOURCES,
+ gcs_destination=constants.GCS_DESTINATION,
+ )
+
+ mock_sdk_init.assert_called_once_with(
+ project=constants.PROJECT, location=constants.LOCATION
+ )
+ mock_init_model.assert_called_once_with(constants.MODEL_NAME)
+ mock_batch_predict_model.assert_called_once_with(
+ job_display_name=constants.DISPLAY_NAME,
+ gcs_source=constants.GCS_SOURCES,
+ gcs_destination_prefix=constants.GCS_DESTINATION,
+ sync=True,
+ )

samples/model-builder/create_training_pipeline_image_classification_sample.py

@@ -0,0 +1,56 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.cloud import aiplatform
+
+# [START aiplatform_sdk_create_training_pipeline_image_classification_sample]
+def create_training_pipeline_image_classification_sample(
+ project: str,
+ display_name: str,
+ dataset_id: int,
+ location: str = "us-central1",
+ model_display_name: str = None,
+ training_fraction_split: float = 0.8,
+ validation_fraction_split: float = 0.1,
+ test_fraction_split: float = 0.1,
+ budget_milli_node_hours: int = 8000,
+ disable_early_stopping: bool = False,
+ sync: bool = True,
+):
+ aiplatform.init(project=project, location=location)
+
+ job = aiplatform.AutoMLImageTrainingJob(display_name=display_name)
+
+ my_image_ds = aiplatform.Dataset(dataset_id)
+
+ model = job.run(
+ dataset=my_image_ds,
+ model_display_name=model_display_name,
+ training_fraction_split=training_fraction_split,
+ validation_fraction_split=validation_fraction_split,
+ test_fraction_split=test_fraction_split,
+ budget_milli_node_hours=budget_milli_node_hours,
+ disable_early_stopping=disable_early_stopping,
+ sync=sync,
+ )
+
+ model.wait()
+
+ print(model.display_name)
+ print(model.resource_name)
+ print(model.uri)
+ return model
+
+
+# [END aiplatform_sdk_create_training_pipeline_image_classification_sample]