large refactor for latest autogluon version (aws#1289)

Author: austinmw

advanced_functionality/autogluon-tabular/AutoGluon_Tabular_SageMaker.ipynb

@@ -49,6 +49,7 @@
  "import sagemaker\n",
  "from time import sleep\n",
  "from collections import Counter\n",
+ "import numpy as np\n",
  "import pandas as pd\n",
  "from sagemaker import get_execution_role, local, Model, utils, fw_utils, s3\n",
  "from sagemaker.estimator import Estimator\n",
@@ -106,12 +107,10 @@
  "source": [
  "if not os.path.exists('package'):\n",
  " !pip install PrettyTable -t package\n",
- " !pip install bokeh -t package\n",
- " !pip install --pre autogluon -t package\n",
- " !pip install numpy==1.16.1 -t package \n",
  " !pip install --upgrade boto3 -t package\n",
  " !pip install bokeh -t package\n",
- " !pip install --upgrade matplotlib -t package"
+ " !pip install --upgrade matplotlib -t package\n",
+ " !pip install autogluon -t package"
  ]
  },
  {
@@ -255,80 +254,83 @@
  "Collapsed": "false"
  },
  "source": [
- "## Train"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "Collapsed": "false"
- },
- "source": [
- "The minimum requirement for hyperparameters is a target label."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "Collapsed": "false"
- },
- "outputs": [],
- "source": [
- "hyperparameters = {'label': 'y'}"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "Collapsed": "false"
- },
- "source": [
- "##### (Optional) hyperparameters can be passed to the `autogluon.task.TabularPrediction.fit` function. \n",
+ "## Hyperparameter Selection\n",
  "\n",
- "Below shows AutoGluon hyperparameters from the example [Predicting Columns in a Table - In Depth](https://autogluon.mxnet.io/tutorials/tabular_prediction/tabular-indepth.html#model-ensembling-with-stacking-bagging). Please see [fit parameters](https://autogluon.mxnet.io/api/autogluon.task.html?highlight=eval_metric#autogluon.task.TabularPrediction.fit) for further information.\n",
+ "The minimum required settings for training is just a target label, `fit_args['label']`.\n",
  "\n",
+ "Additional optional hyperparameters can be passed to the `autogluon.task.TabularPrediction.fit` function via `fit_args`.\n",
  "\n",
- "Here's a more in depth example from the above tutorial that shows how to provide hyperparameter ranges and additional settings:\n",
+ "Below shows a more in depth example of AutoGluon-Tabular hyperparameters from the example [Predicting Columns in a Table - In Depth](https://autogluon.mxnet.io/tutorials/tabular_prediction/tabular-indepth.html#model-ensembling-with-stacking-bagging). Please see [fit parameters](https://autogluon.mxnet.io/api/autogluon.task.html?highlight=eval_metric#autogluon.task.TabularPrediction.fit) for further information. Note that in order for hyperparameter ranges to work in SageMaker, values passed to the `fit_args['hyperparameters']` must be represented as strings.\n",
  "\n",
  "```python\n",
  "nn_options = {\n",
- " 'num_epochs': '10',\n",
+ " 'num_epochs': \"10\",\n",
  " 'learning_rate': \"ag.space.Real(1e-4, 1e-2, default=5e-4, log=True)\",\n",
  " 'activation': \"ag.space.Categorical('relu', 'softrelu', 'tanh')\",\n",
  " 'layers': \"ag.space.Categorical([100],[1000],[200,100],[300,200,100])\",\n",
  " 'dropout_prob': \"ag.space.Real(0.0, 0.5, default=0.1)\"\n",
  "}\n",
  "\n",
  "gbm_options = {\n",
- " 'num_boost_round': '100',\n",
+ " 'num_boost_round': \"100\",\n",
  " 'num_leaves': \"ag.space.Int(lower=26, upper=66, default=36)\"\n",
  "}\n",
  "\n",
  "model_hps = {'NN': nn_options, 'GBM': gbm_options} \n",
  "\n",
+ "fit_args = {\n",
+ " 'label': 'y',\n",
+ " 'presets': ['best_quality', 'optimize_for_deployment'],\n",
+ " 'time_limits': 60*10,\n",
+ " 'hyperparameters': model_hps,\n",
+ " 'hyperparameter_tune': True,\n",
+ " 'search_strategy': 'skopt'\n",
+ "}\n",
+ "\n",
  "hyperparameters = {\n",
- " 'label': 'y',\n",
- " 'time_limits': 2*60,\n",
- " 'hyperparameters': model_hps,\n",
- " 'auto_stack': False, \n",
- " 'hyperparameter_tune': True,\n",
- " 'search_strategy': 'skopt'\n",
+ " 'fit_args': fit_args,\n",
+ " 'feature_importance': True\n",
  "}\n",
  "```\n",
- "**Note:** Your hyperparameter choices may affect the size of the model package, which could result in additional time taken to upload your model and complete training.\n",
+ "**Note:** Your hyperparameter choices may affect the size of the model package, which could result in additional time taken to upload your model and complete training. Including `'optimize_for_deployment'` in the list of `fit_args['presets']` is recommended to greatly reduce upload times.\n",
  "\n",
  "<br>"
  ]
  },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "Collapsed": "false"
+ },
+ "outputs": [],
+ "source": [
+ "# Define required label and optional additional parameters\n",
+ "fit_args = {\n",
+ " 'label': 'y',\n",
+ " # Adding 'best_quality' to presets list will result in better performance (but longer runtime)\n",
+ " 'presets': ['optimize_for_deployment'],\n",
+ "}\n",
+ "\n",
+ "# Pass fit_args to SageMaker estimator hyperparameters\n",
+ "hyperparameters = {\n",
+ " 'fit_args': fit_args,\n",
+ " 'feature_importance': True\n",
+ "}"
+ ]
+ },
  {
  "cell_type": "markdown",
  "metadata": {
  "Collapsed": "false"
  },
  "source": [
- "For local training set `train_instance_type` to `local` . \n",
- "For non-local training the recommended instance type is `ml.m5.2xlarge` ."
+ "## Train\n",
+ "\n",
+ "For local training set `train_instance_type` to `local` . \n",
+ "For non-local training the recommended instance type is `ml.m5.2xlarge`. \n",
+ "\n",
+ "**Note:** Depending on how many underlying models are trained, `train_volume_size` may need to be increased so that they all fit on disk."
  ]
  },
  {
@@ -350,9 +352,13 @@
  " role=role,\n",
  " train_instance_count=1,\n",
  " train_instance_type=instance_type,\n",
- " hyperparameters=hyperparameters)\n",
+ " hyperparameters=hyperparameters,\n",
+ " train_volume_size=100)\n",
  "\n",
- "estimator.fit(train_s3_path)"
+ "# Set inputs. Test data is optional, but requires a label column.\n",
+ "inputs = {'training': train_s3_path, 'testing': test_s3_path}\n",
+ "\n",
+ "estimator.fit(inputs)"
  ]
  },
  {
@@ -516,10 +522,10 @@
  },
  "outputs": [],
  "source": [
- "results = predictor.predict(X_test.to_csv())\n",
+ "results = predictor.predict(X_test.to_csv()).splitlines()\n",
  "\n",
  "# Check output\n",
- "print(Counter(results.splitlines()))"
+ "print(Counter(results))"
  ]
  },
  {
@@ -540,10 +546,10 @@
  },
  "outputs": [],
  "source": [
- "results = predictor.predict(test.to_csv())\n",
+ "results = predictor.predict(test.to_csv()).splitlines()\n",
  "\n",
  "# Check output\n",
- "sleep(0.1); print(Counter(results.splitlines()))"
+ "print(Counter(results))"
  ]
  },
  {
@@ -552,7 +558,7 @@
  "Collapsed": "false"
  },
  "source": [
- "##### Check that performance metrics match evaluation printed to endpoint logs as expected"
+ "##### Check that classification performance metrics match evaluation printed to endpoint logs as expected"
  ]
  },
  {
@@ -563,8 +569,7 @@
  },
  "outputs": [],
  "source": [
- "import numpy as np\n",
- "y_results = np.array(results.splitlines())\n",
+ "y_results = np.array(results)\n",
  "\n",
  "print(\"accuracy: {}\".format(accuracy_score(y_true=y_test, y_pred=y_results)))\n",
  "print(classification_report(y_true=y_test, y_pred=y_results, digits=6))"
@@ -593,7 +598,7 @@
  ],
  "metadata": {
  "kernelspec": {
- "display_name": "conda_mxnet_p36",
+ "display_name": "Environment (conda_mxnet_p36)",
  "language": "python",
  "name": "conda_mxnet_p36"
  },

advanced_functionality/autogluon-tabular/container-training/inference.py

@@ -3,31 +3,37 @@
 import argparse
 import logging
 import warnings
-import os
+import time
 import json
 import subprocess
 
-warnings.filterwarnings("ignore",category=FutureWarning)
+warnings.filterwarnings('ignore', category=FutureWarning)
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '/opt/ml/code/package'))
 
+import numpy as np
 import pandas as pd
 import pickle
 from io import StringIO
 from timeit import default_timer as timer
+from itertools import islice
 from collections import Counter
 
 with warnings.catch_warnings():
- warnings.filterwarnings("ignore",category=DeprecationWarning)
+ warnings.filterwarnings('ignore', category=DeprecationWarning)
  from prettytable import PrettyTable
  from autogluon import TabularPrediction as task
 
-def make_str_table(df): 
+def make_str_table(df):
  table = PrettyTable(['index']+list(df.columns))
  for row in df.itertuples():
  table.add_row(row)
  return str(table)
 
+def take(n, iterable):
+ "Return first n items of the iterable as a list"
+ return list(islice(iterable, n))
+
 # ------------------------------------------------------------ #
 # Hosting methods #
 # ------------------------------------------------------------ #
@@ -39,7 +45,7 @@ def model_fn(model_dir):
  :return: a model (in this case a Gluon network)
  """
  print(f'Loading model from {model_dir} with contents {os.listdir(model_dir)}')
- net = task.load(model_dir, verbosity=True) 
+ net = task.load(model_dir, verbosity=True)
  return net
 
 
@@ -53,37 +59,56 @@ def transform_fn(net, data, input_content_type, output_content_type):
  :return: response payload and content type.
  """
  start = timer()
- 
+
  # text/csv
  if input_content_type == 'text/csv':
- 
+
  # Load dataset
  df = pd.read_csv(StringIO(data))
  ds = task.Dataset(df=df)
-
- # Predict
- predictions = net.predict(ds)
- print(f'Prediction counts: {Counter(predictions.tolist())}')
 
+ try:
+ predictions = net.predict(ds)
+ except:
+ try:
+ predictions = net.predict(ds.fillna(0.0))
+ warnings.warn('Filled NaN\'s with 0.0 in order to predict.')
+ except Exception as e:
+ response_body = e
+ return response_body, output_content_type
+ 
+ # Print prediction counts, limit in case of regression problem
+ pred_counts = Counter(predictions.tolist())
+ n_display_items = 30
+ if len(pred_counts) > n_display_items:
+ print(f'Top {n_display_items} prediction counts: '
+ f'{dict(take(n_display_items, pred_counts.items()))}')
+ else:
+ print(f'Prediction counts: {pred_counts}')
+
  # Form response
  output = StringIO()
  pd.DataFrame(predictions).to_csv(output, header=False, index=False)
- response_body = output.getvalue()  
- 
+ response_body = output.getvalue() 
+
  # If target column passed, evaluate predictions performance
  target = net.label_column
  if target in ds:
  print(f'Label column ({target}) found in input data. '
- 'Therefore, evaluating prediction performance...')
-
- performance = net.evaluate_predictions(y_true=ds[target], y_pred=predictions, 
- auxiliary_metrics=True)
- print(json.dumps(performance, indent=4)) 
- 
- else: 
+ 'Therefore, evaluating prediction performance...') 
+ try:
+ performance = net.evaluate_predictions(y_true=ds[target], 
+ y_pred=predictions, 
+ auxiliary_metrics=True) 
+ print(json.dumps(performance, indent=4))
+ time.sleep(0.1)
+ except Exception as e:
+ # Print exceptions on evaluate, continue to return predictions
+ print(f'Exception: {e}')
+ else:
  raise NotImplementedError("content_type must be 'text/csv'")
 
  elapsed_time = round(timer()-start,3)
  print(f'Elapsed time: {round(timer()-start,3)} seconds') 
 
- return response_body, output_content_type
+ return response_body, output_content_type