First public api propsal (#52)

* Includes following 1) Final proposal for 0.1 public API surface 2) Prefeaturization 3) Splitting train data into train and validate when validation data is null 4) Providing end to end samples one each for regression, binaryclassification and multiclass classification * Incorporating code review feedbacks

Author: vinodshanbhag

src/AutoML/API/AutoFitSettings.cs

@@ -7,29 +7,38 @@
 
 namespace Microsoft.ML.Auto
 {
- public class AutoFitSettings
+ internal static class AutoFitDefaults
  {
+ public const uint TimeOutInMinutes = 24 * 60;
+ public const uint MaxIterations = 1000;
+ }
+
+ internal class AutoFitSettings
+ {
+ // All the following settings only capture the surface area of capabilities we want to ship in future.
+ // However, most certainly they will not ship using following types and structures
+ // These should remain internal until we have rationalized 
+
  public ExperimentStoppingCriteria StoppingCriteria = new ExperimentStoppingCriteria();
  internal IterationStoppingCriteria IterationStoppingCriteria;
  internal Concurrency Concurrency;
  internal Filters Filters;
  internal CrossValidationSettings CrossValidationSettings;
  internal OptimizingMetric OptimizingMetric;
- internal bool EnableEnsembling;
- internal bool EnableModelExplainability;
- internal bool EnableAutoTransformation;
+ internal bool DisableEnsembling;
+ internal bool CaclculateModelExplainability;
+ internal bool DisableFeaturization;
 
- // spec question: Are following automatic or a user setting?
- internal bool EnableSubSampling;
- internal bool EnableCaching;
+ internal bool DisableSubSampling;
+ internal bool DisableCaching;
  internal bool ExternalizeTraining;
- internal TraceLevel TraceLevel; // Should this be controlled through code or appconfig?
+ internal TraceLevel TraceLevel; 
  }
 
- public class ExperimentStoppingCriteria
+ internal class ExperimentStoppingCriteria
  {
- public int MaxIterations = 100;
- public int TimeOutInMinutes = 300;
+ public uint TimeOutInMinutes = AutoFitDefaults.TimeOutInMinutes;
+ public uint MaxIterations = AutoFitDefaults.MaxIterations;
  internal bool StopAfterConverging;
  internal double ExperimentExitScore;
  }
@@ -40,19 +49,20 @@ internal class Filters
  internal IEnumerable<Trainers> BlackListTrainers;
  internal IEnumerable<Transformers> WhitelistTransformers;
  internal IEnumerable<Transformers> BlacklistTransformers;
- internal bool PreferExplainability;
- internal bool PreferInferenceSpeed;
- internal bool PreferSmallDeploymentSize;
- internal bool PreferSmallMemoryFootprint;
+ internal uint? Explainability;
+ internal uint? InferenceSpeed;
+ internal uint? DeploymentSize;
+ internal uint? TrainingMemorySize;
+ internal bool? GpuTraining;
  }
 
- public class IterationStoppingCriteria
+ internal class IterationStoppingCriteria
  {
  internal int TimeOutInSeconds;
  internal bool TerminateOnLowAccuracy;
  }
 
- public class Concurrency
+ internal class Concurrency
  {
  internal int MaxConcurrentIterations;
  internal int MaxCoresPerIteration;

src/AutoML/API/MLContextAutoFitExtensions.cs

@@ -14,32 +14,42 @@ public static class RegressionExtensions
  {
  public static RegressionResult AutoFit(this RegressionContext context,
  IDataView trainData,
- string label,
- IDataView validationData,
- AutoFitSettings settings = null,
- IEnumerable<(string, ColumnPurpose)> purposeOverrides = null,
+ string label = DefaultColumnNames.Label,
+ IDataView validationData = null,
+ uint timeoutInMinutes = AutoFitDefaults.TimeOutInMinutes,
+ IEstimator<ITransformer> preFeaturizers = null,
+ IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
  CancellationToken cancellationToken = default,
  IProgress<RegressionIterationResult> iterationCallback = null)
  {
+ var settings = new AutoFitSettings();
+ settings.StoppingCriteria.TimeOutInMinutes = timeoutInMinutes;
+
  return AutoFit(context, trainData, label, validationData, settings,
- purposeOverrides, cancellationToken, iterationCallback, null);
+ preFeaturizers, columnPurposes, cancellationToken, iterationCallback, null);
  }
 
  internal static RegressionResult AutoFit(this RegressionContext context,
  IDataView trainData,
- string label,
- IDataView validationData,
+ string label = DefaultColumnNames.Label,
+ IDataView validationData = null,
  AutoFitSettings settings = null,
- IEnumerable<(string, ColumnPurpose)> purposeOverrides = null,
+ IEstimator<ITransformer> preFeaturizers = null,
+ IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
  CancellationToken cancellationToken = default,
  IProgress<RegressionIterationResult> iterationCallback = null,
  IDebugLogger debugLogger = null)
  {
- UserInputValidationUtil.ValidateAutoFitArgs(trainData, label, validationData, settings, purposeOverrides);
+ UserInputValidationUtil.ValidateAutoFitArgs(trainData, label, validationData, settings, columnPurposes);
+
+ if (validationData == null)
+ {
+ (trainData, validationData) = context.TestValidateSplit(trainData);
+ }
 
  // run autofit & get all pipelines run in that process
  var (allPipelines, bestPipeline) = AutoFitApi.Fit(trainData, validationData, label,
- settings, TaskKind.Regression, OptimizingMetric.RSquared, purposeOverrides, debugLogger);
+ settings, preFeaturizers, TaskKind.Regression, OptimizingMetric.RSquared, columnPurposes, debugLogger);
 
  var results = new RegressionIterationResult[allPipelines.Length];
  for (var i = 0; i < results.Length; i++)
@@ -57,33 +67,43 @@ public static class BinaryClassificationExtensions
  {
  public static BinaryClassificationResult AutoFit(this BinaryClassificationContext context,
  IDataView trainData,
- string label,
- IDataView validationData,
- AutoFitSettings settings = null,
- IEnumerable<(string, ColumnPurpose)> purposeOverrides = null,
+ string label = DefaultColumnNames.Label,
+ IDataView validationData = null,
+ uint timeoutInMinutes = AutoFitDefaults.TimeOutInMinutes,
+ IEstimator<ITransformer> preFeaturizers = null,
+ IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
  CancellationToken cancellationToken = default,
  IProgress<BinaryClassificationItertionResult> iterationCallback = null)
  {
+ var settings = new AutoFitSettings();
+ settings.StoppingCriteria.TimeOutInMinutes = timeoutInMinutes;
+
  return AutoFit(context, trainData, label, validationData, settings,
- purposeOverrides, cancellationToken, iterationCallback, null);
+ preFeaturizers, columnPurposes, cancellationToken, iterationCallback, null);
  }
 
  internal static BinaryClassificationResult AutoFit(this BinaryClassificationContext context,
  IDataView trainData,
- string label,
- IDataView validationData,
+ string label = DefaultColumnNames.Label,
+ IDataView validationData = null,
  AutoFitSettings settings = null,
- IEnumerable<(string, ColumnPurpose)> purposeOverrides = null,
+ IEstimator<ITransformer> preFeaturizers = null,
+ IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
  CancellationToken cancellationToken = default,
  IProgress<BinaryClassificationItertionResult> iterationCallback = null,
  IDebugLogger debugLogger = null)
  {
- UserInputValidationUtil.ValidateAutoFitArgs(trainData, label, validationData, settings, purposeOverrides);
+ UserInputValidationUtil.ValidateAutoFitArgs(trainData, label, validationData, settings, columnPurposes);
+
+ if (validationData == null)
+ {
+ (trainData, validationData) = context.TestValidateSplit(trainData);
+ }
 
  // run autofit & get all pipelines run in that process
  var (allPipelines, bestPipeline) = AutoFitApi.Fit(trainData, validationData, label,
- settings, TaskKind.BinaryClassification, OptimizingMetric.Accuracy,
- purposeOverrides, debugLogger);
+ settings, preFeaturizers, TaskKind.BinaryClassification, OptimizingMetric.Accuracy,
+ columnPurposes, debugLogger);
 
  var results = new BinaryClassificationItertionResult[allPipelines.Length];
  for (var i = 0; i < results.Length; i++)
@@ -101,32 +121,42 @@ public static class MulticlassExtensions
  {
  public static MulticlassClassificationResult AutoFit(this MulticlassClassificationContext context,
  IDataView trainData,
- string label,
- IDataView validationData,
- AutoFitSettings settings = null,
- IEnumerable<(string, ColumnPurpose)> purposeOverrides = null,
+ string label = DefaultColumnNames.Label,
+ IDataView validationData = null,
+ uint timeoutInMinutes = AutoFitDefaults.TimeOutInMinutes,
+ IEstimator<ITransformer> preFeaturizers = null,
+ IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
  CancellationToken cancellationToken = default,
  IProgress<MulticlassClassificationIterationResult> iterationCallback = null)
  {
+ var settings = new AutoFitSettings();
+ settings.StoppingCriteria.TimeOutInMinutes = timeoutInMinutes;
+
  return AutoFit(context, trainData, label, validationData, settings,
- purposeOverrides, cancellationToken, iterationCallback, null);
+ preFeaturizers, columnPurposes, cancellationToken, iterationCallback, null);
  }
 
  internal static MulticlassClassificationResult AutoFit(this MulticlassClassificationContext context,
  IDataView trainData,
- string label,
- IDataView validationData,
+ string label = DefaultColumnNames.Label,
+ IDataView validationData = null,
  AutoFitSettings settings = null,
- IEnumerable<(string, ColumnPurpose)> purposeOverrides = null,
+ IEstimator<ITransformer> preFeaturizers = null,
+ IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
  CancellationToken cancellationToken = default,
  IProgress<MulticlassClassificationIterationResult> iterationCallback = null, IDebugLogger debugLogger = null)
  {
- UserInputValidationUtil.ValidateAutoFitArgs(trainData, label, validationData, settings, purposeOverrides);
+ UserInputValidationUtil.ValidateAutoFitArgs(trainData, label, validationData, settings, columnPurposes);
+
+ if (validationData == null)
+ {
+ (trainData, validationData) = context.TestValidateSplit(trainData);
+ }
 
  // run autofit & get all pipelines run in that process
  var (allPipelines, bestPipeline) = AutoFitApi.Fit(trainData, validationData, label,
- settings, TaskKind.MulticlassClassification, OptimizingMetric.Accuracy,
- purposeOverrides, debugLogger);
+ settings, preFeaturizers, TaskKind.MulticlassClassification, OptimizingMetric.Accuracy,
+ columnPurposes, debugLogger);
 
  var results = new MulticlassClassificationIterationResult[allPipelines.Length];
  for (var i = 0; i < results.Length; i++)
@@ -142,39 +172,39 @@ internal static MulticlassClassificationResult AutoFit(this MulticlassClassifica
 
  public class BinaryClassificationResult
  {
- public readonly BinaryClassificationItertionResult BestPipeline;
+ public readonly BinaryClassificationItertionResult BestIteration;
  public readonly BinaryClassificationItertionResult[] IterationResults;
 
  public BinaryClassificationResult(BinaryClassificationItertionResult bestPipeline,
  BinaryClassificationItertionResult[] iterationResults)
  {
- BestPipeline = bestPipeline;
+ BestIteration = bestPipeline;
  IterationResults = iterationResults;
  }
  }
 
  public class MulticlassClassificationResult
  {
- public readonly MulticlassClassificationIterationResult BestPipeline;
+ public readonly MulticlassClassificationIterationResult BestIteration;
  public readonly MulticlassClassificationIterationResult[] IterationResults;
 
  public MulticlassClassificationResult(MulticlassClassificationIterationResult bestPipeline,
  MulticlassClassificationIterationResult[] iterationResults)
  {
- BestPipeline = bestPipeline;
+ BestIteration = bestPipeline;
  IterationResults = iterationResults;
  }
  }
 
  public class RegressionResult
  {
- public readonly RegressionIterationResult BestPipeline;
+ public readonly RegressionIterationResult BestIteration;
  public readonly RegressionIterationResult[] IterationResults;
 
  public RegressionResult(RegressionIterationResult bestPipeline,
  RegressionIterationResult[] iterationResults)
  {
- BestPipeline = bestPipeline;
+ BestIteration = bestPipeline;
  IterationResults = iterationResults;
  }
  }

src/AutoML/AutoFitter/AutoFitApi.cs

@@ -4,19 +4,29 @@
 
 using System.Collections.Generic;
 using System.Linq;
+using Microsoft.ML.Core.Data;
 using Microsoft.ML.Data;
 
 namespace Microsoft.ML.Auto
 {
  internal static class AutoFitApi
  {
  public static (InferredPipelineRunResult[] allPipelines, InferredPipelineRunResult bestPipeline) Fit(IDataView trainData, 
- IDataView validationData, string label, AutoFitSettings settings, TaskKind task, OptimizingMetric metric, 
+ IDataView validationData, string label, AutoFitSettings settings, IEstimator<ITransformer> preFeaturizers, TaskKind task, OptimizingMetric metric, 
  IEnumerable<(string, ColumnPurpose)> purposeOverrides, IDebugLogger debugLogger)
  {
  // hack: init new MLContext
  var mlContext = new MLContext();
 
+ ITransformer preprocessorTransform = null;
+ if (preFeaturizers != null)
+ {
+ // preprocess train and validation data
+ preprocessorTransform = preFeaturizers.Fit(trainData);
+ trainData = preprocessorTransform.Transform(trainData);
+ validationData = preprocessorTransform.Transform(validationData);
+ }
+
  var purposeOverridesDict = purposeOverrides?.ToDictionary(p => p.Item1, p => p.Item2);
  var optimizingMetricfInfo = new OptimizingMetricInfo(metric);
 
@@ -25,6 +35,15 @@ public static (InferredPipelineRunResult[] allPipelines, InferredPipelineRunResu
  label, trainData, validationData, purposeOverridesDict, debugLogger);
  var allPipelines = autoFitter.Fit();
 
+ // apply preprocessor to returned models
+ if (preprocessorTransform != null)
+ {
+ for (var i = 0; i < allPipelines.Length; i++)
+ {
+ allPipelines[i].Model = preprocessorTransform.Append(allPipelines[i].Model);
+ }
+ }
+
  var bestScore = allPipelines.Max(p => p.Score);
  var bestPipeline = allPipelines.First(p => p.Score == bestScore);
 

src/AutoML/AutoFitter/AutoFitter.cs

@@ -48,7 +48,7 @@ public InferredPipelineRunResult[] Fit()
  do
  {
  // get next pipeline
- var iterationsRemaining = _settings.StoppingCriteria.MaxIterations - _history.Count;
+ var iterationsRemaining = (int)_settings.StoppingCriteria.MaxIterations - _history.Count;
  var pipeline = PipelineSuggester.GetNextInferredPipeline(_history, columns, _task, iterationsRemaining, _optimizingMetricInfo.IsMaximizing);
 
  // break if no candidates returned, means no valid pipeline available

src/AutoML/AutoMlUtils.cs

@@ -30,6 +30,20 @@ public static IDataView Take(this IDataView data, int count)
  return new CacheDataView(context, filter, Enumerable.Range(0, data.Schema.Count).ToArray());
  }
 
+ public static IDataView DropLastColumn(this IDataView data)
+ {
+ return new MLContext().Transforms.DropColumns(data.Schema[data.Schema.Count - 1].Name).Fit(data).Transform(data);
+ }
+
+ public static (IDataView testData, IDataView validationData) TestValidateSplit(this TrainContextBase context, IDataView trainData)
+ {
+ IDataView validationData;
+ (trainData, validationData) = context.TrainTestSplit(trainData);
+ trainData = trainData.DropLastColumn();
+ validationData = validationData.DropLastColumn();
+ return (trainData, validationData);
+ }
+
  public static IDataView Skip(this IDataView data, int count)
  {
  var context = new MLContext();

src/AutoML/Utils/UserInputValidationUtil.cs

@@ -112,7 +112,7 @@ private static void ValidateValidationData(IDataView trainData, IDataView valida
  {
  if(validationData == null)
  {
- throw new ArgumentNullException("Validation data cannot be null", nameof(validationData));
+ return;
  }
 
  const string schemaMismatchError = "Training data and validation data schemas do not match.";