[AutoML] CLI telemetry rev (#3789)

Author: daholste

src/Microsoft.ML.AutoML/ColumnInference/ColumnInformationUtil.cs

@@ -107,6 +107,24 @@ public static IEnumerable<string> GetColumnNames(ColumnInformation columnInforma
  return columnNames;
  }
 
+ public static IDictionary<ColumnPurpose, int> CountColumnsByPurpose(ColumnInformation columnInformation)
+ {
+ var result = new Dictionary<ColumnPurpose, int>();
+ var columnNames = GetColumnNames(columnInformation);
+ foreach (var columnName in columnNames)
+ {
+ var purpose = columnInformation.GetColumnPurpose(columnName);
+ if (purpose == null)
+ {
+ continue;
+ }
+
+ result.TryGetValue(purpose.Value, out int count);
+ result[purpose.Value] = ++count;
+ }
+ return result;
+ }
+
  private static void AddStringsToListIfNotNull(List<string> list, IEnumerable<string> strings)
  {
  foreach (var str in strings)

src/Microsoft.ML.AutoML/TrainerExtensions/SweepableParams.cs

@@ -61,6 +61,11 @@ private static IEnumerable<SweepableParam> BuildLbfgsArgsParams()
  };
  }
 
+ /// <summary>
+ /// The names of every hyperparameter swept across all trainers.
+ /// </summary>
+ public static ISet<string> AllHyperparameterNames = GetAllSweepableParameterNames();
+
  public static IEnumerable<SweepableParam> BuildAveragePerceptronParams()
  {
  return BuildAveragedLinearArgsParams().Concat(BuildOnlineLinearArgsParams());
@@ -172,5 +177,29 @@ public static IEnumerable<SweepableParam> BuildSymSgdLogisticRegressionParams()
  new SweepableDiscreteParam("UpdateFrequency", new object[] { "<Auto>", 5, 20 })
  };
  }
+
+ /// <summary>
+ /// Gets the name of every hyperparameter swept across all trainers.
+ /// </summary>
+ public static ISet<string> GetAllSweepableParameterNames()
+ {
+ var sweepableParams = new List<SweepableParam>();
+ sweepableParams.AddRange(BuildAveragePerceptronParams());
+ sweepableParams.AddRange(BuildAveragePerceptronParams());
+ sweepableParams.AddRange(BuildFastForestParams());
+ sweepableParams.AddRange(BuildFastTreeParams());
+ sweepableParams.AddRange(BuildFastTreeTweedieParams());
+ sweepableParams.AddRange(BuildLightGbmParamsMulticlass());
+ sweepableParams.AddRange(BuildLightGbmParams());
+ sweepableParams.AddRange(BuildLinearSvmParams());
+ sweepableParams.AddRange(BuildLbfgsLogisticRegressionParams());
+ sweepableParams.AddRange(BuildOnlineGradientDescentParams());
+ sweepableParams.AddRange(BuildLbfgsPoissonRegressionParams());
+ sweepableParams.AddRange(BuildSdcaParams());
+ sweepableParams.AddRange(BuildOlsParams());
+ sweepableParams.AddRange(BuildSgdParams());
+ sweepableParams.AddRange(BuildSymSgdLogisticRegressionParams());
+ return new HashSet<string>(sweepableParams.Select(p => p.Name));
+ }
  }
 }

src/mlnet/CodeGenerator/CodeGenerationHelper.cs

@@ -12,6 +12,7 @@
 using Microsoft.ML.CLI.CodeGenerator.CSharp;
 using Microsoft.ML.CLI.Data;
 using Microsoft.ML.CLI.ShellProgressBar;
+using Microsoft.ML.CLI.Telemetry.Events;
 using Microsoft.ML.CLI.Utilities;
 using Microsoft.ML.Data;
 using NLog;
@@ -51,7 +52,9 @@ public void GenerateCode()
  {
  inputColumnInformation.IgnoredColumnNames.Add(value);
  }
+ var inferColumnsStopwatch = Stopwatch.StartNew();
  columnInference = _automlEngine.InferColumns(context, inputColumnInformation);
+ InferColumnsEvent.TrackEvent(columnInference.ColumnInformation, inferColumnsStopwatch.Elapsed);
  }
  catch (Exception)
  {
@@ -74,6 +77,9 @@ public void GenerateCode()
  // The reason why we are doing this way of defining 3 different results is because of the AutoML API
  // i.e there is no common class/interface to handle all three tasks together.
 
+ // Start a timer for the experiment
+ var stopwatch = Stopwatch.StartNew();
+
  List<RunDetail<BinaryClassificationMetrics>> completedBinaryRuns = new List<RunDetail<BinaryClassificationMetrics>>();
  List<RunDetail<MulticlassClassificationMetrics>> completedMulticlassRuns = new List<RunDetail<MulticlassClassificationMetrics>>();
  List<RunDetail<RegressionMetrics>> completedRegressionRuns = new List<RunDetail<RegressionMetrics>>();
@@ -236,6 +242,7 @@ public void GenerateCode()
  {
  var binaryMetric = new BinaryExperimentSettings().OptimizingMetric;
  var bestBinaryIteration = BestResultUtil.GetBestRun(completedBinaryRuns, binaryMetric);
+ ExperimentCompletedEvent.TrackEvent(bestBinaryIteration, completedBinaryRuns, TaskKind.BinaryClassification, stopwatch.Elapsed);
  bestPipeline = bestBinaryIteration.Pipeline;
  bestModel = bestBinaryIteration.Model;
  ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, _settings.MlTask, _settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), completedBinaryRuns.Count());
@@ -253,6 +260,7 @@ public void GenerateCode()
  {
  var regressionMetric = new RegressionExperimentSettings().OptimizingMetric;
  var bestRegressionIteration = BestResultUtil.GetBestRun(completedRegressionRuns, regressionMetric);
+ ExperimentCompletedEvent.TrackEvent(bestRegressionIteration, completedRegressionRuns, TaskKind.Regression, stopwatch.Elapsed);
  bestPipeline = bestRegressionIteration.Pipeline;
  bestModel = bestRegressionIteration.Model;
  ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, _settings.MlTask, _settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), completedRegressionRuns.Count());
@@ -270,6 +278,7 @@ public void GenerateCode()
  {
  var muliclassMetric = new MulticlassExperimentSettings().OptimizingMetric;
  var bestMulticlassIteration = BestResultUtil.GetBestRun(completedMulticlassRuns, muliclassMetric);
+ ExperimentCompletedEvent.TrackEvent(bestMulticlassIteration, completedMulticlassRuns, TaskKind.MulticlassClassification, stopwatch.Elapsed);
  bestPipeline = bestMulticlassIteration.Pipeline;
  bestModel = bestMulticlassIteration.Model;
  ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, _settings.MlTask, _settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), completedMulticlassRuns.Count());

src/mlnet/Commands/CommandDefinitions.cs

@@ -14,9 +14,11 @@ namespace Microsoft.ML.CLI.Commands
 {
  internal static class CommandDefinitions
  {
+ public const string AutoTrainCommandName = "auto-train";
+
  internal static System.CommandLine.Command AutoTrain(ICommandHandler handler)
  {
- var newCommand = new System.CommandLine.Command("auto-train", "Create a new .NET project using ML.NET to train and run a model", handler: handler)
+ var newCommand = new System.CommandLine.Command(AutoTrainCommandName, "Create a new .NET project using ML.NET to train and run a model", handler: handler)
  {
  MlTask(),
  Dataset(),

src/mlnet/Commands/New/NewCommandHandler.cs

@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Microsoft.DotNet.Cli.Telemetry;
 using Microsoft.ML.CLI.CodeGenerator;
 using Microsoft.ML.CLI.Data;
 
@@ -11,18 +10,14 @@ namespace Microsoft.ML.CLI.Commands.New
  internal class NewCommand : ICommand
  {
  private readonly NewCommandSettings _settings;
- private readonly MlTelemetry _telemetry;
 
- internal NewCommand(NewCommandSettings settings, MlTelemetry telemetry)
+ internal NewCommand(NewCommandSettings settings)
  {
  _settings = settings;
- _telemetry = telemetry;
  }
 
  public void Execute()
  {
- _telemetry.LogAutoTrainMlCommand(_settings.Dataset.Name, _settings.MlTask.ToString(), _settings.Dataset.Length);
-
  CodeGenerationHelper codeGenerationHelper = new CodeGenerationHelper(new AutoMLEngine(_settings), _settings); // Needs to be improved.
  codeGenerationHelper.GenerateCode();
  }

src/mlnet/Program.cs

@@ -5,12 +5,13 @@
 using System;
 using System.CommandLine.Builder;
 using System.CommandLine.Invocation;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
-using Microsoft.DotNet.Cli.Telemetry;
 using Microsoft.ML.CLI.Commands;
 using Microsoft.ML.CLI.Commands.New;
 using Microsoft.ML.CLI.Data;
+using Microsoft.ML.CLI.Telemetry.Events;
 using Microsoft.ML.CLI.Utilities;
 using NLog;
 using NLog.Targets;
@@ -20,24 +21,33 @@ namespace Microsoft.ML.CLI
  public class Program
  {
  private static Logger _logger = LogManager.GetCurrentClassLogger();
+
  public static void Main(string[] args)
  {
- var telemetry = new MlTelemetry();
+ Telemetry.Telemetry.Initialize();
  int exitCode = 1;
+ Exception ex = null;
+ var stopwatch = Stopwatch.StartNew();
+
+ var mlNetCommandEvent = new MLNetCommandEvent();
+
  // Create handler outside so that commandline and the handler is decoupled and testable.
  var handler = CommandHandler.Create<NewCommandSettings>(
  (options) =>
  {
  try
  {
+ // Send telemetry event for command issued
+ mlNetCommandEvent.AutoTrainCommandSettings = options;
+ mlNetCommandEvent.TrackEvent();
+
  // Map the verbosity to internal levels
  var verbosity = Utils.GetVerbosity(options.Verbosity);
 
  // Build the output path
  string outputBaseDir = string.Empty;
  if (options.Name == null)
  {
-
  options.Name = "Sample" + Utils.GetTaskKind(options.MlTask).ToString();
  outputBaseDir = Path.Combine(options.OutputPath.FullName, options.Name);
  }
@@ -50,7 +60,7 @@ public static void Main(string[] args)
  options.OutputPath = new DirectoryInfo(outputBaseDir);
 
  // Instantiate the command
- var command = new NewCommand(options, telemetry);
+ var command = new NewCommand(options);
 
  // Override the Logger Configuration
  var logconsole = LogManager.Configuration.FindTargetByName("logconsole");
@@ -67,6 +77,7 @@ public static void Main(string[] args)
  }
  catch (Exception e)
  {
+ ex = e;
  _logger.Log(LogLevel.Error, e.Message);
  _logger.Log(LogLevel.Debug, e.ToString());
  _logger.Log(LogLevel.Info, Strings.LookIntoLogFile);
@@ -82,7 +93,8 @@ public static void Main(string[] args)
 
  var parseResult = parser.Parse(args);
 
- if (parseResult.Errors.Count == 0)
+ var commandParseSucceeded = !parseResult.Errors.Any();
+ if (commandParseSucceeded)
  {
  if (parseResult.RootCommandResult.Children.Count > 0)
  {
@@ -95,13 +107,20 @@ public static void Main(string[] args)
 
  var explicitlySpecifiedOptions = options.Where(opt => !opt.IsImplicit).Select(opt => opt.Name);
 
- telemetry.SetCommandAndParameters(command.Name, explicitlySpecifiedOptions);
+ mlNetCommandEvent.CommandLineParametersUsed = explicitlySpecifiedOptions;
  }
  }
  }
 
+ // Send system info telemetry
+ SystemInfoEvent.TrackEvent();
+
  parser.InvokeAsync(parseResult).Wait();
+ // Send exit telemetry
+ ApplicationExitEvent.TrackEvent(exitCode, commandParseSucceeded, stopwatch.Elapsed, ex);
+ // Flush pending telemetry logs
+ Telemetry.Telemetry.Flush(TimeSpan.FromSeconds(3));
  Environment.Exit(exitCode);
  }
  }
-}
+}

src/mlnet/Telemetry/DotNetAppInsights/FirstTimeUseNoticeSentinel.cs

@@ -2,8 +2,8 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System.IO;
-using Microsoft.DotNet.AutoML;
 using Microsoft.Extensions.EnvironmentAbstractions;
+using Microsoft.ML.CLI.Telemetry;
 
 namespace Microsoft.DotNet.Configurer
 {

src/mlnet/Telemetry/DotNetAppInsights/TelemetryCommonProperties.cs

@@ -4,8 +4,8 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using Microsoft.DotNet.AutoML;
 using Microsoft.DotNet.Configurer;
+using Microsoft.ML.CLI.Telemetry;
 using RuntimeEnvironment = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment;
 using RuntimeInformation = System.Runtime.InteropServices.RuntimeInformation;
 

src/mlnet/Telemetry/DotNetAppInsights/UserLevelCacheWriter.cs

@@ -3,8 +3,8 @@
 
 using System;
 using System.IO;
-using Microsoft.DotNet.AutoML;
 using Microsoft.Extensions.EnvironmentAbstractions;
+using Microsoft.ML.CLI.Telemetry;
 
 namespace Microsoft.DotNet.Configurer
 {

src/mlnet/Telemetry/Events/ApplicationExitEvent.cs

@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace Microsoft.ML.CLI.Telemetry.Events
+{
+ /// <summary>
+ /// Telemetry event for CLI application exit.
+ /// </summary>
+ internal class ApplicationExitEvent
+ {
+ public static void TrackEvent(int exitCode, bool commandParseSucceeded, TimeSpan duration, Exception ex)
+ {
+ Telemetry.TrackEvent("application-exit",
+ new Dictionary<string, string>
+ {
+ { "CommandParseSucceeded", commandParseSucceeded.ToString() },
+ { "ExitCode", exitCode.ToString() },
+ { "PeakMemory", Process.GetCurrentProcess().PeakWorkingSet64.ToString() },
+ },
+ duration, ex);
+ }
+ }
+}