Updated CopyColumns, DropColumns and SelectColumns samples. (#3268)

Author: zeahmed

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CopyColumns.cs

@@ -12,72 +12,58 @@ public static void Example()
  // as well as the source of randomness.
  var mlContext = new MLContext();
 
- // Get a small dataset as an IEnumerable and them read it as ML.NET's data type.
- IEnumerable<Microsoft.ML.SamplesUtils.DatasetUtils.SampleInfertData> data = Microsoft.ML.SamplesUtils.DatasetUtils.GetInfertData();
- var trainData = mlContext.Data.LoadFromEnumerable(data);
+ // Create a small dataset as an IEnumerable.
+ var samples = new List<InputData>()
+ {
+ new InputData(){ ImageId = 1, Features = new [] { 1.0f, 1.0f, 1.0f} },
+ new InputData(){ ImageId = 2, Features = new [] { 2.0f, 2.0f, 2.0f} },
+ new InputData(){ ImageId = 3, Features = new [] { 3.0f, 3.0f, 3.0f} },
+ new InputData(){ ImageId = 4, Features = new [] { 4.0f, 4.0f, 4.0f} },
+ new InputData(){ ImageId = 5, Features = new [] { 5.0f, 5.0f, 5.0f} },
+ new InputData(){ ImageId = 6, Features = new [] { 6.0f, 6.0f, 6.0f} },
+ };
 
- // Preview of the data.
- //
- // Age Case Education induced parity pooled.stratum row_num ...
- // 26.0 1.0 0-5yrs 1.0 6.0 3.0 1.0 ...
- // 42.0 1.0 0-5yrs 1.0 1.0 1.0 2.0 ...
- // 39.0 1.0 0-5yrs 2.0 6.0 4.0 3.0 ...
- // 34.0 1.0 0-5yrs 2.0 4.0 2.0 4.0 ...
- // 35.0 1.0 6-11yrs 1.0 3.0 32.0 5.0 ...
+ // Convert training data to IDataView.
+ var dataview = mlContext.Data.LoadFromEnumerable(samples);
 
  // CopyColumns is commonly used to rename columns.
- // For example, if you want to train towards Age, and your learner expects a "Label" column, you can
- // use CopyColumns to rename Age to Label. Technically, the Age columns still exists, but it won't be
+ // For example, if you want to train towards ImageId, and your trainer expects a "Label" column, you can
+ // use CopyColumns to rename ImageId to Label. Technically, the ImageId column still exists, but it won't be
  // materialized unless you actually need it somewhere (e.g. if you were to save the transformed data
  // without explicitly dropping the column). This is a general property of IDataView's lazy evaluation.
- string labelColumnName = "Label";
- var pipeline = mlContext.Transforms.CopyColumns(labelColumnName, "Age") as IEstimator<ITransformer>;
-
- // You also may want to copy a column to perform some hand-featurization using built-in transforms or
- // a CustomMapping transform. For example, we could make an indicator variable if a feature, such as Parity
- // goes above some threshold. We simply copy the Parity column to a new column, then pass it through a custom function.
- Action<InputRow, OutputRow> mapping = (input, output) =>output.CustomValue = input.CustomValue > 4 ? 1 : 0;
- pipeline = pipeline.Append(mlContext.Transforms.CopyColumns("CustomValue", "Parity"))
- .Append(mlContext.Transforms.CustomMapping(mapping, null));
+ var pipeline = mlContext.Transforms.CopyColumns("Label", "ImageId");
 
  // Now we can transform the data and look at the output to confirm the behavior of CopyColumns.
  // Don't forget that this operation doesn't actually evaluate data until we read the data below.
- var transformedData = pipeline.Fit(trainData).Transform(trainData);
+ var transformedData = pipeline.Fit(dataview).Transform(dataview);
 
  // We can extract the newly created column as an IEnumerable of SampleInfertDataTransformed, the class we define below.
- var rowEnumerable = mlContext.Data.CreateEnumerable<SampleInfertDataTransformed>(transformedData, reuseRowObject: false);
+ var rowEnumerable = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
 
  // And finally, we can write out the rows of the dataset, looking at the columns of interest.
- Console.WriteLine($"Label, Parity, and CustomValue columns obtained post-transformation.");
+ Console.WriteLine($"Label and ImageId columns obtained post-transformation.");
  foreach (var row in rowEnumerable)
- {
- Console.WriteLine($"Label: {row.Label} Parity: {row.Parity} CustomValue: {row.CustomValue}");
- }
+ Console.WriteLine($"Label: {row.Label} ImageId: {row.ImageId}");
 
  // Expected output:
- // Label, Parity, and CustomValue columns obtained post-transformation.
- // Label: 26 Parity: 6 CustomValue: 1
- // Label: 42 Parity: 1 CustomValue: 0
- // Label: 39 Parity: 6 CustomValue: 1
- // Label: 34 Parity: 4 CustomValue: 0
- // Label: 35 Parity: 3 CustomValue: 0
- }
-
- private class SampleInfertDataTransformed
- {
- public float Label { get; set; }
- public float Parity { get; set; }
- public float CustomValue { get; set; }
+ // ImageId and Label columns obtained post-transformation.
+ // Label: 1 ImageId: 1
+ // Label: 2 ImageId: 2
+ // Label: 3 ImageId: 3
+ // Label: 4 ImageId: 4
+ // Label: 5 ImageId: 5
+ // Label: 6 ImageId: 6
  }
 
- private class OutputRow
+ private class InputData
  {
- public float CustomValue { get; set; }
+ public int ImageId { get; set; }
+ public float[] Features { get; set; }
  }
 
- private class InputRow
+ private class TransformedData : InputData
  {
- public float CustomValue { get; set; }
+ public int Label { get; set; }
  }
  }
 }

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/DropColumns.cs

@@ -12,71 +12,73 @@ public static void Example()
  // as well as the source of randomness.
  var mlContext = new MLContext();
 
- // Get a small dataset as an IEnumerable and them read it as ML.NET's data type.
- IEnumerable<Microsoft.ML.SamplesUtils.DatasetUtils.SampleInfertData> data = Microsoft.ML.SamplesUtils.DatasetUtils.GetInfertData();
- var trainData = mlContext.Data.LoadFromEnumerable(data);
+ // Create a small dataset as an IEnumerable.
+ var samples = new List<InputData>()
+ {
+ new InputData(){ Age = 21, Gender = "Male", Education = "BS", ExtraColumn = 1 },
+ new InputData(){ Age = 23, Gender = "Female", Education = "MBA", ExtraColumn = 2 },
+ new InputData(){ Age = 28, Gender = "Male", Education = "PhD", ExtraColumn = 3 },
+ new InputData(){ Age = 22, Gender = "Male", Education = "BS", ExtraColumn = 4 },
+ new InputData(){ Age = 23, Gender = "Female", Education = "MS", ExtraColumn = 5 },
+ new InputData(){ Age = 27, Gender = "Female", Education = "PhD", ExtraColumn = 6 },
+ };
 
- // Preview of the data.
- //
- // Age Case Education Induced Parity Pooled.stratum Row_num ...
- // 26.0 1.0 0-5yrs 1.0 6.0 3.0 1.0 ...
- // 42.0 1.0 0-5yrs 1.0 1.0 1.0 2.0 ...
- // 39.0 1.0 0-5yrs 2.0 6.0 4.0 3.0 ...
- // 34.0 1.0 0-5yrs 2.0 4.0 2.0 4.0 ...
- // 35.0 1.0 6-11yrs 1.0 3.0 32.0 5.0 ...
+ // Convert training data to IDataView.
+ var dataview = mlContext.Data.LoadFromEnumerable(samples);
 
- // Drop the Age and Education columns from the dataset.
- var pipeline = mlContext.Transforms.DropColumns("Age", "Education");
+ // Drop the ExtraColumn from the dataset.
+ var pipeline = mlContext.Transforms.DropColumns("ExtraColumn");
 
  // Now we can transform the data and look at the output.
  // Don't forget that this operation doesn't actually operate on data until we perform an action that requires 
  // the data to be materialized.
- var transformedData = pipeline.Fit(trainData).Transform(trainData);
+ var transformedData = pipeline.Fit(dataview).Transform(dataview);
 
  // Now let's take a look at what the DropColumns operations did.
- // We can extract the transformed data as an IEnumerable of SampleInfertDataNonExistentColumns, the class we define below.
- // When we try to pull out the Age and Education columns, ML.NET will raise an exception on the first non-existent column
- // that it tries to access. 
+ // We can extract the transformed data as an IEnumerable of InputData, the class we define below.
+ // When we try to pull out the Age, Gender, Education and ExtraColumn columns, ML.NET will raise an exception on the ExtraColumn
  try
  {
- var failingRowEnumerable = mlContext.Data.CreateEnumerable<SampleInfertDataNonExistentColumns>(transformedData, reuseRowObject: false);
- } catch(ArgumentOutOfRangeException exception)
+ var failingRowEnumerable = mlContext.Data.CreateEnumerable<InputData>(transformedData, reuseRowObject: false);
+ }
+ catch (ArgumentOutOfRangeException exception)
  {
- Console.WriteLine($"Age and Education were not available, so an exception was thrown: {exception.Message}.");
+ Console.WriteLine($"ExtraColumn is not available, so an exception is thrown: {exception.Message}.");
  }
 
  // Expected output:
- // Age and Education were not available, so an exception was thrown: Could not find column 'Age'.
+ // ExtraColumn is not available, so an exception is thrown: Could not find column 'ExtraColumn'.
  // Parameter name: Schema
 
  // And we can write a few columns out to see that the rest of the data is still available.
- var rowEnumerable = mlContext.Data.CreateEnumerable<SampleInfertDataTransformed>(transformedData, reuseRowObject: false);
+ var rowEnumerable = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
  Console.WriteLine($"The columns we didn't drop are still available.");
  foreach (var row in rowEnumerable)
- {
- Console.WriteLine($"Case: {row.Case} Induced: {row.Induced} Parity: {row.Parity}");
- }
+ Console.WriteLine($"Age: {row.Age} Gender: {row.Gender} Education: {row.Education}");
 
  // Expected output:
  // The columns we didn't drop are still available.
- // Case: 1 Induced: 1 Parity: 6
- // Case: 1 Induced: 1 Parity: 1
- // Case: 1 Induced: 2 Parity: 6
- // Case: 1 Induced: 2 Parity: 4
- // Case: 1 Induced: 1 Parity: 3
+ // Age: 21 Gender: Male Education: BS
+ // Age: 23 Gender: Female Education: MBA
+ // Age: 28 Gender: Male Education: PhD
+ // Age: 22 Gender: Male Education: BS
+ // Age: 23 Gender: Female Education: MS
+ // Age: 27 Gender: Female Education: PhD
  }
 
- private class SampleInfertDataNonExistentColumns
+ private class InputData
  {
- public float Age { get; set; }
- public float Education { get; set; }
+ public int Age { get; set; }
+ public string Gender { get; set; }
+ public string Education { get; set; }
+ public float ExtraColumn { get; set; }
  }
 
- private class SampleInfertDataTransformed
+ private class TransformedData
  {
- public float Case { get; set; }
- public float Induced { get; set; }
- public float Parity { get; set; }
+ public int Age { get; set; }
+ public string Gender { get; set; }
+ public string Education { get; set; }
  }
  }
 }

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/SelectColumns.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using Microsoft.ML;
 
 namespace Samples.Dynamic
@@ -11,61 +12,63 @@ public static void Example()
  // as well as the source of randomness.
  var mlContext = new MLContext();
 
- // Get a small dataset as an IEnumerable and them read it as ML.NET's data type.
- var enumerableData = Microsoft.ML.SamplesUtils.DatasetUtils.GetInfertData();
- var data = mlContext.Data.LoadFromEnumerable(enumerableData);
-
- // Before transformation, take a look at the dataset
- Console.WriteLine($"Age\tCase\tEducation\tInduced\tParity\tPooledStratum");
- foreach (var row in enumerableData)
+ // Create a small dataset as an IEnumerable.
+ var samples = new List<InputData>()
  {
- Console.WriteLine($"{row.Age}\t{row.Case}\t{row.Education}\t{row.Induced}\t{row.Parity}\t{row.PooledStratum}");
- }
- Console.WriteLine();
- // Expected output:
- // Age Case Education  Induced Parity PooledStratum
- // 26 1 0 - 5yrs 1 6 3
- // 42 1 0 - 5yrs 1 1 1
- // 39 1 12 + yrs 2 6 4
- //  34 1 0 - 5yrs 2 4 2
- // 35 1 6 - 11yrs 1 3 32
+ new InputData(){ Age = 21, Gender = "Male", Education = "BS", ExtraColumn = 1 },
+  new InputData(){ Age = 23, Gender = "Female", Education = "MBA", ExtraColumn = 2 },
+  new InputData(){ Age = 28, Gender = "Male", Education = "PhD", ExtraColumn = 3 },
+  new InputData(){ Age = 22, Gender = "Male", Education = "BS", ExtraColumn = 4 },
+  new InputData(){ Age = 23, Gender = "Female", Education = "MS", ExtraColumn = 5 },
+  new InputData(){ Age = 27, Gender = "Female", Education = "PhD", ExtraColumn = 6 },
+ };
+
+ // Convert training data to IDataView.
+ var dataview = mlContext.Data.LoadFromEnumerable(samples);
 
  // Select a subset of columns to keep.
  var pipeline = mlContext.Transforms.SelectColumns("Age", "Education");
 
- // Now we can transform the data and look at the output to confirm the behavior of CopyColumns.
+ // Now we can transform the data and look at the output to confirm the behavior of SelectColumns.
  // Don't forget that this operation doesn't actually evaluate data until we read the data below,
  // as transformations are lazy in ML.NET.
- var transformedData = pipeline.Fit(data).Transform(data);
+ var transformedData = pipeline.Fit(dataview).Transform(dataview);
 
  // Print the number of columns in the schema
  Console.WriteLine($"There are {transformedData.Schema.Count} columns in the dataset.");
 
  // Expected output:
  // There are 2 columns in the dataset.
 
- // We can extract the newly created column as an IEnumerable of SampleInfertDataTransformed, the class we define below.
- var rowEnumerable = mlContext.Data.CreateEnumerable<SampleInfertDataTransformed>(transformedData, reuseRowObject: false);
+ // We can extract the newly created column as an IEnumerable of TransformedData, the class we define below.
+ var rowEnumerable = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
 
  // And finally, we can write out the rows of the dataset, looking at the columns of interest.
  Console.WriteLine($"Age and Educations columns obtained post-transformation.");
  foreach (var row in rowEnumerable)
- {
  Console.WriteLine($"Age: {row.Age} Education: {row.Education}");
- }
 
  // Expected output:
- // Age and Education columns obtained post-transformation.
- // Age: 26 Education: 0-5yrs
- // Age: 42 Education: 0-5yrs
- // Age: 39 Education: 12+yrs
- // Age: 34 Education: 0-5yrs
- // Age: 35 Education: 6-11yrs
+ // Age and Educations columns obtained post-transformation.
+ // Age: 21 Education: BS
+ // Age: 23 Education: MBA
+ // Age: 28 Education: PhD
+ // Age: 22 Education: BS
+ // Age: 23 Education: MS
+ // Age: 27 Education: PhD
+ }
+
+ private class InputData
+ {
+ public int Age { get; set; }
+ public string Gender { get; set; }
+ public string Education { get; set; }
+ public float ExtraColumn { get; set; }
  }
 
- private class SampleInfertDataTransformed
+ private class TransformedData
  {
- public float Age { get; set; }
+ public int Age { get; set; }
  public string Education { get; set; }
  }
  }