kept overload with Arguments and removed advanced settings

Author: artidoro

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs

@@ -1013,23 +1013,27 @@ private bool HasHeader
  /// <param name="env">The environment to use.</param>
  /// <param name="columns">Defines a mapping between input columns in the file and IDataView columns.</param>
  /// <param name="hasHeader">Whether the file has a header.</param>
- /// <param name="separatorChars">Defines the characters used as separators between data points in a row. By default the tab character is taken as separator.</param>
- /// <param name="advancedSettings">A delegate to apply all the advanced arguments to the algorithm.</param>
+ /// <param name="separatorChar"> The character used as separator between data points in a row. By default the tab character is used as separator.</param>
  /// <param name="dataSample">Allows to expose items that can be used for reading.</param>
- public TextLoader(IHostEnvironment env, Column[] columns, bool hasHeader = false, char[] separatorChars = null, Action<Arguments> advancedSettings = null, IMultiStreamSource dataSample = null)
- : this(env, MakeArgs(columns, hasHeader, separatorChars, advancedSettings), dataSample)
+ public TextLoader(IHostEnvironment env, Column[] columns, bool hasHeader = false, char separatorChar = '\t', IMultiStreamSource dataSample = null)
+ : this(env, MakeArgs(columns, hasHeader, new[] { separatorChar }), dataSample)
  {
  }
 
- private static Arguments MakeArgs(Column[] columns, bool hasHeader, char[] separatorChars, Action<Arguments> advancedSettings)
+ private static Arguments MakeArgs(Column[] columns, bool hasHeader, char[] separatorChars)
  {
  separatorChars = separatorChars ?? new[] { '\t' };
  var result = new Arguments { Column = columns, HasHeader = hasHeader, SeparatorChars = separatorChars};
- advancedSettings?.Invoke(result);
  return result;
  }
 
- internal TextLoader(IHostEnvironment env, Arguments args, IMultiStreamSource dataSample = null)
+ /// <summary>
+ /// Loads a text file into an <see cref="IDataView"/>. Supports basic mapping from input columns to IDataView columns.
+ /// </summary>
+ /// <param name="env">The environment to use.</param>
+ /// <param name="args">Defines the settings of the load operation.</param>
+ /// <param name="dataSample">Allows to expose items that can be used for reading.</param>
+ public TextLoader(IHostEnvironment env, Arguments args, IMultiStreamSource dataSample = null)
  {
  Contracts.CheckValue(env, nameof(env));
  _host = env.Register(RegistrationName);
@@ -1317,11 +1321,19 @@ internal static IDataLoader Create(IHostEnvironment env, Arguments args, IMultiS
  /// <param name="env">The environment to use.</param>
  /// <param name="columns">Defines a mapping between input columns in the file and IDataView columns.</param>
  /// <param name="hasHeader">Whether the file has a header.</param>
- /// <param name="separatorChars">Defines the characters used as separators between data points in a row. By default the tab character is taken as separator.</param>
- /// <param name="advancedSettings">A delegate to apply all the advanced arguments to the algorithm.</param>
+ /// <param name="separatorChar"> The character used as separator between data points in a row. By default the tab character is used as separator.</param>
+ /// <param name="fileSource">Specifies a file from which to read.</param>
+ public static IDataView ReadFile(IHostEnvironment env, IMultiStreamSource fileSource, Column[] columns, bool hasHeader = false, char separatorChar = '\t')
+ => new TextLoader(env, columns, hasHeader, separatorChar, fileSource).Read(fileSource);
+
+ /// <summary>
+ /// Loads a text file into an <see cref="IDataView"/>. Supports basic mapping from input columns to IDataView columns.
+ /// </summary>
+ /// <param name="env">The environment to use.</param>
  /// <param name="fileSource">Specifies a file from which to read.</param>
- public static IDataView ReadFile(IHostEnvironment env, IMultiStreamSource fileSource, Column[] columns, bool hasHeader = false, char[] separatorChars = null, Action<Arguments> advancedSettings = null)
- => new TextLoader(env, columns, hasHeader, separatorChars, advancedSettings, fileSource).Read(fileSource);
+ /// <param name="args">Defines the settings of the load operation.</param>
+ public static IDataView ReadFile(IHostEnvironment env, IMultiStreamSource fileSource, Arguments args)
+ => new TextLoader(env, args, fileSource).Read(fileSource);
 
  public void Save(ModelSaveContext ctx)
  {

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderSaverCatalog.cs

@@ -14,38 +14,63 @@ namespace Microsoft.ML
  public static class TextLoaderSaverCatalog
  {
  /// <summary>
- /// Create a text reader.
+ /// Create a text reader <see cref="TextLoader"/>.
  /// </summary>
  /// <param name="catalog">The catalog.</param>
  /// <param name="columns">The columns of the schema.</param>
  /// <param name="hasHeader">Whether the file has a header.</param>
- /// <param name="separatorChars">Defines the characters used as separators between data points in a row. By default the tab character is taken as separator.</param>
- /// <param name="advancedSettings">The delegate to set additional settings.</param>
+ /// <param name="separatorChar"> The character used as separator between data points in a row. By default the tab character is used as separator.</param>
  /// <param name="dataSample">The optional location of a data sample.</param>
  public static TextLoader TextReader(this DataOperations catalog,
- Column[] columns, bool hasHeader = false, char[] separatorChars = null, Action<Arguments> advancedSettings = null, IMultiStreamSource dataSample = null)
- => new TextLoader(CatalogUtils.GetEnvironment(catalog), columns, hasHeader, separatorChars, advancedSettings, dataSample);
+ Column[] columns, bool hasHeader = false, char separatorChar = '\t', IMultiStreamSource dataSample = null)
+ => new TextLoader(CatalogUtils.GetEnvironment(catalog), columns, hasHeader, separatorChar, dataSample);
+
+ /// <summary>
+ /// Create a text reader <see cref="TextLoader"/>.
+ /// </summary>
+ /// <param name="catalog">The catalog.</param>
+ /// <param name="args">Defines the settings of the load operation.</param>
+ /// <param name="dataSample">Allows to expose items that can be used for reading.</param>
+ public static TextLoader TextReader(this DataOperations catalog, Arguments args, IMultiStreamSource dataSample = null)
+ => new TextLoader(CatalogUtils.GetEnvironment(catalog), args, dataSample);
 
  /// <summary>
  /// Read a data view from a text file using <see cref="TextLoader"/>.
  /// </summary>
  /// <param name="catalog">The catalog.</param>
  /// <param name="columns">The columns of the schema.</param>
  /// <param name="hasHeader">Whether the file has a header.</param>
- /// <param name="separatorChars">Defines the characters used as separators between data points in a row. By default the tab character is taken as separator.</param>
- /// <param name="advancedSettings">The delegate to set additional settings.</param>
+ /// <param name="separatorChar"> The character used as separator between data points in a row. By default the tab character is used as separator.</param>
  /// <param name="path">The path to the file.</param>
  /// <returns>The data view.</returns>
  public static IDataView ReadFromTextFile(this DataOperations catalog,
- string path, Column[] columns, bool hasHeader = false, char[] separatorChars = null, Action<Arguments> advancedSettings = null)
+ string path, Column[] columns, bool hasHeader = false, char separatorChar = '\t')
+ {
+ Contracts.CheckNonEmpty(path, nameof(path));
+
+ var env = catalog.GetEnvironment();
+
+ // REVIEW: it is almost always a mistake to have a 'trainable' text loader here.
+ // Therefore, we are going to disallow data sample.
+ var reader = new TextLoader(env, columns, hasHeader, separatorChar, dataSample: null);
+ return reader.Read(new MultiFileSource(path));
+ }
+
+ /// <summary>
+ /// Read a data view from a text file using <see cref="TextLoader"/>.
+ /// </summary>
+ /// <param name="catalog">The catalog.</param>
+ /// <param name="path">Specifies a file from which to read.</param>
+ /// <param name="args">Defines the settings of the load operation.</param>
+ public static IDataView ReadFromTextFile(this DataOperations catalog, string path, Arguments args)
  {
  Contracts.CheckNonEmpty(path, nameof(path));
 
  var env = catalog.GetEnvironment();
 
  // REVIEW: it is almost always a mistake to have a 'trainable' text loader here.
  // Therefore, we are going to disallow data sample.
- var reader = new TextLoader(env, columns, hasHeader, separatorChars, advancedSettings, dataSample: null);
+ var reader = new TextLoader(env, args, dataSample: null);
  return reader.Read(new MultiFileSource(path));
  }
 

src/Microsoft.ML.Data/Utilities/ModelFileUtils.cs

@@ -283,7 +283,7 @@ public static IEnumerable<KeyValuePair<ColumnRole, string>> LoadRoleMappingsOrNu
  {
  // REVIEW: Should really validate the schema here, and consider
  // ignoring this stream if it isn't as expected.
- var loader = TextLoader.ReadFile(env, new RepositoryStreamWrapper(rep, DirTrainingInfo, RoleMappingFile), null);
+ var loader = TextLoader.ReadFile(env, new RepositoryStreamWrapper(rep, DirTrainingInfo, RoleMappingFile), new TextLoader.Arguments());
 
  using (var cursor = loader.GetRowCursor(c => true))
  {

test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs

@@ -104,7 +104,7 @@ public void SetupBreastCancerPipeline()
  new TextLoader.Column("Label", DataKind.BL, 0),
  new TextLoader.Column("Features", DataKind.R4, new[] { new TextLoader.Range(1, 9) })
  }, 
- false
+ hasHeader: false
  );
 
  IDataView data = reader.Read(_breastCancerDataPath);

test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs

@@ -64,26 +64,29 @@ public void TrainSentiment()
  {
  var env = new MLContext(seed: 1);
  // Pipeline
- var loader = TextLoader.ReadFile(env, new MultiFileSource(_sentimentDataPath),
- columns: new[]
+ var arguemnts = new TextLoader.Arguments()
+ {
+ Column = new TextLoader.Column[]
  {
  new TextLoader.Column()
  {
  Name = "Label",
- Source = new [] { new TextLoader.Range() { Min=0, Max=0} },
+ Source = new[] { new TextLoader.Range() { Min = 0, Max = 0 } },
  Type = DataKind.Num
  },
 
  new TextLoader.Column()
  {
  Name = "SentimentText",
- Source = new [] { new TextLoader.Range() { Min=1, Max=1} },
+ Source = new[] { new TextLoader.Range() { Min = 1, Max = 1 } },
  Type = DataKind.Text
  }
  },
- hasHeader: true,
- advancedSettings: s => { s.AllowQuoting = false; s.AllowSparse = false; }
- );
+ HasHeader = true,
+ AllowQuoting = false,
+ AllowSparse = false
+ };
+ var loader = TextLoader.ReadFile(env, new MultiFileSource(_sentimentDataPath), arguemnts);
 
  var text = TextFeaturizingEstimator.Create(env,
  new TextFeaturizingEstimator.Arguments()

test/Microsoft.ML.Predictor.Tests/TestPredictors.cs

@@ -606,7 +606,7 @@ public void RankingLightGBMTest()
  public void TestTreeEnsembleCombiner()
  {
  var dataPath = GetDataPath("breast-cancer.txt");
- var dataView = TextLoader.ReadFile(Env, new MultiFileSource(dataPath), null);
+ var dataView = TextLoader.ReadFile(Env, new MultiFileSource(dataPath), new TextLoader.Arguments());
 
  var fastTrees = new IPredictorModel[3];
  for (int i = 0; i < 3; i++)
@@ -628,7 +628,7 @@ public void TestTreeEnsembleCombiner()
  public void TestTreeEnsembleCombinerWithCategoricalSplits()
  {
  var dataPath = GetDataPath("adult.tiny.with-schema.txt");
- var dataView = TextLoader.ReadFile(Env, new MultiFileSource(dataPath), null);
+ var dataView = TextLoader.ReadFile(Env, new MultiFileSource(dataPath), new TextLoader.Arguments());
 
  var cat = new OneHotEncodingEstimator(Env, "Categories", "Features").Fit(dataView).Transform(dataView);
  var fastTrees = new IPredictorModel[3];
@@ -729,7 +729,7 @@ private void CombineAndTestTreeEnsembles(IDataView idv, IPredictorModel[] fastTr
  public void TestEnsembleCombiner()
  {
  var dataPath = GetDataPath("breast-cancer.txt");
- var dataView = TextLoader.ReadFile(Env, new MultiFileSource(dataPath), null);
+ var dataView = TextLoader.ReadFile(Env, new MultiFileSource(dataPath), new TextLoader.Arguments());
 
  var predictors = new IPredictorModel[]
  {
@@ -775,7 +775,7 @@ public void TestEnsembleCombiner()
  public void TestMultiClassEnsembleCombiner()
  {
  var dataPath = GetDataPath("breast-cancer.txt");
- var dataView = TextLoader.ReadFile(Env, new MultiFileSource(dataPath), null);
+ var dataView = TextLoader.ReadFile(Env, new MultiFileSource(dataPath), new TextLoader.Arguments());
 
  var predictors = new IPredictorModel[]
  {

test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs

@@ -439,7 +439,7 @@ protected bool SaveLoadText(IDataView view, IHostEnvironment env,
 
  // Note that we don't pass in "args", but pass in a default args so we test
  // the auto-schema parsing.
- var loadedData = TextLoader.ReadFile(env, new MultiFileSource(pathData), null);
+ var loadedData = TextLoader.ReadFile(env, new MultiFileSource(pathData), new TextLoader.Arguments());
  if (!CheckMetadataTypes(loadedData.Schema))
  Failed();
 

test/Microsoft.ML.TestFramework/ModelHelper.cs

@@ -67,7 +67,7 @@ public static IDataView GetKcHouseDataView(string dataPath)
  new Runtime.Data.TextLoader.Column("SqftLot15", Runtime.Data.DataKind.R4, 20)
  }, 
  hasHeader: true,
- separatorChars: new[] { ',' }
+ separatorChar: ','
  );
  }
 

test/Microsoft.ML.Tests/Scenarios/Api/ApiScenariosTests.cs

@@ -64,11 +64,6 @@ private static TextLoader.Column[] MakeIrisColumns()
  };
  }
 
- private static char[] MakeIrisSeparator()
- {
- return new[] { ',' };
- }
-
  private static TextLoader.Column[] MakeSentimentColumns()
  {
  return new[]

test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs

@@ -99,9 +99,9 @@ private void TrainRegression(string trainDataPath, string testDataPath, string m
  new TextLoader.Column("Target", DataKind.R4, 11),
  },
  // First line of the file is a header, not a data row.
- true,
+ hasHeader: true,
  // Default separator is tab, but we need a semicolon.
- new[] { ';' }
+ separatorChar: ';'
  );
 
  // Now read the file (remember though, readers are lazy, so the actual reading will happen when the data is accessed).
@@ -161,7 +161,7 @@ private ITransformer TrainOnIris(string irisDataPath)
  new TextLoader.Column("Label", DataKind.TX, 4),
  },
  // Default separator is tab, but the dataset has comma.
- separatorChars: new[] { ',' }
+ separatorChar: ','
  );
 
  // Retrieve the training data.
@@ -224,7 +224,7 @@ private void NormalizationWorkout(string dataPath)
  new TextLoader.Column("Label", DataKind.TX, 4),
  },
  // Default separator is tab, but the dataset has comma.
- separatorChars: new[] { ',' }
+ separatorChar: ','
  );
 
  // Read the training data.
@@ -283,7 +283,7 @@ private void TextFeaturizationOn(string dataPath)
  new TextLoader.Column("IsToxic", DataKind.BL, 0),
  new TextLoader.Column("Message", DataKind.TX, 1),
  },
- true
+ hasHeader: true
  );
 
  // Read the data.
@@ -355,7 +355,7 @@ private void CategoricalFeaturizationOn(params string[] dataPath)
  // Let's also separately load the 'Workclass' column.
  new TextLoader.Column("Workclass", DataKind.TX, 1),
  },
- true
+ hasHeader: true
  );
 
  // Read the data.
@@ -417,7 +417,7 @@ private void CrossValidationOn(string dataPath)
  new TextLoader.Column("Label", DataKind.TX, 4),
  },
  // Default separator is tab, but the dataset has comma.
- separatorChars: new[] { ',' }
+ separatorChar: ','
  );
 
  // Read the data.
@@ -470,7 +470,7 @@ private void ReadDataDynamic(string dataPath)
  new TextLoader.Column("Target", DataKind.R4, 10)
  },
  // Default separator is tab, but we need a comma.
- separatorChars: new[] { ',' });
+ separatorChar: ',' );
 
  // Now read the file (remember though, readers are lazy, so the actual reading will happen when the data is accessed).
  var data = reader.Read(dataPath);
@@ -496,7 +496,7 @@ public void CustomTransformer()
  {
  new TextLoader.Column("Income", DataKind.R4, 10),
  new TextLoader.Column("Features", DataKind.R4, 12, 14)
- }, true);
+ }, hasHeader: true);
 
  PrepareData(mlContext, data);
  TrainModel(mlContext, data);