Towards 1529: replacing the predicates with an IEnumerable on IRowToRowMapper.GetDependencies (#2504)

* towards 1529: replacing the predicates with an IEnumerable on IRowToRowMapper.GetDependencies * ISchemaBoundRowMapper does not inherit from IRowToRowMapper, but from IRowToRowMapperBase. Renaming ISchemaBoundRowMapper.GetDependencies to GetDependenciesForNewColumns

Author: sfilipi

src/Microsoft.ML.Core/Data/IRowToRowMapper.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Collections.Generic;
 using Microsoft.Data.DataView;
 
 namespace Microsoft.ML.Data
@@ -28,11 +29,9 @@ public interface IRowToRowMapper
  DataViewSchema OutputSchema { get; }
 
  /// <summary>
- /// Given a predicate specifying which columns are needed, return a predicate indicating which input columns are
- /// needed. The domain of the function is defined over the indices of the columns of <see cref="DataViewSchema.Count"/>
- /// for <see cref="InputSchema"/>.
+ /// Given a set of columns, return the input columns that are needed to generate those output columns.
  /// </summary>
- Func<int, bool> GetDependencies(Func<int, bool> predicate);
+ IEnumerable<DataViewSchema.Column> GetDependencies(IEnumerable<DataViewSchema.Column> dependingColumns);
 
  /// <summary>
  /// Get an <see cref="DataViewRow"/> with the indicated active columns, based on the input <paramref name="input"/>.

src/Microsoft.ML.Core/Data/ISchemaBindableMapper.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System;
 using System.Collections.Generic;
 using Microsoft.Data.DataView;
 
@@ -55,15 +56,36 @@ internal interface ISchemaBoundMapper
  }
 
  /// <summary>
- /// This interface combines <see cref="ISchemaBoundMapper"/> with <see cref="IRowToRowMapper"/>.
+ /// This interface extends <see cref="ISchemaBoundMapper"/>.
  /// </summary>
  [BestFriend]
- internal interface ISchemaBoundRowMapper : ISchemaBoundMapper, IRowToRowMapper
+ internal interface ISchemaBoundRowMapper : ISchemaBoundMapper
  {
  /// <summary>
- /// There are two schemas from <see cref="ISchemaBoundMapper"/> and <see cref="IRowToRowMapper"/>.
- /// Since the two parent schema's are identical in all derived classes, we merge them into <see cref="OutputSchema"/>.
+ /// Input schema accepted.
  /// </summary>
- new DataViewSchema OutputSchema { get; }
+ DataViewSchema InputSchema { get; }
+
+ /// <summary>
+ /// Given a set of columns, from the newly generated ones, return the input columns that are needed to generate those output columns.
+ /// </summary>
+ IEnumerable<DataViewSchema.Column> GetDependenciesForNewColumns(IEnumerable<DataViewSchema.Column> dependingColumns);
+
+ /// <summary>
+ /// Get an <see cref="DataViewRow"/> with the indicated active columns, based on the input <paramref name="input"/>.
+ /// The active columns are those for which <paramref name="active"/> returns true. Getting values on inactive
+ /// columns of the returned row will throw. Null predicates are disallowed.
+ ///
+ /// The <see cref="DataViewRow.Schema"/> of <paramref name="input"/> should be the same object as
+ /// <see cref="InputSchema"/>. Implementors of this method should throw if that is not the case. Conversely,
+ /// the returned value must have the same schema as <see cref="ISchemaBoundMapper.OutputSchema"/>.
+ ///
+ /// This method creates a live connection between the input <see cref="DataViewRow"/> and the output <see
+ /// cref="DataViewRow"/>. In particular, when the getters of the output <see cref="DataViewRow"/> are invoked, they invoke the
+ /// getters of the input row and base the output values on the current values of the input <see cref="DataViewRow"/>.
+ /// The output <see cref="DataViewRow"/> values are re-computed when requested through the getters. Also, the returned
+ /// <see cref="DataViewRow"/> will dispose <paramref name="input"/> when it is disposed.
+ /// </summary>
+ DataViewRow GetRow(DataViewRow input, Func<int, bool> active);
  }
 }

src/Microsoft.ML.Data/DataView/CompositeRowToRowMapper.cs

@@ -3,6 +3,8 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using Microsoft.Data.DataView;
 using Microsoft.ML.Internal.Utilities;
 
@@ -36,12 +38,15 @@ public CompositeRowToRowMapper(DataViewSchema inputSchema, IRowToRowMapper[] map
  OutputSchema = Utils.Size(mappers) > 0 ? mappers[mappers.Length - 1].OutputSchema : inputSchema;
  }
 
- public Func<int, bool> GetDependencies(Func<int, bool> predicate)
+ /// <summary>
+ /// Given a set of columns, return the input columns that are needed to generate those output columns.
+ /// </summary>
+ IEnumerable<DataViewSchema.Column> IRowToRowMapper.GetDependencies(IEnumerable<DataViewSchema.Column> columnsNeeded)
  {
- Func<int, bool> toReturn = predicate;
  for (int i = InnerMappers.Length - 1; i >= 0; --i)
- toReturn = InnerMappers[i].GetDependencies(toReturn);
- return toReturn;
+ columnsNeeded = InnerMappers[i].GetDependencies(columnsNeeded);
+
+ return columnsNeeded;
  }
 
  public DataViewRow GetRow(DataViewRow input, Func<int, bool> active)
@@ -71,7 +76,11 @@ public DataViewRow GetRow(DataViewRow input, Func<int, bool> active)
  var deps = new Func<int, bool>[InnerMappers.Length];
  deps[deps.Length - 1] = active;
  for (int i = deps.Length - 1; i >= 1; --i)
- deps[i - 1] = InnerMappers[i].GetDependencies(deps[i]);
+ {
+ var outputColumns = InnerMappers[i].OutputSchema.Where(c => deps[i](c.Index));
+ var cols = InnerMappers[i].GetDependencies(outputColumns).ToArray();
+ deps[i - 1] = c => cols.Length > 0 ? cols.Any(col => col.Index == c) : false;
+ }
 
  DataViewRow result = input;
  for (int i = 0; i < InnerMappers.Length; ++i)

src/Microsoft.ML.Data/DataView/RowToRowMapperTransform.cs

@@ -101,7 +101,6 @@ public RowToRowMapperTransform(IHostEnvironment env, IDataView input, IRowMapper
  _mapperFactory = mapperFactory;
  _bindings = new ColumnBindings(input.Schema, mapper.GetOutputColumns());
  }
-
  public static DataViewSchema GetOutputSchema(DataViewSchema inputSchema, IRowMapper mapper)
  {
  Contracts.CheckValue(inputSchema, nameof(inputSchema));
@@ -143,10 +142,9 @@ private protected override void SaveModel(ModelSaveContext ctx)
 
  /// <summary>
  /// Produces the set of active columns for the data view (as a bool[] of length bindings.ColumnCount),
- /// a predicate for the needed active input columns, and a predicate for the needed active
- /// output columns.
+ /// and the needed active input columns, given a predicate for the needed active output columns.
  /// </summary>
- private bool[] GetActive(Func<int, bool> predicate, out Func<int, bool> predicateInput)
+ private bool[] GetActive(Func<int, bool> predicate, out IEnumerable<DataViewSchema.Column> inputColumns)
  {
  int n = _bindings.Schema.Count;
  var active = Utils.BuildArray(n, predicate);
@@ -162,8 +160,7 @@ private bool[] GetActive(Func<int, bool> predicate, out Func<int, bool> predicat
  var predicateIn = _mapper.GetDependencies(predicateOut);
 
  // Combine the two sets of input columns.
- predicateInput =
- col => 0 <= col && col < activeInput.Length && (activeInput[col] || predicateIn(col));
+ inputColumns = _bindings.InputSchema.Where(col => activeInput[col.Index] || predicateIn(col.Index));
 
  return active;
  }
@@ -192,10 +189,7 @@ private Func<int, bool> GetActiveOutputColumns(bool[] active)
  protected override DataViewRowCursor GetRowCursorCore(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
  {
  var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, OutputSchema);
-
- Func<int, bool> predicateInput;
- var active = GetActive(predicate, out predicateInput);
- var inputCols = Source.Schema.Where(x => predicateInput(x.Index));
+ var active = GetActive(predicate, out IEnumerable<DataViewSchema.Column> inputCols);
 
  return new Cursor(Host, Source.GetRowCursor(inputCols, rand), this, active);
  }
@@ -205,11 +199,8 @@ public override DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.C
  Host.CheckValueOrNull(rand);
 
  var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, OutputSchema);
+ var active = GetActive(predicate, out IEnumerable<DataViewSchema.Column> inputCols);
 
- Func<int, bool> predicateInput;
- var active = GetActive(predicate, out predicateInput);
-
- var inputCols = Source.Schema.Where(x => predicateInput(x.Index));
  var inputs = Source.GetRowCursorSet(inputCols, n, rand);
  Host.AssertNonEmpty(inputs);
 
@@ -243,11 +234,14 @@ void ISaveAsPfa.SaveAsPfa(BoundPfaContext ctx)
  }
  }
 
- public Func<int, bool> GetDependencies(Func<int, bool> predicate)
+ /// <summary>
+ /// Given a set of output columns, return the input columns that are needed to generate those output columns.
+ /// </summary>
+ IEnumerable<DataViewSchema.Column> IRowToRowMapper.GetDependencies(IEnumerable<DataViewSchema.Column> dependingColumns)
  {
- Func<int, bool> predicateInput;
- GetActive(predicate, out predicateInput);
- return predicateInput;
+ var predicate = RowCursorUtils.FromColumnsToPredicate(dependingColumns, OutputSchema);
+ GetActive(predicate, out var inputColumns);
+ return inputColumns;
  }
 
  public DataViewSchema InputSchema => Source.Schema;

src/Microsoft.ML.Data/EntryPoints/TransformModelImpl.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using Microsoft.Data.DataView;
 using Microsoft.ML.Data;
 using Microsoft.ML.Data.IO;
@@ -222,20 +223,23 @@ public static bool IsCompositeRowToRowMapper(IDataView chain)
  return true;
  }
 
- public Func<int, bool> GetDependencies(Func<int, bool> predicate)
+ /// <summary>
+ /// Given a set of columns, return the input columns that are needed to generate those output columns.
+ /// </summary>
+ IEnumerable<DataViewSchema.Column> IRowToRowMapper.GetDependencies(IEnumerable<DataViewSchema.Column> dependingColumns)
  {
  _ectx.Assert(IsCompositeRowToRowMapper(_chain));
 
  var transform = _chain as IDataTransform;
- var pred = predicate;
+ var cols = dependingColumns;
  while (transform != null)
  {
  var mapper = transform as IRowToRowMapper;
  _ectx.AssertValue(mapper);
- pred = mapper.GetDependencies(pred);
+ cols = mapper.GetDependencies(cols);
  transform = transform.Source as IDataTransform;
  }
- return pred;
+ return cols;
  }
 
  public DataViewSchema InputSchema => _rootSchema;
@@ -258,7 +262,8 @@ public DataViewRow GetRow(DataViewRow input, Func<int, bool> active)
  _ectx.AssertValue(mapper);
  mappers.Add(mapper);
  actives.Add(activeCur);
- activeCur = mapper.GetDependencies(activeCur);
+ var activeCurCol = mapper.GetDependencies(mapper.OutputSchema.Where(col => activeCur(col.Index)));
+ activeCur = RowCursorUtils.FromColumnsToPredicate(activeCurCol, mapper.InputSchema);
  transform = transform.Source as IDataTransform;
  }
  mappers.Reverse();

src/Microsoft.ML.Data/Prediction/Calibrator.cs

@@ -615,14 +615,15 @@ public Bound(IHostEnvironment env, SchemaBindableCalibratedModelParameters<TSubM
  OutputSchema = ScoreSchemaFactory.CreateBinaryClassificationSchema();
  }
 
- public Func<int, bool> GetDependencies(Func<int, bool> predicate)
+ /// <summary>
+ /// Given a set of columns, return the input columns that are needed to generate those output columns.
+ /// </summary>
+ IEnumerable<DataViewSchema.Column> ISchemaBoundRowMapper.GetDependenciesForNewColumns(IEnumerable<DataViewSchema.Column> dependingColumns)
  {
- for (int i = 0; i < OutputSchema.Count; i++)
- {
- if (predicate(i))
- return _predictor.GetDependencies(col => true);
- }
- return col => false;
+ if (dependingColumns.Count() > 0)
+ return _predictor.GetDependenciesForNewColumns(OutputSchema);
+
+ return Enumerable.Empty<DataViewSchema.Column>();
  }
 
  public IEnumerable<KeyValuePair<RoleMappedSchema.ColumnRole, string>> GetInputColumnRoles()

src/Microsoft.ML.Data/Scorers/FeatureContributionCalculation.cs

@@ -4,6 +4,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Reflection;
 using System.Text;
 using Microsoft.Data.DataView;
@@ -351,14 +352,12 @@ public RowMapper(IHostEnvironment env, BindableMapper parent, RoleMappedSchema s
  /// <summary>
  /// Returns the input columns needed for the requested output columns.
  /// </summary>
- public Func<int, bool> GetDependencies(Func<int, bool> predicate)
+ IEnumerable<DataViewSchema.Column> ISchemaBoundRowMapper.GetDependenciesForNewColumns(IEnumerable<DataViewSchema.Column> dependingColumns)
  {
- for (int i = 0; i < OutputSchema.Count; i++)
- {
- if (predicate(i))
- return col => col == FeatureColumn.Index;
- }
- return col => false;
+ if (dependingColumns.Count() == 0)
+ return Enumerable.Empty<DataViewSchema.Column>();
+
+ return Enumerable.Repeat(FeatureColumn, 1);
  }
 
  public DataViewRow GetRow(DataViewRow input, Func<int, bool> active)

src/Microsoft.ML.Data/Scorers/MultiClassClassifierScorer.cs

@@ -327,7 +327,11 @@ private DataViewSchema DecorateOutputSchema(DataViewSchema partialSchema, int sc
  return builder.ToSchema();
  }
 
- public Func<int, bool> GetDependencies(Func<int, bool> predicate) => _mapper.GetDependencies(predicate);
+ /// <summary>
+ /// Given a set of columns, return the input columns that are needed to generate those output columns.
+ /// </summary>
+ IEnumerable<DataViewSchema.Column> ISchemaBoundRowMapper.GetDependenciesForNewColumns(IEnumerable<DataViewSchema.Column> dependingColumns)
+ => _mapper.GetDependenciesForNewColumns(dependingColumns);
 
  public IEnumerable<KeyValuePair<RoleMappedSchema.ColumnRole, string>> GetInputColumnRoles() => _mapper.GetInputColumnRoles();