Transformer/estimator are done. Working on Transform

DataView coded, waiting on C wrappers for testing

Author: michaelgsharp

src/Microsoft.ML.AutoMLFeaturizers/CategoryImputer.cs

@@ -378,15 +378,6 @@ protected override bool ReleaseHandle()
 
  #endregion
 
- #region FitResult
-
- internal enum FitResult : byte
- {
- Complete = 0, Continue, ResetAndContinue
- }
-
- #endregion
-
  #region ColumnInfo
 
  // REVIEW: Since we can't do overloading on the native side due to the C style exports,

src/Microsoft.ML.AutoMLFeaturizers/Common.cs

@@ -1,12 +1,19 @@
 using System;
 using System.Collections.Generic;
+using System.Runtime.InteropServices;
+using System.Security;
 using System.Text;
+using Microsoft.Win32.SafeHandles;
 
 namespace Microsoft.ML.AutoMLFeaturizers
 {
+ #region Native Function Declarations
+
+ #endregion
+
  internal enum FitResult : byte
  {
- Complete = 0, Continue, ResetAndContinue
+ Complete = 1, Continue, ResetAndContinue
  }
 
  internal enum TypeId : byte
@@ -15,9 +22,65 @@ internal enum TypeId : byte
  Vector, Array, Tabular
  };
 
+ internal unsafe struct ByteArray
+ {
+ public byte* Data;
+ public IntPtr DataSize;
+ }
+
+ internal class ErrorInfoSafeHandle : SafeHandleZeroOrMinusOneIsInvalid
+ {
+ [DllImport("Featurizers", EntryPoint = "DestroyErrorInfo", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
+ private static extern bool DestroyErrorInfo(IntPtr error);
+
+ public ErrorInfoSafeHandle(IntPtr handle) : base(true)
+ {
+ SetHandle(handle);
+ }
+
+ protected override bool ReleaseHandle()
+ {
+ return DestroyErrorInfo(handle);
+ }
+ }
+
+ internal class ErrorInfoStringSafeHandle : SafeHandleZeroOrMinusOneIsInvalid
+ {
+ [DllImport("Featurizers", EntryPoint = "DestroyErrorInfoString", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
+ private static extern bool DestroyErrorInfoString(IntPtr errorString, IntPtr errorStringSize);
+
+ private IntPtr _length;
+ public ErrorInfoStringSafeHandle(IntPtr handle, IntPtr length) : base(true)
+ {
+ SetHandle(handle);
+ _length = length;
+ }
+
+ protected override bool ReleaseHandle()
+ {
+ return DestroyErrorInfoString(handle, _length);
+ }
+ }
+
  internal static class CommonExtensions
  {
+ [DllImport("Featurizers", EntryPoint = "GetErrorInfoString", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
+ private static extern bool GetErrorInfoString(IntPtr error, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
 
+ internal static string GetErrorDetailsAndFreeNativeMemory(IntPtr errorHandle)
+ {
+ using (var error = new ErrorInfoSafeHandle(errorHandle))
+ {
+ GetErrorInfoString(errorHandle, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
+ using (var errorString = new ErrorInfoStringSafeHandle(errorHandleString, errorHandleStringSize))
+ {
+ byte[] buffer = new byte[errorHandleStringSize.ToInt32()];
+ Marshal.Copy(errorHandleString, buffer, 0, buffer.Length);
+
+ return Encoding.UTF8.GetString(buffer);
+ }
+ }
+ }
  internal static TypeId GetNativeTypeIdFromType(this Type type)
  {
  if (type == typeof(byte))

src/Microsoft.ML.AutoMLFeaturizers/DateTimeTransformer.cs

@@ -32,7 +32,10 @@ namespace Microsoft.ML.AutoMLFeaturizers
  public static class DateTimeTransformerExtensionClass
  {
  public static DateTimeTransformerEstimator DateTimeTransformer(this TransformsCatalog catalog, string inputColumnName, string columnPrefix, params DateTimeTransformerEstimator.ColumnsProduced[] columnsToDrop)
- => DateTimeTransformerEstimator.Create(CatalogUtils.GetEnvironment(catalog), inputColumnName, columnPrefix, columnsToDrop);
+ => new DateTimeTransformerEstimator(CatalogUtils.GetEnvironment(catalog), inputColumnName, columnPrefix, columnsToDrop);
+
+ public static DateTimeTransformerEstimator DateTimeTransformer(this TransformsCatalog catalog, string inputColumnName, string columnPrefix, DateTimeTransformerEstimator.ColumnsProduced[] columnsToDrop = null, DateTimeTransformerEstimator.Countries country = DateTimeTransformerEstimator.Countries.None)
+ => new DateTimeTransformerEstimator(CatalogUtils.GetEnvironment(catalog), inputColumnName, columnPrefix, columnsToDrop, country);
 
  #region ColumnsProduced static extentions
 
@@ -77,21 +80,19 @@ internal sealed class Options: TransformInputBase
 
  [Argument(ArgumentType.MultipleUnique, HelpText = "Columns to drop after the DateTime Expansion", Name = "ColumnsToDrop", ShortName = "drop", SortOrder = 3)]
  public ColumnsProduced[] ColumnsToDrop;
+
+ [Argument(ArgumentType.AtMostOnce, HelpText = "Country to get holidays for. Defaults to none if not passed", Name = "Country", ShortName = "ctry", SortOrder = 4)]
+ public Countries Country = Countries.None;
  }
 
  #endregion
 
- internal static DateTimeTransformerEstimator Create(IHostEnvironment env, string inputColumnName, string columnPrefix, ColumnsProduced[] columnsToDrop)
- {
- return new DateTimeTransformerEstimator(env, inputColumnName, columnPrefix, columnsToDrop);
- }
-
  // Using this to confirm DLL exists. If does it will just return false since no parameters are being passed.
  // Once we have a binary dependency on the dll we can remove this code.
  [DllImport("Featurizers", EntryPoint = "GetErrorInfoString"), SuppressUnmanagedCodeSecurity]
  private static extern bool CheckIfDllExists(IntPtr error, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
 
- public DateTimeTransformerEstimator(IHostEnvironment env, string inputColumnName, string columnPrefix, ColumnsProduced[] columnsToDrop)
+ public DateTimeTransformerEstimator(IHostEnvironment env, string inputColumnName, string columnPrefix, ColumnsProduced[] columnsToDrop, Countries country = Countries.None)
  {
  try
  {
@@ -110,7 +111,8 @@ public DateTimeTransformerEstimator(IHostEnvironment env, string inputColumnName
  {
  Source = inputColumnName,
  Prefix = columnPrefix,
- ColumnsToDrop = columnsToDrop
+ ColumnsToDrop = columnsToDrop == null ? Array.Empty<ColumnsProduced>() : columnsToDrop,
+ Country = country
  };
  }
 
@@ -129,6 +131,7 @@ internal DateTimeTransformerEstimator(IHostEnvironment env, Options options)
  _host = Contracts.CheckRef(env, nameof(env)).Register("DateTimeTransformerEstimator");
 
  _options = options;
+ _options.ColumnsToDrop = _options.ColumnsToDrop == null ? Array.Empty<ColumnsProduced>() : _options.ColumnsToDrop;
  }
 
  public DateTimeTransformer Fit(IDataView input)
@@ -151,14 +154,23 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)
  return new SchemaShape(columns.Values);
  }
 
- #region Column Enums
+ #region Enums
  public enum ColumnsProduced : byte
  {
- Year = 0, Month, Day, Hour, Minute, Second, AmPm, Hour12, DayOfWeek, DayOfQuarter, DayOfYear,
+ Year = 1, Month, Day, Hour, Minute, Second, AmPm, Hour12, DayOfWeek, DayOfQuarter, DayOfYear,
  WeekOfMonth, QuarterOfYear, HalfOfYear, WeekIso, YearIso, MonthLabel, AmPmLabel, DayOfWeekLabel,
  HolidayName, IsPaidTimeOff
  };
 
+ public enum Countries : byte
+ {
+ None = 1,
+ Argentina, Australia, Austria, Belarus, Belgium, Brazil, Canada, Colombia, Croatia, Czech, Denmark,
+ England, Finland, France, Germany, Hungary, India, Ireland, IsleofMan, Italy, Japan, Mexico, Netherlands,
+ NewZealand, NorthernIreland, Norway, Poland, Portugal, Scotland, Slovenia, SouthAfrica, Spain, Sweden, Switzerland,
+ Ukraine, UnitedKingdom, UnitedStates, Wales
+ }
+
  #endregion
  }
 
@@ -473,14 +485,17 @@ internal unsafe TimePoint(byte* rawData)
  private static unsafe string GetStringFromPointer(ref byte* rawData, int intPtrSize)
  {
  byte[] buffer;
- byte* temp = rawData + intPtrSize;
- long tempSize = *(long*)(temp);
- int itempSize = *(int*)(temp);
  if (intPtrSize == 4) // 32 bit machine
  buffer = new byte[*(uint*)(rawData + intPtrSize)];
  else // 64 bit machine
  buffer = new byte[*(ulong*)(rawData + intPtrSize)];
 
+ if (buffer.Length == 0)
+ {
+ rawData += intPtrSize * 2;
+ return string.Empty;
+ }
+
  Marshal.Copy(new IntPtr(*(int**)rawData), buffer, 0, buffer.Length);
  rawData += intPtrSize * 2;
 
@@ -772,7 +787,8 @@ protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, b
 
  var outputColumn = (int)_parent._activeColumnMapping[iinfo];
 
- return Utils.MarshalInvoke(MakeGetter<int>, ((DateTimeTransformerEstimator.ColumnsProduced)outputColumn).GetRawColumnType(), input, outputColumn);
+ // Have to subtract 1 from the output column since the enum starts and 1 and not 0.
+ return Utils.MarshalInvoke(MakeGetter<int>, ((DateTimeTransformerEstimator.ColumnsProduced)outputColumn).GetRawColumnType(), input, outputColumn - 1);
  }
 
  private protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
@@ -802,7 +818,7 @@ internal static class DateTimeTransformerEntrypoint
  public static CommonOutputs.TransformOutput DateTimeSplit(IHostEnvironment env, DateTimeTransformerEstimator.Options input)
  {
  var h = EntryPointUtils.CheckArgsAndCreateHost(env, DateTimeTransformer.ShortName, input);
- var xf = DateTimeTransformerEstimator.Create(h, input.Source, input.Prefix, input.ColumnsToDrop).Fit(input.Data).Transform(input.Data);
+ var xf = new DateTimeTransformerEstimator(h, input).Fit(input.Data).Transform(input.Data);
  return new CommonOutputs.TransformOutput()
  {
  Model = new TransformModelImpl(h, xf, input.Data),