Merge pull request #115 from saddam213/FeatureExtractor

Feature extractor update

Author: saddam213

OnnxStack.Console/Examples/ControlNetFeatureExample.cs

@@ -35,10 +35,10 @@ public async Task RunAsync()
  var inputImage = await InputImage.FromFileAsync("D:\\Repositories\\OnnxStack\\Assets\\Samples\\Img2Img_Start.bmp");
 
  // Create Annotation pipeline
- var annotationPipeline = AnnotationPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators");
+ var annotationPipeline = FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\depth.onnx", true);
 
  // Create Depth Image
- var controlImage = await annotationPipeline.DepthImage(inputImage);
+ var controlImage = await annotationPipeline.RunAsync(inputImage);
 
  // Save Depth Image (Debug Only)
  await controlImage.Image.SaveAsPngAsync(Path.Combine(_outputDirectory, $"Depth.png"));

OnnxStack.Console/Examples/FeatureExtractorExample.cs

@@ -0,0 +1,68 @@
+using OnnxStack.Core.Image;
+using OnnxStack.FeatureExtractor.Pipelines;
+using OnnxStack.StableDiffusion.Config;
+using SixLabors.ImageSharp;
+using System.Diagnostics;
+
+namespace OnnxStack.Console.Runner
+{
+ public sealed class FeatureExtractorExample : IExampleRunner
+ {
+ private readonly string _outputDirectory;
+ private readonly StableDiffusionConfig _configuration;
+
+ public FeatureExtractorExample(StableDiffusionConfig configuration)
+ {
+ _configuration = configuration;
+ _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(FeatureExtractorExample));
+ Directory.CreateDirectory(_outputDirectory);
+ }
+
+ public int Index => 13;
+
+ public string Name => "Feature Extractor Example";
+
+ public string Description => "Simple exmaple using ControlNet feature extractors";
+
+ /// <summary>
+ /// ControlNet Example
+ /// </summary>
+ public async Task RunAsync()
+ {
+ // Load Control Image
+ var inputImage = await InputImage.FromFileAsync("D:\\Repositories\\OnnxStack\\Assets\\Samples\\Img2Img_Start.bmp");
+
+ var pipelines = new[]
+ {
+ FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\canny.onnx"),
+ FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\hed.onnx"),
+ FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\depth.onnx", true),
+
+ // FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\depth-anything-large-hf\\onnx\\model.onnx", normalize: true, sampleSize: 504),
+ // FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\sentis-MiDaS\\dpt_beit_large_512.onnx", normalize: true, sampleSize: 384),
+ };
+
+ foreach (var pipeline in pipelines)
+ {
+ var timestamp = Stopwatch.GetTimestamp();
+ OutputHelpers.WriteConsole($"Load pipeline`{pipeline.Name}`", ConsoleColor.Cyan);
+
+ // Run Pipeline
+ var imageFeature = await pipeline.RunAsync(inputImage);
+
+ OutputHelpers.WriteConsole($"Generating image", ConsoleColor.Cyan);
+
+ // Save Image
+ await imageFeature.Image.SaveAsPngAsync(Path.Combine(_outputDirectory, $"{pipeline.Name}.png"));
+
+
+ OutputHelpers.WriteConsole($"Unload pipeline", ConsoleColor.Cyan);
+
+ //Unload
+ await pipeline.UnloadAsync();
+
+ OutputHelpers.WriteConsole($"Elapsed: {Stopwatch.GetElapsedTime(timestamp)}ms", ConsoleColor.Yellow);
+ }
+ }
+ }
+}

OnnxStack.Core/Extensions/TensorExtension.cs

@@ -32,6 +32,28 @@ public static DenseTensor<float> Repeat(this DenseTensor<float> tensor1, int cou
  }
 
 
+ /// <summary>
+ /// Normalize the data using Min-Max scaling to ensure all values are in the range [0, 1].
+ /// </summary>
+ /// <param name="tensor">The tensor.</param>
+ public static void NormalizeMinMax(this DenseTensor<float> tensor)
+ {
+ var values = tensor.Buffer.Span;
+ float min = float.PositiveInfinity, max = float.NegativeInfinity;
+ foreach (var val in values)
+ {
+ if (min > val) min = val;
+ if (max < val) max = val;
+ }
+
+ var range = max - min;
+ for (var i = 0; i < values.Length; i++)
+ {
+ values[i] = (values[i] - min) / range;
+ }
+ }
+
+
  /// <summary>
  /// Concatenates the specified tensors along the specified axis.
  /// </summary>

OnnxStack.Core/Video/VideoFrame.cs

@@ -2,5 +2,8 @@
 
 namespace OnnxStack.Core.Video
 {
- public record VideoFrame(byte[] Frame, InputImage ControlImage = default);
+ public record VideoFrame(byte[] Frame)
+ {
+ public InputImage ExtraFrame { get; set; }
+ }
 }

OnnxStack.FeatureExtractor/Common/FeatureExtractorModel.cs

@@ -7,25 +7,35 @@ namespace OnnxStack.FeatureExtractor.Common
  public class FeatureExtractorModel : OnnxModelSession
  {
  private readonly int _sampleSize;
+ private readonly bool _normalize;
+ private readonly int _channels;
 
  public FeatureExtractorModel(FeatureExtractorModelConfig configuration)
  : base(configuration)
  {
  _sampleSize = configuration.SampleSize;
+ _normalize = configuration.Normalize;
+ _channels = configuration.Channels;
  }
 
  public int SampleSize => _sampleSize;
 
+ public bool Normalize => _normalize;
+
+ public int Channels => _channels;
+
  public static FeatureExtractorModel Create(FeatureExtractorModelConfig configuration)
  {
  return new FeatureExtractorModel(configuration);
  }
 
- public static FeatureExtractorModel Create(string modelFile, int deviceId = 0, ExecutionProvider executionProvider = ExecutionProvider.DirectML)
+ public static FeatureExtractorModel Create(string modelFile, bool normalize = false, int sampleSize = 512, int channels = 3, int deviceId = 0, ExecutionProvider executionProvider = ExecutionProvider.DirectML)
  {
  var configuration = new FeatureExtractorModelConfig
  {
- SampleSize = 512,
+ SampleSize = sampleSize,
+ Normalize = normalize,
+ Channels = channels,
  DeviceId = deviceId,
  ExecutionProvider = executionProvider,
  ExecutionMode = ExecutionMode.ORT_SEQUENTIAL,
@@ -40,5 +50,7 @@ public static FeatureExtractorModel Create(string modelFile, int deviceId = 0, E
  public record FeatureExtractorModelConfig : OnnxModelConfig
  {
  public int SampleSize { get; set; }
+ public bool Normalize { get; set; }
+ public int Channels { get; set; }
  }
 }

OnnxStack.FeatureExtractor/Common/AnnotationModelSet.cs renamed to OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs

@@ -4,7 +4,7 @@
 
 namespace OnnxStack.FeatureExtractor.Common
 {
- public record AnnotationModelSet : IOnnxModelSetConfig
+ public record FeatureExtractorModelSet : IOnnxModelSetConfig
  {
  public string Name { get; set; }
  public bool IsEnabled { get; set; }
@@ -15,14 +15,6 @@ public record AnnotationModelSet : IOnnxModelSetConfig
  public ExecutionProvider ExecutionProvider { get; set; }
 
  [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
- public FeatureExtractorModelConfig CannyImageConfig { get; set; }
-
- [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
- public FeatureExtractorModelConfig HedImageConfig { get; set; }
-
- [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
- public FeatureExtractorModelConfig DepthImageConfig { get; set; }
+ public FeatureExtractorModelConfig FeatureExtractorConfig { get; set; }
  }
-
-
 }