Video Support

OnnxStack.Console/Examples/StableDiffusionBatch.cs

@@ -1,7 +1,9 @@
-using OnnxStack.StableDiffusion.Common;
+using OnnxStack.Core.Image;
+using OnnxStack.StableDiffusion.Common;
 using OnnxStack.StableDiffusion.Config;
 using OnnxStack.StableDiffusion.Enums;
 using OnnxStack.StableDiffusion.Helpers;
+using OnnxStack.StableDiffusion.Models;
 using SixLabors.ImageSharp;
 
 namespace OnnxStack.Console.Runner
@@ -58,13 +60,13 @@ public async Task RunAsync()
  await _stableDiffusionService.LoadModelAsync(model);
 
  var batchIndex = 0;
- var callback = (int batch, int batchCount, int step, int steps) =>
+ var callback = (DiffusionProgress progress) =>
  {
- batchIndex = batch;
- OutputHelpers.WriteConsole($"Image: {batch}/{batchCount} - Step: {step}/{steps}", ConsoleColor.Cyan);
+ batchIndex = progress.BatchValue;
+ OutputHelpers.WriteConsole($"Image: {progress.BatchValue}/{progress.BatchMax} - Step: {progress.StepValue}/{progress.StepMax}", ConsoleColor.Cyan);
  };
 
- await foreach (var result in _stableDiffusionService.GenerateBatchAsync(model, promptOptions, schedulerOptions, batchOptions, callback))
+ await foreach (var result in _stableDiffusionService.GenerateBatchAsync(model, promptOptions, schedulerOptions, batchOptions, default))
  {
  var outputFilename = Path.Combine(_outputDirectory, $"{batchIndex}_{result.SchedulerOptions.Seed}.png");
  var image = result.ImageResult.ToImage();

OnnxStack.Console/OnnxStack.Console.csproj

@@ -6,7 +6,7 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>disable</Nullable>
  <PlatformTarget>x64</PlatformTarget>
- <Configurations>Debug;Release;Debug-DirectML;Debug-Cuda;Debug-TensorRT;Release-DirectML;Release-Cuda;Release-TensorRT</Configurations>
+ <Configurations>Debug;Release;Debug-Cuda;Debug-TensorRT;Release-Cuda;Release-TensorRT</Configurations>
  </PropertyGroup>
 
  <ItemGroup>
@@ -15,17 +15,16 @@
  </ItemGroup>
 
  <ItemGroup>
- <PackageReference Include="OnnxStack.StableDiffusion" Version="0.14.0" Condition=" '$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release-DirectML' OR '$(Configuration)' == 'Release-Cuda' OR '$(Configuration)' == 'Release-TensorRT'" />
- <PackageReference Include="OnnxStack.ImageUpscaler" Version="0.14.0" Condition=" '$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release-DirectML' OR '$(Configuration)' == 'Release-Cuda' OR '$(Configuration)' == 'Release-TensorRT'" />
- <ProjectReference Include="..\OnnxStack.StableDiffusion\OnnxStack.StableDiffusion.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-DirectML' OR '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Debug-TensorRT'" />
- <ProjectReference Include="..\OnnxStack.ImageUpscaler\OnnxStack.ImageUpscaler.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-DirectML' OR '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Debug-TensorRT'" />
+ <PackageReference Include="OnnxStack.StableDiffusion" Version="0.14.0" Condition=" '$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release-Cuda' OR '$(Configuration)' == 'Release-TensorRT'" />
+ <PackageReference Include="OnnxStack.ImageUpscaler" Version="0.14.0" Condition=" '$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release-Cuda' OR '$(Configuration)' == 'Release-TensorRT'" />
+ <ProjectReference Include="..\OnnxStack.StableDiffusion\OnnxStack.StableDiffusion.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Debug-TensorRT'" />
+ <ProjectReference Include="..\OnnxStack.ImageUpscaler\OnnxStack.ImageUpscaler.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Debug-TensorRT'" />
  </ItemGroup>
 
  <ItemGroup>
- <PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Release' " />
+ <PackageReference Include="Microsoft.ML.OnnxRuntime.DirectML" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Release' " />
  <PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug-TensorRT' OR '$(Configuration)' == 'Release-TensorRT'" />
  <PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Release-Cuda'" />
- <PackageReference Include="Microsoft.ML.OnnxRuntime.DirectML" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug-DirectML' OR '$(Configuration)' == 'Release-DirectML'" />
  </ItemGroup>
 
  <ItemGroup>

OnnxStack.Core/Config/OnnxStackConfig.cs

@@ -4,6 +4,10 @@ namespace OnnxStack.Core.Config
 {
  public class OnnxStackConfig : IConfigSection
  {
+ public string TempPath { get; set; } = ".temp";
+ public string FFmpegPath { get; set; } = "ffmpeg.exe";
+ public string FFprobePath { get; set; } = "ffprobe.exe";
+
  public void Initialize()
  {
  }

OnnxStack.Core/Image/Extensions.cs

@@ -0,0 +1,73 @@
+using Microsoft.ML.OnnxRuntime.Tensors;
+using SixLabors.ImageSharp.PixelFormats;
+using SixLabors.ImageSharp;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.IO;
+
+namespace OnnxStack.Core.Image
+{
+ public static class Extensions
+ {
+ public static Image<Rgba32> ToImage(this DenseTensor<float> imageTensor)
+ {
+ var height = imageTensor.Dimensions[2];
+ var width = imageTensor.Dimensions[3];
+ var hasAlpha = imageTensor.Dimensions[1] == 4;
+ var result = new Image<Rgba32>(width, height);
+ for (var y = 0; y < height; y++)
+ {
+ for (var x = 0; x < width; x++)
+ {
+ result[x, y] = new Rgba32(
+ CalculateByte(imageTensor, 0, y, x),
+ CalculateByte(imageTensor, 1, y, x),
+ CalculateByte(imageTensor, 2, y, x),
+ hasAlpha ? CalculateByte(imageTensor, 3, y, x) : byte.MaxValue
+ );
+ }
+ }
+ return result;
+ }
+
+ /// <summary>
+ /// Converts to image byte array.
+ /// </summary>
+ /// <param name="imageTensor">The image tensor.</param>
+ /// <returns></returns>
+ public static byte[] ToImageBytes(this DenseTensor<float> imageTensor)
+ {
+ using (var image = imageTensor.ToImage())
+ using (var memoryStream = new MemoryStream())
+ {
+ image.SaveAsPng(memoryStream);
+ return memoryStream.ToArray();
+ }
+ }
+
+ /// <summary>
+ /// Converts to image byte array.
+ /// </summary>
+ /// <param name="imageTensor">The image tensor.</param>
+ /// <returns></returns>
+ public static async Task<byte[]> ToImageBytesAsync(this DenseTensor<float> imageTensor)
+ {
+ using (var image = imageTensor.ToImage())
+ using (var memoryStream = new MemoryStream())
+ {
+ await image.SaveAsPngAsync(memoryStream);
+ return memoryStream.ToArray();
+ }
+ }
+
+
+ private static byte CalculateByte(Tensor<float> imageTensor, int index, int y, int x)
+ {
+ return (byte)Math.Round(Math.Clamp(imageTensor[0, index, y, x] / 2 + 0.5, 0, 1) * 255);
+ }
+
+ }
+}

OnnxStack.Core/OnnxStack.Core.csproj

@@ -35,13 +35,15 @@
  </ItemGroup>
 
  <ItemGroup>
+ <PackageReference Include="FFMpegCore" Version="5.1.0" />
  <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
  <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
  <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
  <PackageReference Include="Microsoft.ML" Version="3.0.0" />
  <PackageReference Include="Microsoft.ML.OnnxRuntime.Extensions" Version="0.9.0" />
  <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.16.3" />
  <PackageReference Include="SixLabors.ImageSharp" Version="3.1.1" />
+ <PackageReference Include="System.Linq.Async" Version="6.0.1" />
  </ItemGroup>
 
  <ItemGroup>

OnnxStack.Core/Services/IVideoService.cs

@@ -0,0 +1,109 @@
+using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core.Video;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace OnnxStack.Core.Services
+{
+ /// <summary>
+ /// Service with basic handling of video for use in OnnxStack, Frame->Video and Video->Frames
+ /// </summary>
+ public interface IVideoService
+ {
+ /// <summary>
+ /// Gets the video information asynchronous.
+ /// </summary>
+ /// <param name="videoBytes">The video bytes.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ Task<VideoInfo> GetVideoInfoAsync(byte[] videoBytes, CancellationToken cancellationToken = default);
+
+ /// <summary>
+ /// Gets the video information asynchronous.
+ /// </summary>
+ /// <param name="videoStream">The video stream.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ Task<VideoInfo> GetVideoInfoAsync(Stream videoStream, CancellationToken cancellationToken = default);
+
+ /// <summary>
+ /// Gets the video information, Size, FPS, Duration etc.
+ /// </summary>
+ /// <param name="videoInput">The video input.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ /// <exception cref="ArgumentException">No video data found</exception>
+ Task<VideoInfo> GetVideoInfoAsync(VideoInput videoInput, CancellationToken cancellationToken = default);
+
+
+ /// <summary>
+ /// Creates a collection of PNG frames from a video source
+ /// </summary>
+ /// <param name="videoBytes">The video bytes.</param>
+ /// <param name="videoFPS">The video FPS.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float videoFPS, CancellationToken cancellationToken = default);
+
+
+ /// <summary>
+ /// Creates a collection of PNG frames from a video source
+ /// </summary>
+ /// <param name="videoStream">The video stream.</param>
+ /// <param name="videoFPS">The video FPS.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ Task<VideoFrames> CreateFramesAsync(Stream videoStream, float videoFPS, CancellationToken cancellationToken = default);
+
+
+ /// <summary>
+ /// Creates a collection of PNG frames from a video source
+ /// </summary>
+ /// <param name="videoInput">The video input.</param>
+ /// <param name="videoFPS">The video FPS.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ /// <exception cref="NotSupportedException">VideoTensor not supported</exception>
+ /// <exception cref="ArgumentException">No video data found</exception>
+ Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float videoFPS, CancellationToken cancellationToken = default);
+
+
+ /// <summary>
+ /// Creates and MP4 video from a collection of PNG images.
+ /// </summary>
+ /// <param name="videoFrames">The video frames.</param>
+ /// <param name="videoFPS">The video FPS.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ Task<VideoOutput> CreateVideoAsync(IEnumerable<byte[]> videoFrames, float videoFPS, CancellationToken cancellationToken = default);
+
+
+ /// <summary>
+ /// Creates and MP4 video from a collection of PNG images.
+ /// </summary>
+ /// <param name="videoFrames">The video frames.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ Task<VideoOutput> CreateVideoAsync(VideoFrames videoFrames, CancellationToken cancellationToken = default);
+
+ // <summary>
+ /// Creates and MP4 video from a collection of PNG images.
+ /// </summary>
+ /// <param name="videoTensor">The video frames.</param>
+ /// <param name="videoFPS">The video FPS.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ Task<VideoOutput> CreateVideoAsync(DenseTensor<float> videoTensor, float videoFPS, CancellationToken cancellationToken = default);
+
+ /// <summary>
+ /// Streams frames as PNG as they are processed from a video source
+ /// </summary>
+ /// <param name="videoBytes">The video bytes.</param>
+ /// <param name="targetFPS">The target FPS.</param>
+ /// <param name="cancellationToken">The cancellation token.</param>
+ /// <returns></returns>
+ IAsyncEnumerable<byte[]> StreamFramesAsync(byte[] videoBytes, float targetFPS, CancellationToken cancellationToken = default);
+ }
+}