Add support for freeform models, move printing to the SDK (#29)

* Add support for freeform models, move printing to the SDK * Check return code of ei_set_freeform_output, upstream latest eim.cpp * Freeform helper * OpenCV 4.7, switch name for freeform init function

Author: janjongboom

Makefile

@@ -12,7 +12,6 @@ CFLAGS += -Ithird_party/
 CFLAGS += -Iutils/
 CFLAGS += -Os
 CFLAGS += -DNDEBUG
-CFLAGS += -DEI_CLASSIFIER_ENABLE_DETECTION_POSTPROCESS_OP=1
 CFLAGS += -g
 ifeq (${CC}, clang)
 CFLAGS += -Wno-asm-operand-widths

build-opencv-linux-aarch64-cross-compile.sh

@@ -27,11 +27,11 @@ cd $OPENCV_DIR
 
 if [ ! -d "opencv" ]; then
  git clone https://github.com/opencv/opencv.git
- cd opencv && git checkout 69357b1e88680658a07cffde7678a4d697469f03 && cd .. # v4.5.2
+ cd opencv && git checkout 4.7.0 && cd ..
 fi
 if [ ! -d "opencv_contrib" ]; then
  git clone https://github.com/opencv/opencv_contrib.git
- cd opencv_contrib && git checkout f5d7f6712d4ff229ba4f45cf79dfd11c557d56fd && cd ..
+ cd opencv_contrib && git checkout 4.7.0 && cd ..
 fi
 
 CMAKE_ARGS="

build-opencv-linux.sh

@@ -11,15 +11,15 @@ cd $OPENCV_DIR
 
 if [ ! -d "opencv" ]; then
  git clone https://github.com/opencv/opencv.git
- cd opencv && git checkout 69357b1e88680658a07cffde7678a4d697469f03 && cd .. # v4.5.2
+ cd opencv && git checkout 4.7.0 && cd ..
 fi
 if [ ! -d "opencv_contrib" ]; then
  git clone https://github.com/opencv/opencv_contrib.git
- cd opencv_contrib && git checkout f5d7f6712d4ff229ba4f45cf79dfd11c557d56fd && cd ..
+ cd opencv_contrib && git checkout 4.7.0 && cd ..
 fi
 
 mkdir -p build_opencv
 cd build_opencv
 cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_EXAMPLES=ON ../opencv
-make -j3
+make -j`nproc`
 sudo make install

inc/bitmap_helper.h

@@ -58,7 +58,7 @@ int create_bitmap_file(const char *filename, uint16_t *buffer, size_t w, size_t
  }
  fwrite(bmpfileheader, 1, 14, f);
  fwrite(bmpinfoheader, 1, 40, f);
- for(int i = 0; i < h; i++) {
+ for(int i = 0; i < (int)h; i++) {
  fwrite(img + (w * (h - i - 1) * 3), 3, w, f);
  fwrite(bmppad, 1, (4 - (w * 3) % 4 ) % 4, f);
  }
@@ -118,7 +118,7 @@ int create_bitmap_file(const char *filename, float *buffer, size_t w, size_t h)
  }
  fwrite(bmpfileheader, 1, 14, f);
  fwrite(bmpinfoheader, 1, 40, f);
- for(int i = 0; i < h; i++) {
+ for(int i = 0; i < (int)h; i++) {
  fwrite(img + (w * (h - i - 1) * 3), 3, w, f);
  fwrite(bmppad, 1, (4 - (w * 3) % 4 ) % 4, f);
  }

inc/freeform_output_helper.h

@@ -0,0 +1,89 @@
+/* The Clear BSD License
+ *
+ * Copyright (c) 2025 EdgeImpulse Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted (subject to the limitations in the disclaimer
+ * below) provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
+ * THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _FREEFORM_OUTPUT_HELPER_H_
+#define _FREEFORM_OUTPUT_HELPER_H_
+
+#include <vector>
+#include <stdio.h>
+#include "edge-impulse-sdk/classifier/ei_model_types.h"
+#include "edge-impulse-sdk/classifier/ei_run_classifier.h"
+
+#if EI_CLASSIFIER_FREEFORM_OUTPUT
+
+static std::map<ei_impulse_handle_t *, std::vector<matrix_t>> freeform_output_map;
+
+/**
+ * For "freeform" outputs, the application needs to allocate the memory (one matrix_t per output tensor).
+ * Here we'll use a global map (one per impulse handle) to allocate this memory
+ * (on the heap, using a matrix_t wrapper). In your own application you can just use something like this:
+ *
+ * std::vector<matrix_t> freeform_outputs;
+ * freeform_outputs.reserve(ei_default_impulse.impulse->freeform_outputs_size);
+ * for (size_t ix = 0; ix < ei_default_impulse.impulse->freeform_outputs_size; ++ix) {
+ * freeform_outputs.emplace_back(ei_default_impulse.impulse->freeform_outputs[ix], 1);
+ * }
+ * EI_IMPULSE_ERROR set_freeform_res = ei_set_freeform_output(freeform_outputs.data(), freeform_outputs.size());
+ * if (set_freeform_res != EI_IMPULSE_OK) {
+ * printf("ei_set_freeform_output failed with %d\n", set_freeform_res);
+ * exit(1);
+ * }
+ *
+ * Where the memory is fully owned by your application (when freeform_outputs goes out of scope ->
+ * memory is freed).
+ */
+void freeform_outputs_init(ei_impulse_handle_t *impulse_handle) {
+ // make new object in the map based on handle (so this works with multiple impulses)
+ auto& freeform_outputs = freeform_output_map[impulse_handle];
+
+ // reserve memory using the matrix_t (allocs on the heap)
+ freeform_outputs.reserve(impulse_handle->impulse->freeform_outputs_size);
+ for (size_t ix = 0; ix < impulse_handle->impulse->freeform_outputs_size; ++ix) {
+ freeform_outputs.emplace_back(impulse_handle->impulse->freeform_outputs[ix], 1);
+ }
+ // and set the freeform output
+ EI_IMPULSE_ERROR set_freeform_res = ei_set_freeform_output(freeform_outputs.data(), freeform_outputs.size());
+ if (set_freeform_res != EI_IMPULSE_OK) {
+ printf("ei_set_freeform_output failed with %d\n", set_freeform_res);
+ exit(1);
+ }
+}
+
+#else
+
+void freeform_outputs_init(ei_impulse_handle_t *impulse_handle) { }
+
+#endif // EI_CLASSIFIER_FREEFORM_OUTPUT
+
+#endif // _FREEFORM_OUTPUT_HELPER_H_

source/audio.cpp

@@ -42,6 +42,7 @@
 #include <signal.h>
 #include "edge-impulse-sdk/classifier/ei_run_classifier.h"
 #include <alsa/asoundlib.h>
+#include "inc/freeform_output_helper.h"
 
 // Forward declarations
 int microphone_audio_signal_get_data(size_t, size_t, float *);
@@ -240,14 +241,8 @@ void classify_current_buffer() {
  return;
  }
 
- printf("%d ms. ", result.timing.dsp + result.timing.classification);
- for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
- printf("%s: %.05f", result.classification[ix].label, result.classification[ix].value);
- if (ix != EI_CLASSIFIER_LABEL_COUNT - 1) {
- printf(", ");
- }
- }
- printf("\n");
+ // Print results, see edge-impulse-sdk/classifier/ei_print_results.h
+ ei_print_results(&ei_default_impulse, &result);
 }
 
 /**
@@ -284,6 +279,9 @@ int main(int argc, char **argv)
 
  run_classifier_init();
 
+ // Freeform models need to reserve their own memory. Set it up (see inc/freeform_output_helper.h)
+ freeform_outputs_init(&ei_default_impulse);
+
  while (1) {
  int x = snd_pcm_readi(capture_handle, classifier_slice_buffer, EI_CLASSIFIER_SLICE_SIZE);
  if (x != EI_CLASSIFIER_SLICE_SIZE) {

source/camera.cpp

@@ -37,6 +37,7 @@
 #include "opencv2/videoio/videoio_c.h"
 #include "edge-impulse-sdk/classifier/ei_run_classifier.h"
 #include "iostream"
+#include "inc/freeform_output_helper.h"
 
 static bool use_debug = false;
 
@@ -100,6 +101,9 @@ int main(int argc, char** argv) {
 
  run_classifier_init();
 
+ // Freeform models need to reserve their own memory. Set it up (see inc/freeform_output_helper.h)
+ freeform_outputs_init(&ei_default_impulse);
+
  // open the webcam...
  cv::VideoCapture camera(atoi(argv[1]));
  if (!camera.isOpened()) {
@@ -150,54 +154,8 @@ int main(int argc, char** argv) {
  return 1;
  }
 
- // print the predictions
- printf("Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n",
- result.timing.dsp, result.timing.classification, result.timing.anomaly);
-
- #if EI_CLASSIFIER_OBJECT_TRACKING_ENABLED == 1
- printf("#Object tracking results:\n");
- for (uint32_t ix = 0; ix < result.postprocessed_output.object_tracking_output.open_traces_count; ix++) {
- ei_object_tracking_trace_t trace = result.postprocessed_output.object_tracking_output.open_traces[ix];
- printf("%s (ID %d) [ x: %u, y: %u, width: %u, height: %u ]\n", trace.label, (int)trace.id, trace.x, trace.y, trace.width, trace.height);
- }
-
- if (result.postprocessed_output.object_tracking_output.open_traces_count == 0) {
- printf(" No objects found\n");
- }
- #elif EI_CLASSIFIER_OBJECT_DETECTION == 1
- printf("#Object detection results:\n");
- bool found_bb = false;
- for (size_t ix = 0; ix < result.bounding_boxes_count; ix++) {
- auto bb = result.bounding_boxes[ix];
- if (bb.value == 0) {
- continue;
- }
-
- found_bb = true;
- printf(" %s (%f) [ x: %u, y: %u, width: %u, height: %u ]\n", bb.label, bb.value, bb.x, bb.y, bb.width, bb.height);
- }
-
- if (!found_bb) {
- printf(" no objects found\n");
- }
- #else
- printf("#Classification results:\n");
- for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
- printf("%s: %.05f\n", result.classification[ix].label, result.classification[ix].value);
- }
- #endif
-
- #if EI_CLASSIFIER_HAS_VISUAL_ANOMALY // visual AD
- printf("#Visual anomaly grid results:\n");
- for (uint32_t i = 0; i < result.visual_ad_count; i++) {
- ei_impulse_result_bounding_box_t bb = result.visual_ad_grid_cells[i];
- if (bb.value == 0) {
- continue;
- }
- printf(" %s (%f) [ x: %u, y: %u, width: %u, height: %u ]\n", bb.label, bb.value, bb.x, bb.y, bb.width, bb.height);
- }
- printf("Visual anomaly values: Mean %.3f Max %.3f\n", result.visual_ad_result.mean_value, result.visual_ad_result.max_value);
- #endif
+ // Print results, see edge-impulse-sdk/classifier/ei_print_results.h
+ ei_print_results(&ei_default_impulse, &result);
 
  // show the image on the window
  if (use_debug) {

source/custom.cpp

@@ -38,6 +38,7 @@
 #include <sstream>
 #include "edge-impulse-sdk/classifier/ei_run_classifier.h"
 #include "inc/bitmap_helper.h"
+#include "inc/freeform_output_helper.h"
 
 std::string trim(const std::string& str) {
  size_t first = str.find_first_not_of(' ');
@@ -132,6 +133,9 @@ int main(int argc, char **argv) {
 
  run_classifier_init();
 
+ // Freeform models need to reserve their own memory. Set it up (see inc/freeform_output_helper.h)
+ freeform_outputs_init(&ei_default_impulse);
+
  ei_impulse_result_t result;
 
  signal_t signal;
@@ -143,64 +147,8 @@ int main(int argc, char **argv) {
  return 1;
  }
 
- // print the predictions
- printf("Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n",
- result.timing.dsp, result.timing.classification, result.timing.anomaly);
-
-#if EI_CLASSIFIER_OBJECT_TRACKING_ENABLED == 1
- printf("#Object tracking results:\n");
- for (uint32_t ix = 0; ix < result.postprocessed_output.object_tracking_output.open_traces_count; ix++) {
- ei_object_tracking_trace_t trace = result.postprocessed_output.object_tracking_output.open_traces[ix];
- printf("%s (ID %d) [ x: %u, y: %u, width: %u, height: %u ]\n", trace.label, (int)trace.id, trace.x, trace.y, trace.width, trace.height);
- }
-
- if (result.postprocessed_output.object_tracking_output.open_traces_count == 0) {
- printf(" No objects found\n");
- }
-#elif EI_CLASSIFIER_OBJECT_DETECTION == 1
- printf("#Object detection results:\n");
- bool bb_found = result.bounding_boxes[0].value > 0;
- for (size_t ix = 0; ix < result.bounding_boxes_count; ix++) {
- auto bb = result.bounding_boxes[ix];
- if (bb.value == 0) {
- continue;
- }
-
- printf("%s (%f) [ x: %u, y: %u, width: %u, height: %u ]\n", bb.label, bb.value, bb.x, bb.y, bb.width, bb.height);
- }
-
- if (!bb_found) {
- printf(" No objects found\n");
- }
-
-#elif (EI_CLASSIFIER_LABEL_COUNT == 1) && (!EI_CLASSIFIER_HAS_ANOMALY) // regression
- printf("#Regression results:\n");
- printf(" %s: ", result.classification[0].label);
- printf("%.5f", result.classification[0].value);
- printf("\n");
-
-#elif EI_CLASSIFIER_LABEL_COUNT > 1 // if there is only one label, this is an anomaly only
- printf("#Classification results:\n");
- for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
- printf(" %s: ", result.classification[ix].label);
- printf("%.5f", result.classification[ix].value);
- printf("\n");
- }
-#endif
-#if EI_CLASSIFIER_HAS_VISUAL_ANOMALY // visual AD
- printf("#Visual anomaly grid results:\n");
- for (uint32_t i = 0; i < result.visual_ad_count; i++) {
- ei_impulse_result_bounding_box_t bb = result.visual_ad_grid_cells[i];
- if (bb.value == 0) {
- continue;
- }
-
- printf("%s (%f) [ x: %u, y: %u, width: %u, height: %u ]\n", bb.label, bb.value, bb.x, bb.y, bb.width, bb.height);
- }
- printf("Visual anomaly values: Mean %.3f Max %.3f\n", result.visual_ad_result.mean_value, result.visual_ad_result.max_value);
-#elif (EI_CLASSIFIER_HAS_ANOMALY > 0) // except for visual AD
- printf("Anomaly prediction: %.3f\n", result.anomaly);
-#endif
+ // Print results, see edge-impulse-sdk/classifier/ei_print_results.h
+ ei_print_results(&ei_default_impulse, &result);
 
  create_debug_bmp(&result, &raw_features);
 }