SDK release 14 Mar 2024

Author: mmajchrzycki

classifier/ei_classifier_smooth.h

@@ -79,7 +79,7 @@ const char* ei_classifier_smooth_update(ei_classifier_smooth_t *smooth, ei_impul
             reading = (int)ix;
         }
     }
-#if EI_CLASSIFIER_HAS_ANOMALY == 1
+#if EI_CLASSIFIER_HAS_ANOMALY
     if (result->anomaly >= smooth->anomaly_confidence) {
         reading = -2; // anomaly
     }

classifier/ei_classifier_types.h

@@ -35,6 +35,11 @@ typedef struct {
     float value;
 } ei_impulse_result_classification_t;
 
+typedef struct {
+    float mean_value;
+    float max_value;
+} ei_impulse_visual_ad_result_t;
+
 typedef struct {
     const char *label;
     uint32_t x;
@@ -60,6 +65,12 @@ typedef struct {
     ei_impulse_result_classification_t classification[EI_CLASSIFIER_MAX_LABELS_COUNT];
     float anomaly;
     ei_impulse_result_timing_t timing;
+    bool copy_output;
+#if EI_CLASSIFIER_HAS_VISUAL_ANOMALY
+    ei_impulse_result_bounding_box_t *visual_ad_grid_cells;
+    uint32_t visual_ad_count;
+    ei_impulse_visual_ad_result_t visual_ad_result;
+#endif // EI_CLASSIFIER_HAS_VISUAL_ANOMALY
 } ei_impulse_result_t;
 
 #endif // _EDGE_IMPULSE_RUN_CLASSIFIER_TYPES_H_

classifier/ei_fill_result_struct.h

@@ -21,6 +21,7 @@
 #include <stdint.h>
 
 #include "edge-impulse-sdk/classifier/ei_classifier_types.h"
+#include "edge-impulse-sdk/dsp/ei_dsp_handle.h"
 #include "edge-impulse-sdk/dsp/numpy.hpp"
 #if EI_CLASSIFIER_USE_FULL_TFLITE || (EI_CLASSIFIER_INFERENCING_ENGINE == EI_CLASSIFIER_AKIDA) || (EI_CLASSIFIER_INFERENCING_ENGINE == EI_CLASSIFIER_MEMRYX)
 #include "tensorflow-lite/tensorflow/lite/c/common.h"
@@ -61,14 +62,33 @@
 #define EI_CLASSIFIER_LAST_LAYER_YOLOX                 4
 #define EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI       5
 #define EI_CLASSIFIER_LAST_LAYER_YOLOV7                6
+#define EI_CLASSIFIER_LAST_LAYER_TAO_RETINANET         7
+#define EI_CLASSIFIER_LAST_LAYER_TAO_SSD               8
+#define EI_CLASSIFIER_LAST_LAYER_TAO_YOLOV3            9
+#define EI_CLASSIFIER_LAST_LAYER_TAO_YOLOV4            10
 
 #define EI_CLASSIFIER_IMAGE_SCALING_NONE          0
 #define EI_CLASSIFIER_IMAGE_SCALING_0_255         1
 #define EI_CLASSIFIER_IMAGE_SCALING_TORCH         2
 #define EI_CLASSIFIER_IMAGE_SCALING_MIN1_1        3
+#define EI_CLASSIFIER_IMAGE_SCALING_MIN128_127    4
+
+// maps back to ClassificationMode in keras-types.ts
+#define EI_CLASSIFIER_CLASSIFICATION_MODE_CLASSIFICATION      1
+#define EI_CLASSIFIER_CLASSIFICATION_MODE_REGRESSION          2
+#define EI_CLASSIFIER_CLASSIFICATION_MODE_OBJECT_DETECTION    3
+#define EI_CLASSIFIER_CLASSIFICATION_MODE_ANOMALY_GMM         4
+#define EI_CLASSIFIER_CLASSIFICATION_MODE_VISUAL_ANOMALY      5
+#define EI_CLASSIFIER_CLASSIFICATION_MODE_ANOMALY_KMEANS      6
+#define EI_CLASSIFIER_CLASSIFICATION_MODE_DSP                 7
 
 struct ei_impulse;
 
+typedef struct {
+    ei::matrix_t* matrix;
+    uint32_t blockId;
+} ei_feature_t;
+
 typedef struct {
     uint16_t implementation_version;
     bool is_configured;
@@ -79,11 +99,15 @@ typedef struct {
 } ei_model_performance_calibration_t;
 
 typedef struct {
+    uint32_t blockId;
     size_t n_output_features;
     int (*extract_fn)(ei::signal_t *signal, ei::matrix_t *output_matrix, void *config, const float frequency);
     void *config;
     uint8_t *axes;
     size_t axes_size;
+    int version;  // future proof, can easily add to this struct now
+    DspHandle* (*factory)(void* config); // nullptr means no state
+    // v1 ends here
 } ei_model_dsp_t;
 
 typedef struct {
@@ -92,9 +116,14 @@ typedef struct {
 } ei_classifier_anom_cluster_t;
 
 typedef struct {
-    EI_IMPULSE_ERROR (*infer_fn)(const ei_impulse *impulse, ei::matrix_t *fmatrix, ei_impulse_result_t *result, void *config, bool debug);
+    uint32_t blockId;
+    bool keep_output;
+    EI_IMPULSE_ERROR (*infer_fn)(const ei_impulse *impulse, ei_feature_t *fmatrix, uint32_t learn_block_index, uint32_t* input_block_ids, uint32_t input_block_ids_size, ei_impulse_result_t *result, void *config, bool debug);
     void *config;
     int image_scaling;
+    const uint32_t* input_block_ids;
+    const uint32_t input_block_ids_size;
+    uint32_t output_features_count;
 } ei_learning_block_t;
 
 typedef struct {
@@ -127,6 +156,7 @@ typedef struct {
 
 typedef struct {
     uint16_t implementation_version;
+    uint8_t classification_mode;
     uint32_t block_id;
     /* object detection */
     bool object_detection;
@@ -143,6 +173,7 @@ typedef struct {
 
 typedef struct {
     uint16_t implementation_version;
+    uint8_t classification_mode;
     const uint16_t *anom_axis;
     uint16_t anom_axes_size;
     const ei_classifier_anom_cluster_t *anom_clusters;
@@ -153,11 +184,19 @@ typedef struct {
 
 typedef struct {
     uint16_t implementation_version;
+    uint8_t classification_mode;
     const uint16_t *anom_axis;
     uint16_t anom_axes_size;
+    float anomaly_threshold;
+    bool visual;
     void* graph_config;
 } ei_learning_block_config_anomaly_gmm_t;
 
+typedef struct {
+    float confidence_threshold;
+    float iou_threshold;
+} ei_object_detection_nms_config_t;
+
 typedef struct ei_impulse {
     /* project details */
     uint32_t project_id;
@@ -200,12 +239,77 @@ typedef struct ei_impulse {
     uint32_t slices_per_model_window;
 
     /* output details */
-    bool has_anomaly;
+    uint16_t has_anomaly;
     uint16_t label_count;
     const ei_model_performance_calibration_t calibration;
     const char **categories;
+    ei_object_detection_nms_config_t object_detection_nms;
 } ei_impulse_t;
 
+class ei_impulse_state_t {
+typedef DspHandle* _dsp_handle_ptr_t;
+public:
+    const ei_impulse_t *impulse; // keep a pointer to the impulse
+    _dsp_handle_ptr_t *dsp_handles;
+    bool is_temp_handle = false; // to know if we're using the old (stateless) API
+    ei_impulse_state_t(const ei_impulse_t *impulse)
+        : impulse(impulse)
+    {
+        const auto num_dsp_blocks = impulse->dsp_blocks_size;
+        dsp_handles = (_dsp_handle_ptr_t*)ei_malloc(sizeof(_dsp_handle_ptr_t)*num_dsp_blocks);
+        for(size_t ix = 0; ix < num_dsp_blocks; ix++) {
+            dsp_handles[ix] = nullptr;
+        }
+    }
+
+    DspHandle* get_dsp_handle(size_t ix) {
+        if (dsp_handles[ix] == nullptr) {
+            dsp_handles[ix] = impulse->dsp_blocks[ix].factory(impulse->dsp_blocks[ix].config);
+        }
+        return dsp_handles[ix];
+    }
+
+    void reset()
+    {
+        for (size_t ix = 0; ix < impulse->dsp_blocks_size; ix++) {
+            if (dsp_handles[ix] != nullptr) {
+                delete dsp_handles[ix];
+                dsp_handles[ix] = nullptr;
+            }
+        }
+    }
+
+    void* operator new(size_t size) {
+        return ei_malloc(size);
+    }
+
+    void operator delete(void* ptr) {
+        ei_free(ptr);
+    }
+
+    void* operator new[](size_t size) {
+        return ei_malloc(size);
+    }
+
+    void operator delete[](void* ptr) {
+        ei_free(ptr);
+    }
+
+    ~ei_impulse_state_t()
+    {
+        reset();
+        ei_free(dsp_handles);
+    }
+};
+
+class ei_impulse_handle_t {
+public:
+    ei_impulse_handle_t(const ei_impulse_t *impulse)
+        : state(impulse), impulse(impulse) {};
+    ei_impulse_state_t state;
+    const ei_impulse_t *impulse;
+};
+
 typedef struct {
     uint32_t block_id;
     uint16_t implementation_version;