Number of TF patches, image classification without external buffers, remove uTensor

Author: janjongboom

.mbedignore

@@ -3,5 +3,6 @@ tensorflow/lite/micro/mbed/
 porting/arduino/
 porting/ecm3532/
 porting/posix/
+porting/silabs/
 porting/stm32-cubeai/
 classifier/ei_run_classifier_c*

CMSIS/NN/Source/NNSupportFunctions/arm_nn_mat_mult_nt_t_s8.c

@@ -37,7 +37,7 @@
 __STATIC_FORCEINLINE uint32_t __patched_SXTB16_RORn(uint32_t op1, uint32_t rotate) {
 uint32_t result;
 if (__builtin_constant_p (rotate) && ((rotate == 8U) || (rotate == 16U) || (rotate == 24U))) {
-  asm volatile ("sxtb16 %0, %1, ROR %2" : "=r" (result) : "r" (op1), "i" (rotate) );
+  __ASM volatile ("sxtb16 %0, %1, ROR %2" : "=r" (result) : "r" (op1), "i" (rotate) );
 } else {
   result = __SXTB16(__ROR(op1, rotate)) ;
 }

classifier/ei_run_classifier.h

@@ -0,0 +1,8 @@
+#ifndef _EDGE_IMPULSE_RUN_CLASSIFIER_IMAGE_H_
+#define _EDGE_IMPULSE_RUN_CLASSIFIER_IMAGE_H_
+
+#include "ei_run_classifier.h"
+
+
+
+#endif // _EDGE_IMPULSE_RUN_CLASSIFIER_IMAGE_H_

classifier/ei_run_classifier_image.h

@@ -477,49 +477,104 @@ __attribute__((unused)) int extract_image_features(signal_t *signal, matrix_t *o
 
     int16_t channel_count = strcmp(config.channels, "Grayscale") == 0 ? 1 : 3;
 
-    if (output_matrix->rows * output_matrix->cols != EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT * channel_count) {
+    if (output_matrix->rows * output_matrix->cols != static_cast<uint32_t>(EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT * channel_count)) {
         ei_printf("out_matrix = %hu items\n", output_matrix->rows, output_matrix->cols);
-        ei_printf("calculated size = %hu items\n", EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT * channel_count);
+        ei_printf("calculated size = %hu items\n", static_cast<uint32_t>(EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT * channel_count));
         EIDSP_ERR(EIDSP_MATRIX_SIZE_MISMATCH);
     }
 
     size_t output_ix = 0;
 
     // buffered read from the signal
     size_t bytes_left = signal->total_length;
-    for (size_t ix = 0; ix < signal->total_length; ix += 4096) {
-        size_t bytes_to_read = bytes_left > 4096 ? 4096 : bytes_left;
+    for (size_t ix = 0; ix < signal->total_length; ix += 1024) {
+        size_t elements_to_read = bytes_left > 1024 ? 1024 : bytes_left;
 
-        matrix_t input_matrix(bytes_to_read, config.axes);
+        matrix_t input_matrix(elements_to_read, config.axes);
         if (!input_matrix.buffer) {
             EIDSP_ERR(EIDSP_OUT_OF_MEM);
         }
-        signal->get_data(ix, bytes_to_read, input_matrix.buffer);
+        signal->get_data(ix, elements_to_read, input_matrix.buffer);
 
-        for (size_t jx = 0; jx < bytes_to_read; jx++) {
+        for (size_t jx = 0; jx < elements_to_read; jx++) {
             uint32_t pixel = static_cast<uint32_t>(input_matrix.buffer[jx]);
+
+            // rgb to 0..1
+            float r = static_cast<float>(pixel >> 16 & 0xff) / 255.0f;
+            float g = static_cast<float>(pixel >> 8 & 0xff) / 255.0f;
+            float b = static_cast<float>(pixel & 0xff) / 255.0f;
+
+            if (channel_count == 3) {
+                output_matrix->buffer[output_ix++] = r;
+                output_matrix->buffer[output_ix++] = g;
+                output_matrix->buffer[output_ix++] = b;
+            }
+            else {
+                // ITU-R 601-2 luma transform
+                // see: https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.Image.convert
+                float v = (0.299f * r) + (0.587f * g) + (0.114f * b);
+                output_matrix->buffer[output_ix++] = v;
+            }
+        }
+
+        bytes_left -= elements_to_read;
+    }
+
+    return EIDSP_OK;
+}
+
+#if EI_CLASSIFIER_TFLITE_INPUT_QUANTIZED == 1
+__attribute__((unused)) int extract_image_features_quantized(signal_t *signal, matrix_i8_t *output_matrix, void *config_ptr) {
+    ei_dsp_config_image_t config = *((ei_dsp_config_image_t*)config_ptr);
+
+    int16_t channel_count = strcmp(config.channels, "Grayscale") == 0 ? 1 : 3;
+
+    if (output_matrix->rows * output_matrix->cols != static_cast<uint32_t>(EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT * channel_count)) {
+        ei_printf("out_matrix = %hu items\n", output_matrix->rows, output_matrix->cols);
+        ei_printf("calculated size = %hu items\n", static_cast<uint32_t>(EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT * channel_count));
+        EIDSP_ERR(EIDSP_MATRIX_SIZE_MISMATCH);
+    }
+
+    size_t output_ix = 0;
+
+    // buffered read from the signal
+    size_t bytes_left = signal->total_length;
+    for (size_t ix = 0; ix < signal->total_length; ix += 1024) {
+        size_t elements_to_read = bytes_left > 1024 ? 1024 : bytes_left;
+
+        matrix_t input_matrix(elements_to_read, config.axes);
+        if (!input_matrix.buffer) {
+            EIDSP_ERR(EIDSP_OUT_OF_MEM);
+        }
+        signal->get_data(ix, elements_to_read, input_matrix.buffer);
+
+        for (size_t jx = 0; jx < elements_to_read; jx++) {
+            uint32_t pixel = static_cast<uint32_t>(input_matrix.buffer[jx]);
+
+            // rgb to 0..1
+            float r = static_cast<float>(pixel >> 16 & 0xff) / 255.0f;
+            float g = static_cast<float>(pixel >> 8 & 0xff) / 255.0f;
+            float b = static_cast<float>(pixel & 0xff) / 255.0f;
+
             if (channel_count == 3) {
-                // rgb to 0..1
-                output_matrix->buffer[output_ix++] = static_cast<float>(pixel >> 16 & 0xff) / 255.0f;
-                output_matrix->buffer[output_ix++] = static_cast<float>(pixel >> 8 & 0xff) / 255.0f;
-                output_matrix->buffer[output_ix++] = static_cast<float>(pixel & 0xff) / 255.0f;
+                output_matrix->buffer[output_ix++] = static_cast<int8_t>(round(r / EI_CLASSIFIER_TFLITE_INPUT_SCALE) + EI_CLASSIFIER_TFLITE_INPUT_ZEROPOINT);
+                output_matrix->buffer[output_ix++] = static_cast<int8_t>(round(g / EI_CLASSIFIER_TFLITE_INPUT_SCALE) + EI_CLASSIFIER_TFLITE_INPUT_ZEROPOINT);
+                output_matrix->buffer[output_ix++] = static_cast<int8_t>(round(b / EI_CLASSIFIER_TFLITE_INPUT_SCALE) + EI_CLASSIFIER_TFLITE_INPUT_ZEROPOINT);
             }
             else {
-                // grayscale conversion (also to 0..1)
-                float r = static_cast<float>(pixel >> 16 & 0xff) / 255.0f;
-                float g = static_cast<float>(pixel >> 8 & 0xff) / 255.0f;
-                float b = static_cast<float>(pixel & 0xff) / 255.0f;
                 // ITU-R 601-2 luma transform
                 // see: https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.Image.convert
-                output_matrix->buffer[output_ix++] = (0.299f * r) + (0.587f * g) + (0.114f * b);
+                float v = (0.299f * r) + (0.587f * g) + (0.114f * b);
+                output_matrix->buffer[output_ix++] = static_cast<int8_t>(round(v / EI_CLASSIFIER_TFLITE_INPUT_SCALE) + EI_CLASSIFIER_TFLITE_INPUT_ZEROPOINT);
             }
         }
 
-        bytes_left -= bytes_to_read;
+        bytes_left -= elements_to_read;
     }
 
     return EIDSP_OK;
 }
+#endif // EI_CLASSIFIER_TFLITE_INPUT_QUANTIZED == 1
 
 #ifdef __cplusplus
 }

classifier/ei_run_dsp.h

@@ -24,6 +24,7 @@ rm -rf $SCRIPTPATH/tensorflow/lite/micro/mbed/
 rm -rf $SCRIPTPATH/porting/ecm3532/
 rm -rf $SCRIPTPATH/porting/mbed/
 rm -rf $SCRIPTPATH/porting/posix/
+rm -rf $SCRIPTPATH/porting/silabs/
 rm -rf $SCRIPTPATH/porting/stm32-cubeai/
 rm -rf $SCRIPTPATH/classifier/ei_run_classifier_c*
 rm -rf $SCRIPTPATH/CMSIS/DSP/Source/TransformFunctions/arm_bitreversal2.S

create-arduino-library.sh

@@ -126,6 +126,84 @@ typedef struct ei_matrix {
 #endif // #ifdef __cplusplus
 } matrix_t;
 
+/**
+ * A matrix structure that allocates a matrix on the **heap**.
+ * Freeing happens by calling `delete` on the object or letting the object go out of scope.
+ */
+typedef struct ei_matrix_i8 {
+    int8_t *buffer;
+    uint32_t rows;
+    uint32_t cols;
+    bool buffer_managed_by_me;
+
+#if EIDSP_TRACK_ALLOCATIONS
+    const char *_fn;
+    const char *_file;
+    int _line;
+#endif
+
+#ifdef __cplusplus
+    /**
+     * Create a new matrix
+     * @param n_rows Number of rows
+     * @param n_cols Number of columns
+     * @param a_buffer Buffer, if not provided we'll alloc on the heap
+     */
+    ei_matrix_i8(
+        uint32_t n_rows,
+        uint32_t n_cols,
+        int8_t *a_buffer = NULL
+#if EIDSP_TRACK_ALLOCATIONS
+        ,
+        const char *fn = NULL,
+        const char *file = NULL,
+        int line = 0
+#endif
+        )
+    {
+        if (a_buffer) {
+            buffer = a_buffer;
+            buffer_managed_by_me = false;
+        }
+        else {
+            buffer = (int8_t*)calloc(n_rows * n_cols * sizeof(int8_t), 1);
+            buffer_managed_by_me = true;
+        }
+        rows = n_rows;
+        cols = n_cols;
+
+        if (!a_buffer) {
+#if EIDSP_TRACK_ALLOCATIONS
+            _fn = fn;
+            _file = file;
+            _line = line;
+            if (_fn) {
+                ei_dsp_register_matrix_alloc_internal(fn, file, line, rows, cols, sizeof(int8_t));
+            }
+            else {
+                ei_dsp_register_matrix_alloc(rows, cols, sizeof(int8_t));
+            }
+#endif
+        }
+    }
+
+    ~ei_matrix_i8() {
+        if (buffer && buffer_managed_by_me) {
+            free(buffer);
+
+#if EIDSP_TRACK_ALLOCATIONS
+            if (_fn) {
+                ei_dsp_register_matrix_free_internal(_fn, _file, _line, rows, cols, sizeof(int8_t));
+            }
+            else {
+                ei_dsp_register_matrix_free(rows, cols, sizeof(int8_t));
+            }
+#endif
+        }
+    }
+#endif // #ifdef __cplusplus
+} matrix_i8_t;
+
 /**
  * Another matrix structure that allocates a matrix on the **heap**.
  * Freeing happens by calling `delete` on the object or letting the object go out of scope.

dsp/numpy_types.h

@@ -38,7 +38,9 @@ typedef enum {
     EI_IMPULSE_DSP_ERROR = -5,
     EI_IMPULSE_TFLITE_ARENA_ALLOC_FAILED = -6,
     EI_IMPULSE_CUBEAI_ERROR = -7,
-    EI_IMPULSE_ALLOC_FAILED = -8
+    EI_IMPULSE_ALLOC_FAILED = -8,
+    EI_IMPULSE_ONLY_SUPPORTED_FOR_IMAGES = -9,
+    EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE = -10
 } EI_IMPULSE_ERROR;
 
 /**

porting/ei_classifier_porting.h

@@ -0,0 +1,34 @@
+/* Edge Impulse inferencing library
+ * Copyright (c) 2020 EdgeImpulse Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "tensorflow/lite/micro/debug_log.h"
+#include "../ei_classifier_porting.h"
+#include <stdio.h>
+#include <stdarg.h>
+
+// Redirect TFLite DebugLog to ei_printf
+#if defined(__cplusplus) && EI_C_LINKAGE == 1
+extern "C"
+#endif // defined(__cplusplus) && EI_C_LINKAGE == 1
+void DebugLog(const char* s) {
+    ei_printf("%s", s);
+}

porting/silabs/debug_log.cpp

@@ -0,0 +1,65 @@
+/* Edge Impulse inferencing library
+ * Copyright (c) 2020 EdgeImpulse Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdarg.h>
+#include "../ei_classifier_porting.h"
+#include "ingestion-sdk-platform/SiliconLabs/ei_device_silabs_efm32mg.h"
+
+
+__attribute__((weak)) EI_IMPULSE_ERROR ei_run_impulse_check_canceled() {
+    return EI_IMPULSE_OK;
+}
+
+/**
+ * Cancelable sleep, can be triggered with signal from other thread
+ */
+__attribute__((weak)) EI_IMPULSE_ERROR ei_sleep(int32_t time_ms) {
+    EiDevice.delay_ms(time_ms);
+    return EI_IMPULSE_OK;
+}
+
+uint64_t ei_read_timer_ms() {
+    return EiDevice.get_ms();
+}
+
+uint64_t ei_read_timer_us() {
+
+    return 0;//EiDevice.get_ms() * 1000;
+}
+
+__attribute__((weak)) void ei_printf(const char *format, ...) {
+    va_list myargs;
+    va_start(myargs, format);
+    vprintf(format, myargs);
+    va_end(myargs);
+}
+
+__attribute__((weak)) void ei_printf_float(float f) {
+    ei_printf("%f", f);
+}
+
+#if defined(__cplusplus) && EI_C_LINKAGE == 1
+extern "C"
+#endif
+__attribute__((weak)) void DebugLog(const char* s) {
+    ei_printf("%s", s);
+}

porting/silabs/ei_classifier_porting.cpp

@@ -4,5 +4,4 @@ The folders were taken from `tensorflow/lite/micro/tools/make/gen/osx_x86_64/prj
 These files and directories were then deleted:
  - `tensorflow/lite/micro/debug_log.cc`
  - `tensorflow/lite/micro/examples/hello_world`
-utensor based on: https://github.com/janjongboom/utensor/#3a76b5870aee0a3d4ab3f17a9ed434fcca897520
-CMSIS-DSP based on: https://github.com/ARM-software/CMSIS_5/tree/4d378e81968c6bec5441a42885b24db7cf189bca
+CMSIS-DSP based on: https://github.com/ARM-software/CMSIS_5/tree/4d378e81968c6bec5441a42885b24db7cf189bca

sources.txt

@@ -19,7 +19,9 @@ limitations under the License.
 
 namespace tflite {
 
-#if defined(TF_LITE_USE_GLOBAL_MAX) || defined(__ZEPHYR__)
+// Patched by Edge Impulse, Arduino ESP32 targets have no std::fmax (at least on some targets)
+// see https://forum.edgeimpulse.com/t/esp32-cam-support/797/21
+#if defined(TF_LITE_USE_GLOBAL_MAX) || defined(__ZEPHYR__) || defined(ARDUINO_ARCH_ESP32)
 inline float TfLiteMax(const float& x, const float& y) {
   return std::max(x, y);
 }

tensorflow/lite/kernels/internal/max.h

@@ -401,12 +401,13 @@ TfLiteStatus FlatBufferVectorToTfLiteTypeArray(
     kTfLiteArrayType** result) {
   TFLITE_DCHECK(error_reporter != nullptr);
   TFLITE_DCHECK(flatbuffer_array != nullptr);
+  // Patched out by Edge Impulse, does not compile on ESP32 targets
   // Only two conversions are supported - float and int32 - ensure that these
   // match at compile time instead of duplicating functions here:
-  static_assert((std::is_same<kFlatBufferVectorType, int32_t>() &&
-                 std::is_same<kTfLiteArrayType, TfLiteIntArray>()) ||
-                (std::is_same<kFlatBufferVectorType, float>() &&
-                 std::is_same<kTfLiteArrayType, TfLiteFloatArray>()));
+  // static_assert((std::is_same<kFlatBufferVectorType, int32_t>() &&
+  //                std::is_same<kTfLiteArrayType, TfLiteIntArray>()) ||
+  //               (std::is_same<kFlatBufferVectorType, float>() &&
+  //                std::is_same<kTfLiteArrayType, TfLiteFloatArray>()));
   if (FLATBUFFERS_LITTLEENDIAN) {
     // On little-endian machines, TfLite*Array happens to have the same memory
     // layout as flatbuffers:Vector<kFlatBufferVectorType>, so we can