[ESP32] Delta OTA Feature (#29011)

* [ESP32] Delta OTA Feature

* Fix some issues with header

* Update readme

* Some refactoring and cleanups

* Address review comments

* Apply suggestions from code review

Co-authored-by: Shubham Patil <shubham.patil@espressif.com>

---------

Co-authored-by: PSONALl <sonali.patil@espressif.com>
Co-authored-by: Rohit Jadhav <rohit.jadhav@espressif.com>
Co-authored-by: Rohit Jadhav <69809379+jadhavrohit924@users.noreply.github.com>
Co-authored-by: Shubham Patil <shubham.patil@espressif.com>

Author: 4 people

config/esp32/components/chip/CMakeLists.txt

@@ -483,6 +483,12 @@ endif()
 if (NOT EXECUTABLE_COMPONENT_NAME)
   set(EXECUTABLE_COMPONENT_NAME "main")
 endif()
+
+if (CONFIG_ENABLE_DELTA_OTA)
+    idf_component_get_property(esp_delta_ota_lib espressif__esp_delta_ota COMPONENT_LIB)
+    list(APPEND chip_libraries $<TARGET_FILE:${esp_delta_ota_lib}>)
+endif()
+
 idf_component_get_property(main_lib ${EXECUTABLE_COMPONENT_NAME} COMPONENT_LIB)
 list(APPEND chip_libraries $<TARGET_FILE:${main_lib}>)
 

config/esp32/components/chip/Kconfig

@@ -256,6 +256,15 @@ menu "CHIP Core"
             help
                 Enable this option to use the pre encrypted OTA image
 
+        config ENABLE_DELTA_OTA
+            bool "Enable delta OTA"
+            depends on ENABLE_OTA_REQUESTOR
+            default n
+            help
+                Enable this option for delta OTA image updates. 
+                Delta OTA updates allow for smaller, more efficient updates by only 
+                sending the changes between the current and new firmware versions.
+
         config OTA_AUTO_REBOOT_ON_APPLY
             bool "Reboot the device after applying the OTA image"
             depends on ENABLE_OTA_REQUESTOR

config/esp32/components/chip/idf_component.yml

@@ -30,3 +30,9 @@ dependencies:
         version: "1.0.3"
         rules:
             - if: "idf_version >=5.0"
+
+    espressif/esp_delta_ota:
+        version: "^1.1.0"
+        require: public
+        rules:
+            - if: "idf_version >=4.3"

docs/guides/esp32/ota.md

@@ -121,3 +121,66 @@ Please follow the steps below to generate an application image for OTA upgrades:
         ```
 
 3. Use the `lighting-app-encrypted-ota.bin` file with the OTA Provider app.
+
+## Delta OTA
+
+Delta OTA Updates is a feature that enables Over-the-Air (OTA) firmware update
+with compressed delta binaries. Patch files have smaller size than the original
+firmware file, which reduces the time and network usage to download the file
+from the server. Also, no additional storage partition is required for the
+"patch".
+
+### Firmware Changes
+
+-   Enable configuration options for OTA requestor and Delta OTA:
+
+    ```
+    CONFIG_ENABLE_OTA_REQUESTOR=y
+    CONFIG_ENABLE_DELTA_OTA=y
+    ```
+
+Please follow the steps below to generate an application image for OTA upgrades:
+
+1. Delta binary needs to be generated using binary delta encoding in Python
+   3.6+. You can install `detools` using the following command.
+
+    ```
+    pip install detools>=0.49.0
+    ```
+
+2. Generate delta binary.
+
+    ```
+    python managed_components/espressif__esp_delta_ota/examples/https_delta_ota/tools/esp_delta_ota_patch_gen.py --chip <chip> --base_binary <base-binary> --new_binary <new-binary> --patch_file_name <patch-file-name.bin>
+    ```
+
+3. Append the Matter OTA header:
+   `src/app/ota_image_tool.py create --vendor-id 0xFFF1 --product-id 0x8000 --version 2 --version-str "v2.0" -da sha256 <patch-file-name.bin> lighting-app-delta-ota.bin`
+
+4. Use the `lighting-app-delta-ota.bin` file with the OTA Provider app.
+
+## Encrypted Delta OTA
+
+To use encrypted delta OTA, follow the below steps.
+
+### Firmware Changes
+
+-   Enable configuration options for OTA requestor, Delta OTA and Encrypted OTA:
+
+    ```
+    CONFIG_ENABLE_OTA_REQUESTOR=y
+    CONFIG_ENABLE_DELTA_OTA=y
+    CONFIG_ENABLE_ENCRYPTED_OTA=y
+    ```
+
+1. Follow the step of `Generate a new RSA-3072 key pair or use an existing one`
+   of
+   [Encrypted OTA](https://github.com/project-chip/connectedhomeip/blob/master/docs/guides/esp32/ota.md#encrypted-ota)
+   to build your binary.
+
+2. Create delta binary as shown in `Generate delta binary` of
+   [Delta OTA](https://github.com/project-chip/connectedhomeip/blob/master/docs/guides/esp32/ota.md#delta-ota)
+
+3. Encrypt the application binary.
+
+4. Append the Matter OTA header.

src/platform/ESP32/OTAImageProcessorImpl.cpp

@@ -21,6 +21,7 @@
 #include <platform/ESP32/ESP32Utils.h>
 
 #include "OTAImageProcessorImpl.h"
+#include "esp_app_format.h"
 #include "esp_err.h"
 #include "esp_log.h"
 #include "esp_ota_ops.h"
@@ -31,7 +32,17 @@
 #include <esp_encrypted_img.h>
 #endif // CONFIG_ENABLE_ENCRYPTED_OTA
 
+#ifdef CONFIG_ENABLE_DELTA_OTA
+#include <esp_delta_ota.h>
+#endif // CONFIG_ENABLE_DELTA_OTA
+
 #define TAG "OTAImageProcessor"
+
+#ifdef CONFIG_ENABLE_DELTA_OTA
+#define PATCH_HEADER_SIZE 64
+#define DIGEST_SIZE 32
+#endif // CONFIG_ENABLE_DELTA_OTA
+
 using namespace chip::System;
 using namespace ::chip::DeviceLayer::Internal;
 
@@ -123,6 +134,152 @@ CHIP_ERROR OTAImageProcessorImpl::ProcessBlock(ByteSpan & block)
     return CHIP_NO_ERROR;
 }
 
+#ifdef CONFIG_ENABLE_DELTA_OTA
+bool OTAImageProcessorImpl::VerifyChipId(esp_chip_id_t chipId)
+{
+    if (chipId != CONFIG_IDF_FIRMWARE_CHIP_ID)
+    {
+        ESP_LOGE(TAG, "Mismatch chip id, expected %d, found %d", CONFIG_IDF_FIRMWARE_CHIP_ID, chipId);
+        return false;
+    }
+    return true;
+}
+
+bool OTAImageProcessorImpl::VerifyPatchHeader(void * imgHeaderData)
+{
+    const uint32_t espDeltaOtaMagic = 0xfccdde10;
+    if (!imgHeaderData)
+    {
+        return false;
+    }
+    uint32_t recvMagic = *(uint32_t *) imgHeaderData;
+    uint8_t * digest   = (uint8_t *) ((uint8_t *) imgHeaderData + 4);
+    if (recvMagic != espDeltaOtaMagic)
+    {
+        ESP_LOGE(TAG, "Invalid magic word in patch");
+        return false;
+    }
+    uint8_t sha_256[DIGEST_SIZE] = { 0 };
+    esp_partition_get_sha256(esp_ota_get_running_partition(), sha_256);
+    if (memcmp(sha_256, digest, DIGEST_SIZE) != 0)
+    {
+        ESP_LOGE(TAG, "SHA256 of current firmware differs from than in patch header. Invalid patch for current firmware");
+        return false;
+    }
+    return true;
+}
+
+esp_err_t OTAImageProcessorImpl::VerifyHeaderData(const uint8_t * buf, size_t size, int * index)
+{
+    static char patchHeader[PATCH_HEADER_SIZE];
+    static int headerDataRead = 0;
+    if (!patchHeaderVerified)
+    {
+        if (headerDataRead + size < PATCH_HEADER_SIZE)
+        {
+            memcpy(patchHeader + headerDataRead, buf, size);
+            headerDataRead += size;
+            return ESP_OK;
+        }
+        else
+        {
+            *index = PATCH_HEADER_SIZE - headerDataRead;
+            memcpy(patchHeader + headerDataRead, buf, *index);
+            if (!VerifyPatchHeader(patchHeader))
+            {
+                return ESP_ERR_INVALID_VERSION;
+            }
+            headerDataRead      = 0;
+            *index              = PATCH_HEADER_SIZE;
+            patchHeaderVerified = true;
+        }
+    }
+    return ESP_OK;
+}
+
+void OTAImageProcessorImpl::DeltaOTACleanUp(intptr_t context)
+{
+    auto * imageProcessor = reinterpret_cast<OTAImageProcessorImpl *>(context);
+    if (imageProcessor == nullptr)
+    {
+        ChipLogError(SoftwareUpdate, "ImageProcessor context is null");
+        return;
+    }
+    imageProcessor->patchHeaderVerified = false;
+    imageProcessor->chipIdVerified      = false;
+    return;
+}
+
+esp_err_t OTAImageProcessorImpl::DeltaOTAReadCallback(uint8_t * buf, size_t size, int srcOffset)
+{
+    if (size <= 0 || buf == NULL)
+    {
+        return ESP_ERR_INVALID_ARG;
+    }
+
+    const esp_partition_t * currentPartition = esp_ota_get_running_partition();
+    if (currentPartition == NULL)
+    {
+        return ESP_FAIL;
+    }
+
+    esp_err_t err = esp_partition_read(currentPartition, srcOffset, buf, size);
+
+    if (err != ESP_OK)
+    {
+        ESP_LOGE(TAG, "esp_partition_read failed (%s)!", esp_err_to_name(err));
+    }
+
+    return err;
+}
+
+esp_err_t OTAImageProcessorImpl::DeltaOTAWriteCallback(const uint8_t * buf, size_t size, void * arg)
+{
+    auto * imageProcessor = reinterpret_cast<OTAImageProcessorImpl *>(arg);
+    if (size <= 0 || buf == NULL)
+    {
+        return ESP_ERR_INVALID_ARG;
+    }
+
+    int index                 = 0;
+    static int headerDataRead = 0;
+    static char headerData[IMG_HEADER_LEN];
+
+    if (!imageProcessor->chipIdVerified)
+    {
+        if (headerDataRead + size - index <= IMG_HEADER_LEN)
+        {
+            memcpy(headerData + headerDataRead, buf, size - index);
+            headerDataRead += size - index;
+            return ESP_OK;
+        }
+        else
+        {
+            index = IMG_HEADER_LEN - headerDataRead;
+            memcpy(headerData + headerDataRead, buf, index);
+
+            esp_image_header_t * header = (esp_image_header_t *) headerData;
+            if (!VerifyChipId(header->chip_id))
+            {
+                return ESP_ERR_INVALID_VERSION;
+            }
+            imageProcessor->chipIdVerified = true;
+
+            // Write data in headerData buffer.
+            return esp_ota_write(imageProcessor->mOTAUpdateHandle, headerData, IMG_HEADER_LEN);
+        }
+    }
+
+    esp_err_t err = esp_ota_write(imageProcessor->mOTAUpdateHandle, buf + index, size - index);
+    if (err != ESP_OK)
+    {
+        ESP_LOGE(TAG, "esp_ota_write failed (%s)!", esp_err_to_name(err));
+    }
+
+    return err;
+}
+#endif // CONFIG_ENABLE_DELTA_OTA
+
 void OTAImageProcessorImpl::HandlePrepareDownload(intptr_t context)
 {
     auto * imageProcessor = reinterpret_cast<OTAImageProcessorImpl *>(context);
@@ -142,13 +299,32 @@ void OTAImageProcessorImpl::HandlePrepareDownload(intptr_t context)
         ChipLogError(SoftwareUpdate, "OTA partition not found");
         return;
     }
+#ifdef CONFIG_ENABLE_DELTA_OTA
+    // New image size is unknown for delta OTA, so we use OTA_SIZE_UNKNOWN flag.
+    esp_err_t err = esp_ota_begin(imageProcessor->mOTAUpdatePartition, OTA_SIZE_UNKNOWN, &(imageProcessor->mOTAUpdateHandle));
+#else
     esp_err_t err =
         esp_ota_begin(imageProcessor->mOTAUpdatePartition, OTA_WITH_SEQUENTIAL_WRITES, &(imageProcessor->mOTAUpdateHandle));
+#endif // CONFIG_ENABLE_DELTA_OTA
+
     if (err != ESP_OK)
     {
         imageProcessor->mDownloader->OnPreparedForDownload(ESP32Utils::MapError(err));
         return;
     }
+#ifdef CONFIG_ENABLE_DELTA_OTA
+    imageProcessor->deltaOtaCfg.user_data               = imageProcessor,
+    imageProcessor->deltaOtaCfg.read_cb                 = &(imageProcessor->DeltaOTAReadCallback),
+    imageProcessor->deltaOtaCfg.write_cb_with_user_data = &(imageProcessor->DeltaOTAWriteCallback),
+
+    imageProcessor->mDeltaOTAUpdateHandle = esp_delta_ota_init(&imageProcessor->deltaOtaCfg);
+    if (imageProcessor->mDeltaOTAUpdateHandle == NULL)
+    {
+        ChipLogError(SoftwareUpdate, "esp_delta_ota_init failed");
+        imageProcessor->mDownloader->OnPreparedForDownload(CHIP_ERROR_INTERNAL);
+        return;
+    }
+#endif // CONFIG_ENABLE_DELTA_OTA
 
 #ifdef CONFIG_ENABLE_ENCRYPTED_OTA
     CHIP_ERROR chipError = imageProcessor->DecryptStart();
@@ -182,7 +358,30 @@ void OTAImageProcessorImpl::HandleFinalize(intptr_t context)
     }
 #endif // CONFIG_ENABLE_ENCRYPTED_OTA
 
+#ifdef CONFIG_ENABLE_DELTA_OTA
+    esp_err_t err = esp_delta_ota_finalize(imageProcessor->mDeltaOTAUpdateHandle);
+    if (err != ESP_OK)
+    {
+        ESP_LOGE(TAG, "esp_delta_ota_finalize() failed (%s)!", esp_err_to_name(err));
+        esp_ota_abort(imageProcessor->mOTAUpdateHandle);
+        imageProcessor->ReleaseBlock();
+        PostOTAStateChangeEvent(DeviceLayer::kOtaDownloadFailed);
+    }
+
+    err = esp_delta_ota_deinit(imageProcessor->mDeltaOTAUpdateHandle);
+    if (err != ESP_OK)
+    {
+        ESP_LOGE(TAG, "esp_delta_ota_deinit() failed (%s)!", esp_err_to_name(err));
+        esp_ota_abort(imageProcessor->mOTAUpdateHandle);
+        imageProcessor->ReleaseBlock();
+        PostOTAStateChangeEvent(DeviceLayer::kOtaDownloadFailed);
+    }
+
+    err = esp_ota_end(imageProcessor->mOTAUpdateHandle);
+    DeltaOTACleanUp(reinterpret_cast<intptr_t>(imageProcessor));
+#else
     esp_err_t err = esp_ota_end(imageProcessor->mOTAUpdateHandle);
+#endif // CONFIG_ENABLE_DELTA_OTA
     if (err != ESP_OK)
     {
         if (err == ESP_ERR_OTA_VALIDATE_FAILED)
@@ -217,6 +416,10 @@ void OTAImageProcessorImpl::HandleAbort(intptr_t context)
     imageProcessor->DecryptAbort();
 #endif // CONFIG_ENABLE_ENCRYPTED_OTA
 
+#ifdef CONFIG_ENABLE_DELTA_OTA
+    DeltaOTACleanUp(reinterpret_cast<intptr_t>(imageProcessor));
+#endif // CONFIG_ENABLE_DELTA_OTA
+
     if (esp_ota_abort(imageProcessor->mOTAUpdateHandle) != ESP_OK)
     {
         ESP_LOGE(TAG, "ESP OTA abort failed");
@@ -264,7 +467,24 @@ void OTAImageProcessorImpl::HandleProcessBlock(intptr_t context)
     }
 #endif // CONFIG_ENABLE_ENCRYPTED_OTA
 
-    err = esp_ota_write(imageProcessor->mOTAUpdateHandle, blockToWrite.data(), blockToWrite.size());
+#ifdef CONFIG_ENABLE_DELTA_OTA
+
+    int index = 0;
+    err       = imageProcessor->VerifyHeaderData(blockToWrite.data(), blockToWrite.size(), &index);
+
+    if (err != ESP_OK)
+    {
+        ESP_LOGE(TAG, "Header data verification failed (%s)", esp_err_to_name(err));
+        imageProcessor->mDownloader->EndDownload(CHIP_ERROR_INVALID_SIGNATURE);
+        PostOTAStateChangeEvent(DeviceLayer::kOtaDownloadFailed);
+        return;
+    }
+
+    // Apply the patch and writes that data to the passive partition.
+    err = esp_delta_ota_feed_patch(imageProcessor->mDeltaOTAUpdateHandle, blockToWrite.data() + index, blockToWrite.size() - index);
+#else
+    err           = esp_ota_write(imageProcessor->mOTAUpdateHandle, blockToWrite.data(), blockToWrite.size());
+#endif // CONFIG_ENABLE_DELTA_OTA
 
 #ifdef CONFIG_ENABLE_ENCRYPTED_OTA
     free((void *) (blockToWrite.data()));