nrf_compress: external dictionary interface

This commit introduces additional interface to LZMA decompression.
It provides the user with possibility to store decompression dictionary
in memory area of their choice by implementing POSIX-like interfaces
(open, close, write, read) and possibly limit RAM usage.

In lzma.c, simple caching mechanism for dictionary was introduced
to limit 'write to dictionary' operations. Further optimizations may be
introduced to also limit 'read' operations if performance of accesing
external dictionary is not good enough.

Ref: NCSDK-31444

Signed-off-by: Michal Kozikowski <michal.kozikowski@nordicsemi.no>

Author: nordic-mik7

.checkpatch.conf

@@ -51,3 +51,4 @@
 --exclude lib/at_parser/generated
 --exclude lib/bin/lwm2m_carrier/include
 --exclude tests/lib/uicc_lwm2m
+--exclude subsys/nrf_compress/lzma

include/nrf_compress/implementation.h

@@ -12,6 +12,7 @@
 #ifndef NRF_COMPRESS_IMPLEMENTATION_H_
 #define NRF_COMPRESS_IMPLEMENTATION_H_
 
+#include "lzma_types.h"
 #include <stdint.h>
 #include <stdlib.h>
 #include <zephyr/kernel.h>
@@ -32,7 +33,8 @@ extern "C" {
  * @typedef		nrf_compress_init_func_t
  * @brief		Initialize compression implementation.
  *
- * @param[in] inst	Reserved for future use, must be NULL.
+ * @param[in] inst	Implementation specific initialization context.
+ *			Concrete implementation may cast it to predefined type.
  *
  * @retval		0 Success.
  * @retval		-errno Negative errno code on other failure.
@@ -43,7 +45,8 @@ typedef int (*nrf_compress_init_func_t)(void *inst);
  * @typedef		nrf_compress_deinit_func_t
  * @brief		De-initialize compression implementation.
  *
- * @param[in] inst	Reserved for future use, must be NULL.
+ * @param[in] inst	Implementation specific initialization context.
+ *			Concrete implementation may cast it to predefined type.
  *
  * @retval		0 Success.
  * @retval		-errno Negative errno code on other failure.
@@ -55,7 +58,8 @@ typedef int (*nrf_compress_deinit_func_t)(void *inst);
  * @brief		Reset compression state function. Used to abort current compression or
  *			decompression task before starting a new one.
  *
- * @param[in] inst	Reserved for future use, must be NULL.
+ * @param[in] inst	Implementation specific initialization context.
+ *			Concrete implementation may cast it to predefined type.
  *
  * @retval		0 Success.
  * @retval		-errno Negative errno code on other failure.
@@ -66,7 +70,8 @@ typedef int (*nrf_compress_reset_func_t)(void *inst);
  * @typedef		nrf_compress_compress_func_t
  * @brief		Placeholder function for future use, do not use.
  *
- * @param[in] inst	Reserved for future use, must be NULL.
+ * @param[in] inst	Implementation specific initialization context.
+ *			Concrete implementation may cast it to predefined type.
  *
  * @retval		0 Success.
  * @retval		-errno Negative errno code on other failure.
@@ -80,7 +85,8 @@ typedef int (*nrf_compress_compress_func_t)(void *inst);
  *			be provided if more is not available (for example, end of data or data is
  *			is being streamed).
  *
- * @param[in] inst	Reserved for future use, must be NULL.
+ * @param[in] inst	Implementation specific initialization context.
+ *			Concrete implementation may cast it to predefined type.
  *
  * @retval		Positive value Success indicating chunk size.
  * @retval		-errno Negative errno code on other failure.
@@ -92,7 +98,8 @@ typedef size_t (*nrf_compress_decompress_bytes_needed_t)(void *inst);
  *				be called one or more times with compressed data to decompress it
  *				into its natural form.
  *
- * @param[in] inst		Reserved for future use, must be NULL.
+ * @param[in] inst		Implementation specific initialization context.
+ *				Concrete implementation may cast it to predefined type.
  * @param[in] input		Input data buffer, containing the compressed data.
  * @param[in] input_size	Size of the input data buffer.
  * @param[in] last_part		Last part of compressed data. This should be set to true if this is
@@ -103,9 +110,13 @@ typedef size_t (*nrf_compress_decompress_bytes_needed_t)(void *inst);
  *				offset the input data buffer by this amount of bytes.
  * @param[out] output		Output data buffer pointer to pointer. This will be set to the
  *				compression's output buffer when decompressed data is available to
- *				be used or copied.
- * @param[out] output_size	Size of data in output data buffer pointer. Data should only be
- *				read when the value in this pointer is greater than 0.
+ *				be used or copied. WARNING: For LZMA implementation, it is only
+ *				valid if no external dictionary is used. For external dictionary
+ *				variant (indicated by *inst parameter in init function), this
+ *				will be set to NULL and user should read from ones own dictionary.
+ * @param[out] output_size	Size of data in output data buffer pointer (or in external
+ *				dictionary). Data should only be read when the value in this pointer
+ *				is greater than 0.
  *
  * @retval			0 Success.
  * @retval			-errno Negative errno code on other failure.

include/nrf_compress/lzma_types.h

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2025 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+ */
+
+/**
+ * @file
+ * @brief LZMA API types for compression/decompression subsystem
+ */
+
+#ifndef NRF_COMPRESS_LZMA_TYPES_H_
+#define NRF_COMPRESS_LZMA_TYPES_H_
+
+#include <zephyr/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @typedef		lzma_dictionary_open_func_t
+ * @brief		Open dictionary interface. It is up to the user
+ *			how big is the dictionary buffer, but it must be
+ *			at least @a dict_size long.
+ *
+ * @param[in]		dict_size dictionary size; minimum length of the
+ *			buffer that is requested.
+ * @param[out]		buff_size size of the dictionary buffer provided.
+ *
+ * @retval		0 Success.
+ * @retval		-ENOMEM if cannot provide requested size. @a buff_size is valid.
+ * @retval		-errno Negative errno code on other failure @a buff_size is invalid.
+ */
+typedef int (*lzma_dictionary_open_func_t)(size_t dict_size, size_t *buff_size);
+
+/**
+ * @typedef		lzma_dictionary_close_func_t
+ * @brief		Close dictionary interface.
+ *
+ * @retval		0 Success.
+ * @retval		-errno Negative errno code on other failure.
+ */
+typedef int (*lzma_dictionary_close_func_t)(void);
+
+/**
+ * @typedef		lzma_dictionary_write_func_t
+ * @brief		Write dictionary interface.
+ *
+ * @param[in]		pos Position (byte-wise) of the dictionary to start writing to.
+ * @param[in]		data Data to write.
+ * @param[in]		len Length of @a data buffer.
+ *
+ * @retval		Number of written bytes to the dictionary (length).
+ */
+typedef size_t (*lzma_dictionary_write_func_t)(size_t pos, const uint8_t *data, size_t len);
+
+/**
+ * @typedef		lzma_dictionary_read_func_t
+ * @brief		Read dictionary interface.
+ *
+ * @param[in]		pos Position of the dictionary to start reading from.
+ * @param[in]		data Data buffer to read into.
+ * @param[in]		len Length of @a data buffer, number of bytes to read.
+ *
+ * @retval		Number of bytes read from the dictionary (length).
+ */
+typedef size_t (*lzma_dictionary_read_func_t)(size_t pos, uint8_t *data, size_t len);
+
+typedef struct lzma_dictionary_interface_t {
+	const lzma_dictionary_open_func_t open;
+	const lzma_dictionary_close_func_t close;
+	const lzma_dictionary_write_func_t write;
+	const lzma_dictionary_read_func_t read;
+} lzma_dictionary_interface;
+
+/**
+ * @brief This is an initialization context struct type. Instantionize and pass it to
+ * interface functions like for e.g. nrf_compress_init_func_t, nrf_compress_decompress_func_t.
+ */
+typedef struct lzma_codec_t {
+	const lzma_dictionary_interface dict_if;
+} lzma_codec;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* NRF_COMPRESS_IMPLEMENTATION_H_ */

scripts/ci/license_allow_list.yaml

@@ -36,6 +36,7 @@ none: |
 # Ignore all detected licenses in that file.
 any: |
   .*/license-texts.yaml$
+  ^nrf/subsys/nrf_compress/lzma/
 
 # Allow different licenses from external sources
 LicenseRef-west-ncs-sbom-iperf-BSD-3-Clause: "^nrf/ext/"

subsys/nrf_compress/Kconfig

@@ -96,6 +96,20 @@ config NRF_COMPRESS_MEMORY_TYPE_MALLOC
 
 endchoice
 
+config NRF_COMPRESS_EXTERNAL_DICTIONARY
+	bool "External dictionary"
+	help
+	  Use external dictionary API for LZMA decompression. It provides the user with
+	  possibility to store dictionary data in memory areas of their choice (e.g. MRAM).
+
+config NRF_COMPRESS_DICTIONARY_CACHE_SIZE
+	int "Dictionary cache size"
+	default 1024
+	depends on NRF_COMPRESS_EXTERNAL_DICTIONARY
+	help
+	  Cache for last written dictionary data. It limits the number of external dictionary API calls:
+	  'write' and (possibly but not optimized for) 'read'.
+
 config NRF_COMPRESS_MEMORY_ALIGNMENT
 	int "Buffer memory alignment"
 	default 4

subsys/nrf_compress/lzma/Lzma2Dec.c

@@ -1,6 +1,8 @@
 /* Lzma2Dec.c -- LZMA2 Decoder
 2024-03-01 : Igor Pavlov : Public domain */
 
+/** With changes by Nordic Semiconductor ASA */
+
 /* #define SHOW_DEBUG_INFO */
 
 #include "Precomp.h"
@@ -165,7 +167,15 @@ static unsigned Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b)
 
 static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT size)
 {
+#ifdef CONFIG_NRF_COMPRESS_EXTERNAL_DICTIONARY
+	if (LzmaDictionaryWrite(p->dicHandle, p->dicPos, src, size) != size) {
+		p->state = LZMA2_STATE_ERROR;
+		return;
+	}
+#else
 	memcpy(p->dic + p->dicPos, src, size);
+#endif
+
 	p->dicPos += size;
 	if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= size) {
 		p->checkDicSize = p->prop.dicSize;
@@ -240,6 +250,9 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, const Byte *src, SizeT *
 				}
 
 				LzmaDec_UpdateWithUncompressed(&p->decoder, src, inCur);
+				if (p->state == LZMA2_STATE_ERROR) {
+					continue;
+				}
 
 				src += inCur;
 				*srcLen += inCur;
@@ -419,6 +432,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, SizeT outSize, const Byte *src, S
 	return (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED;
 }
 
+#ifndef CONFIG_NRF_COMPRESS_EXTERNAL_DICTIONARY
 SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen,
 			  ELzmaFinishMode finishMode, ELzmaStatus *status)
 {
@@ -481,5 +495,6 @@ SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, Byt
 	Lzma2Dec_FreeProbs(&p, alloc);
 	return res;
 }
+#endif
 
 #undef PRF

subsys/nrf_compress/lzma/LzmaDec.c

@@ -1,6 +1,8 @@
 /* LzmaDec.h -- LZMA Decoder
 2023-04-02 : Igor Pavlov : Public domain */
 
+/** With changes by Nordic Semiconductor ASA */
+
 #ifndef ZIP7_INC_LZMA_DEC_H
 #define ZIP7_INC_LZMA_DEC_H
 
@@ -47,13 +49,29 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size);
 
 #define LZMA_REQUIRED_INPUT_MAX 20
 
+#ifdef CONFIG_NRF_COMPRESS_EXTERNAL_DICTIONARY
+/* Handler type for external dictionary operations.
+ * See LzmaDictionaryOpen() and other dictionary interfaces to be implemented
+ * by the user of this library below.
+ */
+typedef struct DictHandle_t {
+	BoolInt isOpened;
+	SizeT dicBufSize;
+
+} DictHandle;
+#endif
+
 typedef struct {
 	/* Don't change this structure. ASM code can use it. */
 	CLzmaProps prop;
 	CLzmaProb *probs;
 	CLzmaProb *probs_1664;
+#ifdef CONFIG_NRF_COMPRESS_EXTERNAL_DICTIONARY
+	DictHandle *dicHandle;
+#else
 	Byte *dic;
 	SizeT dicBufSize;
+#endif
 	SizeT dicPos;
 	const Byte *buf;
 	UInt32 range;
@@ -69,11 +87,19 @@ typedef struct {
 	Byte tempBuf[LZMA_REQUIRED_INPUT_MAX];
 } CLzmaDec;
 
+#ifdef CONFIG_NRF_COMPRESS_EXTERNAL_DICTIONARY
+#define LzmaDec_CONSTRUCT(p)                                                                       \
+	{                                                                                          \
+		(p)->dicHandle = NULL;                                                                   \
+		(p)->probs = NULL;                                                                 \
+	}
+#else
 #define LzmaDec_CONSTRUCT(p)                                                                       \
 	{                                                                                          \
 		(p)->dic = NULL;                                                                   \
 		(p)->probs = NULL;                                                                 \
 	}
+#endif
 #define LzmaDec_Construct(p) LzmaDec_CONSTRUCT(p)
 
 void LzmaDec_Init(CLzmaDec *p);
@@ -229,6 +255,66 @@ SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, cons
 		unsigned propSize, ELzmaFinishMode finishMode, ELzmaStatus *status,
 		ISzAllocPtr alloc);
 
+
+#ifdef CONFIG_NRF_COMPRESS_EXTERNAL_DICTIONARY
+/* Interfaces to be implemented by the user of this library.
+   They are added to allow for different memory storage of dictionary,
+   not only as directly referenced memory through CLzmaDec::dic.
+*/
+
+/* Open dictionary with requested size.
+
+minSize:
+  Minimal size of the dictionary.
+Returns:
+  Pointer to a dictionary handle struct for subsequent API calls
+  or NULL if failed to open.
+*/
+DictHandle *LzmaDictionaryOpen(SizeT minSize);
+
+/* Write to dictionary.
+
+handle:
+  Pointer to dictionary handle struct, returned from LzmaDictionaryOpen().
+pos:
+  Position (byte-wise) in dictionary to start writing to.
+data
+  Data to be written.
+len
+  Length of @a data.
+Returns:
+  Number of written bytes.
+*/
+SizeT LzmaDictionaryWrite(DictHandle *handle, SizeT pos, const Byte *data, SizeT len);
+
+/* Read from dictionary.
+
+handle:
+  Pointer to dictionary handle struct, returned from LzmaDictionaryOpen().
+pos:
+  Position (byte-wise) in dictionary to start reading from.
+data
+  Data buffer to fill in with read data.
+len
+  Length of @a data; number of bytes to be read.
+Returns:
+  Number of read bytes.
+*/
+SizeT LzmaDictionaryRead(DictHandle *handle, SizeT pos, Byte *data, SizeT len);
+
+/* Close dictionary.
+
+handle:
+  Pointer to dictionary handle struct, returned from LzmaDictionaryOpen().
+
+Returns:
+  SZ_OK
+  SZ_ERROR_PARAM - Invalid input parameters
+  SZ_ERROR_FAIL  - Some unexpected error: internal error of code, memory corruption or hardware
+ */
+SRes LzmaDictionaryClose(DictHandle *handle);
+#endif
+
 EXTERN_C_END
 
 #endif

subsys/nrf_compress/lzma/LzmaDec.h

@@ -23,3 +23,6 @@ tests:
       - CONFIG_NRF_COMPRESS_MEMORY_TYPE_MALLOC=y
       - CONFIG_COMMON_LIBC_MALLOC=y
       - CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=162000
+  nrf_compress.decompression.lzma.external_dict:
+    extra_configs:
+      - CONFIG_NRF_COMPRESS_EXTERNAL_DICTIONARY=y