[scripts] Fixed nrfconnect factory generation scripts (project-chip#32892)

The factory_data.hex file is not generated as an intermediate
product of factory generation process. In result, factory
generation works only if merging with firmware is used or
scripts are manually invoked.

Added optional --size and --offset arguments to
generate_nrfconnect_chip_factory_data.py script that results in
calling nrfconnect_generate_partition.py internally. It solves
an issue and additionally simplifies manual generation process
(if selected).

Author: kkasperczyk-no

config/nrfconnect/chip-module/generate_factory_data.cmake

@@ -15,25 +15,29 @@
 #
 
 
-# Create a JSON file based on factory data given via kConfigs.
+# Create a .hex file in CBOR format based on factory data given via kConfigs.
 #
 # This function creates a list of arguments for external script and then run it to write a JSON file.
 # Created JSON file can be checked using JSON SCHEMA file if it is provided.
+# Next, the resulting .hex file is generated based on previously created JSON file.
 #
 # This script can be manipulated using following kConfigs:
 # - To merge generated factory data with final zephyr.hex file set kConfig CONFIG_CHIP_FACTORY_DATA_MERGE_WITH_FIRMWARE=y
 # - To use default certification paths set CONFIG_CHIP_FACTORY_DATA_USE_DEFAULTS_CERTS_PATH=y 
 # 
-# During generation process a some file will be created in zephyr's build directory:
+# During generation process the following files will be created in zephyr's build directory:
 # - <factory_data_target>.json a file containing all factory data written in JSON format.
+# - <factory_data_target>.hex a file containing all factory data in CBOR format.
+# - <factory_data_target>.bin a binary file containing all raw factory data in CBOR format.
+# - <factory_data_target>.cbor a file containing all factory data in CBOR format.
 #
 # [Args]:
 #   factory_data_target - a name for target to generate factory_data.
 #   script_path         - a path to script that makes a JSON factory data file from given arguments.
 #   schema_path         - a path to JSON schema file which can be used to verify generated factory data JSON file.
 #                         This argument is optional, if you don't want to verify the JSON file put it empty "".
-#   output_path         - a path to output directory, where created JSON file will be stored.
-function(nrfconnect_create_factory_data_json factory_data_target script_path schema_path output_path)
+#   output_path         - a path to output directory, where created hex and JSON files will be stored.
+function(nrfconnect_create_factory_data factory_data_target script_path schema_path output_path)
 
 # set script args for future purpose
 set(script_args)
@@ -120,62 +124,25 @@ if(CONFIG_CHIP_DEVICE_ENABLE_KEY)
 string(APPEND script_args "--enable_key \"${CONFIG_CHIP_DEVICE_ENABLE_KEY}\"\n")
 endif()
 
-# Set output JSON file and path to SCHEMA file to validate generated factory data
-set(factory_data_json ${output_path}/${factory_data_target}.json)
-string(APPEND script_args "-o \"${factory_data_json}\"\n")
+# Set output path and path to SCHEMA file to validate generated factory data
+set(factory_data_output_path ${output_path}/${factory_data_target})
+string(APPEND script_args "-o \"${factory_data_output_path}\"\n")
 string(APPEND script_args "-s \"${schema_path}\"\n")
 
+# Add optional offset and size arguments to generate both .hex and .json files.
+string(APPEND script_args "--offset $<TARGET_PROPERTY:partition_manager,PM_FACTORY_DATA_ADDRESS>\n")
+string(APPEND script_args "--size $<TARGET_PROPERTY:partition_manager,PM_FACTORY_DATA_OFFSET>\n")
+
 # execute first script to create a JSON file
 separate_arguments(separated_script_args NATIVE_COMMAND ${script_args})
 add_custom_command(
-    OUTPUT ${factory_data_json}
+    OUTPUT ${factory_data_output_path}.hex
     DEPENDS ${FACTORY_DATA_SCRIPT_PATH}
     COMMAND ${Python3_EXECUTABLE} ${FACTORY_DATA_SCRIPT_PATH} ${separated_script_args}
     COMMENT "Generating new Factory Data..."
     )
 add_custom_target(${factory_data_target} ALL
-    DEPENDS ${factory_data_json}
-    )
-
-endfunction()
-
-
-# Create a .hex file with factory data in CBOR format.
-#
-# This function creates a .hex and .cbor files from given JSON factory data file.
-#
-# 
-# During generation process some files will be created in zephyr's build directory:
-# - <factory_data_target>.hex a file containing all factory data in CBOR format.
-# - <factory_data_target>.bin a binary file containing all raw factory data in CBOR format.
-# - <factory_data_target>.cbor a file containing all factory data in CBOR format.
-#
-# [Args]:
-#   factory_data_hex_target - a name for target to generate factory data HEX file.
-#   factory_data_target     - a name for target to generate factory data JSON file.
-#   script_path             - a path to script that makes a factory data .hex file from given arguments.
-#   output_path             - a path to output directory, where created JSON file will be stored.
-function(nrfconnect_create_factory_data_hex_file factory_data_hex_target factory_data_target script_path output_path)
-
-# Pass the argument list via file
-set(cbor_script_args "-i ${output_path}/${factory_data_target}.json\n")
-string(APPEND cbor_script_args "-o ${output_path}/${factory_data_target}\n")
-# get partition address and offset from partition manager during compilation
-string(APPEND cbor_script_args "--offset $<TARGET_PROPERTY:partition_manager,PM_FACTORY_DATA_ADDRESS>\n")
-string(APPEND cbor_script_args "--size $<TARGET_PROPERTY:partition_manager,PM_FACTORY_DATA_OFFSET>\n")
-string(APPEND cbor_script_args "-r\n")
-
-# execute second script to create a hex file containing factory data in cbor format
-separate_arguments(separated_cbor_script_args NATIVE_COMMAND ${cbor_script_args})
-set(factory_data_hex ${output_path}/${factory_data_target}.hex)
-
-add_custom_command(OUTPUT ${factory_data_hex}
-    COMMAND ${Python3_EXECUTABLE} ${script_path} ${separated_cbor_script_args}
-    COMMENT "Generating factory data HEX file..."
-    DEPENDS ${factory_data_target} ${script_path}
-    )
-add_custom_target(${factory_data_hex_target}
-    DEPENDS ${factory_data_hex}
+    DEPENDS ${factory_data_output_path}.hex
     )
 
 endfunction()
@@ -202,22 +169,16 @@ set(GENERATE_CBOR_SCRIPT_PATH ${CHIP_ROOT}/scripts/tools/nrfconnect/nrfconnect_g
 set(FACTORY_DATA_SCHEMA_PATH ${CHIP_ROOT}/scripts/tools/nrfconnect/nrfconnect_factory_data.schema)
 set(OUTPUT_FILE_PATH ${APPLICATION_BINARY_DIR}/zephyr)
 
-# create a JSON file with all factory data
-nrfconnect_create_factory_data_json(factory_data
-                                    ${FACTORY_DATA_SCRIPT_PATH} 
-                                    ${FACTORY_DATA_SCHEMA_PATH} 
-                                    ${OUTPUT_FILE_PATH})
-
 # create a .hex file with factory data in CBOR format based on the JSON file created previously 
-nrfconnect_create_factory_data_hex_file(factory_data_hex
-                                        factory_data
-                                        ${GENERATE_CBOR_SCRIPT_PATH} 
-                                        ${OUTPUT_FILE_PATH})
+nrfconnect_create_factory_data(factory_data
+                               ${FACTORY_DATA_SCRIPT_PATH} 
+                               ${FACTORY_DATA_SCHEMA_PATH} 
+                               ${OUTPUT_FILE_PATH})
 
 if(CONFIG_CHIP_FACTORY_DATA_MERGE_WITH_FIRMWARE)                                   
     # set custom target for merging factory_data hex file
     set_property(GLOBAL PROPERTY factory_data_PM_HEX_FILE ${OUTPUT_FILE_PATH}/factory_data.hex)
-    set_property(GLOBAL PROPERTY factory_data_PM_TARGET factory_data_hex)
+    set_property(GLOBAL PROPERTY factory_data_PM_TARGET factory_data)
 endif()
 
 

docs/guides/nrfconnect_factory_data_configuration.md

@@ -37,7 +37,7 @@ data secure by applying hardware write protection.
             -   [Appearance field description](#appearance-field-description)
     -   [Enabling factory data support](#enabling-factory-data-support)
     -   [Generating factory data](#generating-factory-data)
-        -   [Creating the factory data JSON file with the first script](#creating-the-factory-data-json-file-with-the-first-script)
+        -   [Creating the factory data JSON and HEX files with the first script](#creating-the-factory-data-json-and-hex-files-with-the-first-script)
         -   [How to set user data](#how-to-set-user-data)
             -   [How to handle user data](#how-to-handle-user-data)
         -   [Verifying using the JSON Schema tool](#verifying-using-the-json-schema-tool)
@@ -218,14 +218,19 @@ file written in another way. To make sure that the JSON file is correct and the
 device is able to read out parameters,
 [verify the file using the JSON schema tool](#verifying-using-the-json-schema-tool).
 
-### Creating the factory data JSON file with the first script
+You can also use only the first script to generate both JSON and HEX files, by
+providing optional `offset` and `size` arguments, which results in invoking the
+script internally. Such option is the recommended one, but invoking two scripts
+one by one is also supported to provide backward compatibility.
+
+### Creating the factory data JSON and HEX files with the first script
 
 A Matter device needs a proper factory data partition stored in the flash memory
 to read out all required parameters during startup. To simplify the factory data
 generation, you can use the
 [generate_nrfconnect_chip_factory_data.py](../../scripts/tools/nrfconnect/generate_nrfconnect_chip_factory_data.py)
 Python script to provide all required parameters and generate a human-readable
-JSON file.
+JSON file and save it to a HEX file.
 
 To use this script, complete the following steps:
 
@@ -245,10 +250,10 @@ To use this script, complete the following steps:
     --sn --vendor_id, --product_id, --vendor_name, --product_name, --date, --hw_ver, --hw_ver_str, --spake2_it, --spake2_salt, --discriminator
     ```
 
-    b. Add output file path:
+    b. Add output path to store .json file, e.g. my_dir/output:
 
     ```
-    -o <path_to_output_json_file>
+    -o <path_to_output_file>
     ```
 
     c. Generate SPAKE2 verifier using one of the following methods:
@@ -341,6 +346,34 @@ To use this script, complete the following steps:
     > `chip-cert` executable. See the note at the end of this section to learn
     > how to get it.
 
+    k. (optional) Partition offset that is an address in device's NVM memory,
+    where factory data will be stored.
+
+    ```
+    --offset <offset>
+    ```
+
+    > **Note:** To generate a HEX file with factory data, you need to provide
+    > both `offset` and `size` optional arguments. As a result,
+    > `factory_data.hex` and `factory_data.bin` files are created in the
+    > `output` directory. The first file contains the required memory offset.
+    > For this reason, it can be programmed directly to the device using a
+    > programmer (for example, `nrfjprog`).
+
+    l. (optional) The maximum partition size in device's NVM memory, where
+    factory data will be stored.
+
+    ```
+    --size <size>
+    ```
+
+    > **Note:** To generate a HEX file with factory data, you need to provide
+    > both `offset` and `size` optional arguments. As a result,
+    > `factory_data.hex` and `factory_data.bin` files are created in the
+    > `output` directory. The first file contains the required memory offset.
+    > For this reason, it can be programmed directly to the device using a
+    > programmer (for example, `nrfjprog`).
+
 4. Run the script using the prepared list of arguments:
 
     ```
@@ -370,8 +403,10 @@ $ python scripts/tools/nrfconnect/generate_nrfconnect_chip_factory_data.py \
 --passcode 20202021 \
 --product_finish "matte" \
 --product_color "black" \
---out "build.json" \
---schema "scripts/tools/nrfconnect/nrfconnect_factory_data.schema"
+--out "build" \
+--schema "scripts/tools/nrfconnect/nrfconnect_factory_data.schema" \
+--offset 0xf7000 \
+--size 0x1000
 ```
 
 As the result of the above example, a unique ID for the rotating device ID is
@@ -570,7 +605,7 @@ To generate a manual pairing code and a QR code, complete the following steps:
     ```
 
 2. Complete steps 1, 2, and 3 from the
-   [Creating the factory data JSON file with the first script](#creating-the-factory-data-json-file-with-the-first-script)
+   [Creating the factory data JSON and HEX files with the first script](#creating-the-factory-data-json-and-hex-files-with-the-first-script)
    section to prepare the final invocation of the Python script.
 
 3. Add the `--generate_onboarding` argument to the Python script final
@@ -686,10 +721,15 @@ The output will look similar to the following one:
 ### Creating a factory data partition with the second script
 
 To store the factory data set in the device's persistent storage, convert the
-data from the JSON file to its binary representation in the CBOR format. To do
-this, use the
+data from the JSON file to its binary representation in the CBOR format. This is
+done by the
+[generate_nrfconnect_chip_factory_data.py](../../scripts/tools/nrfconnect/generate_nrfconnect_chip_factory_data.py),
+if you provide optional `offset` and `size` arguments. If you provided these
+arguments, skip the following steps of this section.
+
+You can skip these optional arguments and do this, using the
 [nrfconnect_generate_partition.py](../../scripts/tools/nrfconnect/nrfconnect_generate_partition.py)
-to generate the factory data partition:
+script, but this is obsolete solution kept only for backward compatibility:
 
 1. Navigate to the _connectedhomeip_ root directory
 2. Run the following command pattern:
@@ -924,14 +964,13 @@ $ west flash
 ## Using own factory data implementation
 
 The [factory data generation process](#generating-factory-data) described above
-is only an example valid for the nRF Connect platform. You can also create a HEX
-file containing all components from the
-[factory data component table](#factory-data-component-table) in any format and
-then implement a parser to read out all parameters and pass them to a provider.
-Each manufacturer can implement a factory data set on its own by implementing a
-parser and a factory data accessor inside the Matter stack. Use the
-[nRF Connect Provider](../../src/platform/nrfconnect/FactoryDataProvider.h) and
-[FactoryDataParser](../../src/platform/nrfconnect/FactoryDataParser.h) as
+is only an example valid for the nRF Connect platform. You can well create a HEX
+file containing all [factory data components](#factory-data-component-table) in
+any format and then implement a parser to read out all parameters and pass them
+to a provider. Each manufacturer can implement a factory data set on its own by
+implementing a parser and a factory data accessor inside the Matter stack. Use
+the [nRF Connect Provider](../../src/platform/nrfconnect/FactoryDataProvider.h)
+and [FactoryDataParser](../../src/platform/nrfconnect/FactoryDataParser.h) as
 examples.
 
 You can read the factory data set from the device's flash memory in different

scripts/tools/nrfconnect/generate_nrfconnect_chip_factory_data.py

@@ -28,6 +28,7 @@
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.serialization import load_der_private_key
+from nrfconnect_generate_partition import PartitionCreator
 
 try:
     import qrcode
@@ -228,6 +229,10 @@ def __init__(self, arguments) -> None:
         self._factory_data = list()
         self._user_data = dict()
 
+        # If .json extension is included in the output path, remove it, as script adds it automatically.
+        if self._args.output.endswith(".json"):
+            self._args.output = self._args.output[:-len(".json")]
+
         try:
             self._validate_args()
         except AssertionError as e:
@@ -244,8 +249,6 @@ def _validate_args(self):
             "Cannot find Spake2+ verifier, to generate a new one please provide passcode (--passcode)"
         assert ((self._args.gen_certs and self._args.chip_cert_path) or (self._args.dac_cert and self._args.pai_cert and self._args.dac_key)), \
             "Cannot find paths to DAC or PAI certificates .der files. To generate a new ones please provide a path to chip-cert executable (--chip_cert_path) and add --gen_certs argument"
-        assert self._args.output.endswith(".json"), \
-            "Output path doesn't contain .json file path. ({})".format(self._args.output)
         assert self._args.passcode not in INVALID_PASSCODES, \
             "Provided invalid passcode!"
 
@@ -310,9 +313,9 @@ def generate_json(self):
             sys.exit(-1)
 
         try:
-            json_file = open(self._args.output, "w+")
+            json_file = open(self._args.output+".json", "w+")
         except FileNotFoundError:
-            print("Cannot create JSON file in this location: {}".format(self._args.output))
+            print("Cannot create JSON file in this location: {}".format(self._args.output+".json"))
             sys.exit(-1)
         with json_file:
             # serialize data
@@ -422,11 +425,11 @@ def _generate_onboarding_data(self):
                                      flow=CommissioningFlow.Standard,
                                      vid=self._args.vendor_id,
                                      pid=self._args.product_id)
-        with open(self._args.output[:-len(".json")] + ".txt", "w") as manual_code_file:
+        with open(self._args.output + ".txt", "w") as manual_code_file:
             manual_code_file.write("Manualcode : " + setup_payload.generate_manualcode() + "\n")
             manual_code_file.write("QRCode : " + setup_payload.generate_qrcode())
         qr = qrcode.make(setup_payload.generate_qrcode())
-        qr.save(self._args.output[:-len(".json")] + ".png")
+        qr.save(self._args.output + ".png")
 
 
 def main():
@@ -441,7 +444,10 @@ def base64_str(s): return base64.b64decode(s)
     parser.add_argument("-s", "--schema", type=str,
                         help="JSON schema file to validate JSON output data")
     parser.add_argument("-o", "--output", type=str, required=True,
-                        help="Output path to store .json file, e.g. my_dir/output.json")
+                        help="Output path to store .json file, e.g. my_dir/output."
+                             "The .json extension will be automatically added by the script and does not need to be provided."
+                             "If provided, an extension will not be added."
+                             "If optional --size and --offset arguments are provided, the script also generates .hex file with factory data.")
     parser.add_argument("-v", "--verbose", action="store_true",
                         help="Run this script with DEBUG logging level")
     parser.add_argument("--include_passcode", action="store_true",
@@ -543,6 +549,10 @@ def base64_str(s): return base64.b64decode(s)
                                     help="[string] Provide one of the product finishes")
     optional_arguments.add_argument("--product_color", type=str, choices=PRODUCT_COLOR_ENUM.keys(),
                                     help="[string] Provide one of the product colors.")
+    optional_arguments.add_argument("--offset", type=allow_any_int,
+                                    help="Partition offset - an address in device's NVM memory, where factory data will be stored.")
+    optional_arguments.add_argument("--size", type=allow_any_int,
+                                    help="The maximum partition size.")
     args = parser.parse_args()
 
     if args.verbose:
@@ -551,9 +561,9 @@ def base64_str(s): return base64.b64decode(s)
         log.basicConfig(format='[%(levelname)s] %(message)s', level=log.INFO)
 
     # check if json file already exist
-    if (exists(args.output) and not args.overwrite):
+    if (exists(args.output + ".json") and not args.overwrite):
         log.error(("Output file: {} already exist, to create a new one add argument '--overwrite'. "
-                  "By default overwriting is disabled").format(args.output))
+                  "By default overwriting is disabled").format(args.output+".json"))
         return
 
     if args.schema and no_jsonschema_module:
@@ -572,6 +582,13 @@ def base64_str(s): return base64.b64decode(s)
     generator = FactoryDataGenerator(args)
     generator.generate_json()
 
+    # If optional partition's offset and size were provided, generate factory data output .hex file.
+    if args.offset and args.size:
+        partition_creator = PartitionCreator(args.offset, args.size, args.output + ".json", args.output)
+        cbor_data = partition_creator.generate_cbor()
+        partition_creator.create_hex(cbor_data)
+        partition_creator.create_bin()
+
 
 if __name__ == "__main__":
     main()