Revert "Revert a reverted multi-protocol commit 61d328c" (#1299)

This reverts commit 61d328c.

Fixing Custom XML issues:
- Cleaning up the custom xml UI update as well as the custom xml package uploading such that there are no duplicates
- Fixing the custom xml UI for attributes, commands and events when cluster extensions are added through custom xml. Needed the custom xml package ids to be included
- Fixing the duplicate attributes and commands which show up in the attribute and command UI when a custom cluster with custom attributes and commands is added. This is solved by using a set in : reduceAndConcatenateZclEntity(db, id, [...new Set([...standAlonePackageIds, clusterInfo.packageRef])],
- Adding unique constraint to Attribute table and adding insert or replace to attribute insert query such that custom cluster extensions are not reloaded into the db.
- Making sure the global attributes are part of the custom clusters in static-zcl.js while calling reduceAndConcatenateZclEntity
- Making sure that session partition numbering is correct for the partitionNumber column
- For loading custom cluster extensions such as attributes and commands. Making sure they are loaded into the database based on the top level zcl package loaded. This is needed for sdk upgrades or custom xml in multi-protocol. See changes to zcl-loader-silabs and zcl-loader
- When adding custom xml, there are instances where multiple zcl packages are loaded for the same xml file. This was an issue with the mutation observer. Getting the load new package call out of the observer to fix this. Reproducer: Open zap, add custom xml from extensions, close extensions pane, add another xml from extensions and you will see two instances of the xml added. If you add another xml then you see 3 instances. This is fixed in ZclCustomZclView.vue
- Making sure that the delete button appears across custom xml for removal and doing some minor UI cleanup
- Add more unit tests for custom xml
- Cleaning up zap toolbar UI for logos
- JIRA: ZAPP-1351

Author: brdandu

apack.json

@@ -4,7 +4,7 @@
   "description": "Graphical configuration tool for application and libraries based on Zigbee Cluster Library.",
   "path": [".", "node_modules/.bin/", "ZAP.app/Contents/MacOS"],
   "requiredFeatureLevel": "apack.core:9",
-  "featureLevel": 100,
+  "featureLevel": 101,
   "uc.triggerExtension": "zap",
   "executable": {
     "zap:win32.x86_64": {

docs/zap-schema.svg

@@ -58,6 +58,7 @@
     "package-metadata": "node src-script/zap-package-metadata.js",
     "zap": "node src-script/zap-start.js --logToStdout --gen ./test/gen-template/zigbee/gen-templates.json",
     "zapall": "node src-script/zap-start.js --logToStdout --zcl ./test/resource/meta/zcl.json --zcl ./zcl-builtin/silabs/zcl.json  --zcl ./zcl-builtin/matter/zcl.json --gen ./test/gen-template/zigbee/gen-templates.json --gen ./test/gen-template/matter/gen-test.json --gen ./test/resource/meta/gen-test.json",
+    "zapmultiprotocol": "node src-script/zap-start.js --logToStdout --zcl ./zcl-builtin/silabs/zcl.json  --zcl ./zcl-builtin/matter/zcl-with-test-extensions.json --gen ./test/gen-template/zigbee/gen-templates.json --gen ./test/gen-template/matter/gen-test.json",
     "zapzigbee": "node src-script/zap-start.js --logToStdout --zcl ./zcl-builtin/silabs/zcl.json --gen ./test/gen-template/zigbee/gen-templates.json",
     "zapmatter": "node src-script/zap-start.js --logToStdoutc --zcl ./zcl-builtin/matter/zcl.json --gen ./test/gen-template/matter/gen-test.json",
     "zapmeta": "node src-script/zap-start.js --logToStdout --zcl ./test/resource/meta/zcl.json --gen ./test/resource/meta/gen-test.json --in ./test/resource/test-meta.zap",
@@ -66,6 +67,7 @@
     "zigbeezap-devserver": "node src-script/zap-start.js server --stateDirectory ~/.zap/zigbee-server/ --allowCors --logToStdout --gen ./test/gen-template/zigbee/gen-templates.json --reuseZapInstance",
     "matterzap-devserver": "node src-script/zap-start.js server --stateDirectory ~/.zap/matter-server/ --allowCors --logToStdout --zcl ./zcl-builtin/matter/zcl.json --gen ./test/resource/meta/gen-test.json --reuseZapInstance",
     "matterzap-devserver2": "node src-script/zap-start.js server --stateDirectory --zcl ./zcl-builtin/matter/zcl.json --allowCors --logToStdout --gen ./test/gen-template/matter3/t.json --reuseZapInstance",
+    "zapall-devserver": "node src-script/zap-start.js server --stateDirectory --zcl ./zcl-builtin/silabs/zcl.json  --zcl ./zcl-builtin/matter/zcl.json --allowCors --logToStdout --gen ./test/gen-template/zigbee/gen-templates.json --gen ./test/gen-template/matter/gen-test.json --gen --reuseZapInstance",
     "server": "node src-script/zap-start.js server --stateDirectory ~/.zap/server/ --allowCors --logToStdout --zcl ./zcl-builtin/silabs/zcl.json  --zcl ./zcl-builtin/matter/zcl.json --gen ./test/gen-template/zigbee/gen-templates.json  --gen ./test/gen-template/matter/gen-test.json ",
     "stop": "node src-script/zap-start.js stop --reuseZapInstance",
     "status": "node src-script/zap-start.js status --reuseZapInstance",

package.json

@@ -384,6 +384,23 @@ exports.map = {
       name: x.NAME,
       caption: x.DESCRIPTION,
       class: x.CLASS,
+      packageRef: x.PACKAGE_REF,
+    }
+  },
+
+  deviceTypeExtended: (x) => {
+    if (x == null) return undefined
+    return {
+      id: x.DEVICE_TYPE_ID,
+      code: x.CODE,
+      profileId: x.PROFILE_ID,
+      domain: x.DOMAIN,
+      label: x.NAME,
+      name: x.NAME,
+      caption: x.DESCRIPTION,
+      class: x.CLASS,
+      packageRef: x.PACKAGE_REF,
+      category: x.CATEGORY,
     }
   },
 
@@ -452,6 +469,25 @@ exports.map = {
       parentEndpointIdentifier: x.PARENT_ENDPOINT_IDENTIFIER,
     }
   },
+  endpointExtended: (x) => {
+    if (x == null) return undefined
+    return {
+      id: x.ENDPOINT_ID,
+      endpointRef: x.ENDPOINT_ID,
+      sessionRef: x.SESSION_REF,
+      endpointIdentifier: x.ENDPOINT_IDENTIFIER,
+      endpointId: x.ENDPOINT_IDENTIFIER,
+      endpointTypeRef: x.ENDPOINT_TYPE_REF,
+      profileId: x.PROFILE,
+      networkId: x.NETWORK_IDENTIFIER,
+      endpointVersion: x.DEVICE_VERSION, // Left for backwards compatibility
+      deviceVersion: x.DEVICE_VERSION,
+      deviceIdentifier: x.DEVICE_IDENTIFIER,
+      parentRef: x.PARENT_ENDPOINT_REF,
+      parentEndpointIdentifier: x.PARENT_ENDPOINT_IDENTIFIER,
+      category: x.CATEGORY, // Category of the device type coming from the zcl package it belongs to.
+    }
+  },
   endpointType: (x) => {
     if (x == null) return undefined
     return {
@@ -643,6 +679,8 @@ exports.map = {
       sessionRef: x.SESSION_REF,
       required: x.REQUIRED,
       type: x.TYPE,
+      sessionPartitionId: x.SESSION_PARTITION_ID,
+      category: x.CATEGORY,
     }
   },
   sessionLog: (x) => {
@@ -664,6 +702,16 @@ exports.map = {
       newNotification: x.NEW_NOTIFICATION == 1,
     }
   },
+
+  sessionPartition: (x) => {
+    if (x == null) return undefined
+    return {
+      sessionPartitionId: x.SESSION_PARTITION_ID,
+      sessionRef: x.SESSION_REF,
+      sessionPartitionNumber: x.SESSION_PARTITION_NUMBER,
+    }
+  },
+
   user: (x) => {
     if (x == null) return undefined
     return {

src-electron/db/db-mapping.js

@@ -1102,8 +1102,12 @@ INNER JOIN
   CLUSTER
 ON
   ENDPOINT_TYPE_CLUSTER.CLUSTER_REF = CLUSTER.CLUSTER_ID
+INNER JOIN
+  SESSION_PARTITION
+ON
+ENDPOINT_TYPE.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
 WHERE
-  ENDPOINT_TYPE.SESSION_REF = ${sessionId}
+  SESSION_PARTITION.SESSION_REF = ${sessionId}
 AND
   ENDPOINT_TYPE_CLUSTER.ENABLED=1
 AND

src-electron/db/query-attribute.js

@@ -218,8 +218,12 @@ async function selectAllUserClustersWithTokenAttributes(
     CLUSTER
   ON
     ENDPOINT_TYPE_CLUSTER.CLUSTER_REF = CLUSTER.CLUSTER_ID
+  INNER JOIN
+    SESSION_PARTITION
+  ON
+  ENDPOINT_TYPE.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
   WHERE
-    ENDPOINT_TYPE.SESSION_REF = ?
+    SESSION_PARTITION.SESSION_REF = ?
   AND
     ENDPOINT_TYPE_CLUSTER.ENABLED=1
   AND

src-electron/db/query-cluster.js

@@ -27,6 +27,7 @@ const dbEnum = require('../../src-shared/db-enum.js')
 const queryZcl = require('./query-zcl.js')
 const queryUpgrade = require('../sdk/matter.js')
 const queryDeviceType = require('./query-device-type')
+const querySession = require('./query-session')
 const queryCommand = require('./query-command.js')
 const restApi = require('../../src-shared/rest-api.js')
 const _ = require('lodash')
@@ -576,16 +577,18 @@ async function selectCountOfEndpointsWithGivenEndpointIdentifier(
 /**
  * Promises to add an endpoint type.
  *
- * @export
  * @param {*} db
- * @param {*} sessionId
+ * @param {*} sessionPartitionInfo
  * @param {*} name
  * @param {*} deviceTypeRef
+ * @param {*} deviceTypeIdentifier
+ * @param {*} deviceTypeVersion
+ * @param {*} doTransaction
  * @returns Promise to update endpoints.
  */
 async function insertEndpointType(
   db,
-  sessionId,
+  sessionPartitionInfo,
   name,
   deviceTypeRef,
   deviceTypeIdentifier,
@@ -604,8 +607,8 @@ async function insertEndpointType(
   // Insert endpoint type
   let newEndpointTypeId = await dbApi.dbInsert(
     db,
-    'INSERT OR REPLACE INTO ENDPOINT_TYPE ( SESSION_REF, NAME ) VALUES ( ?, ?)',
-    [sessionId, name]
+    'INSERT OR REPLACE INTO ENDPOINT_TYPE ( SESSION_PARTITION_REF, NAME ) VALUES ( ?, ?)',
+    [sessionPartitionInfo.sessionPartitionId, name]
   )
 
   // Creating endpoint type and device type ref combinations along with order of insertion
@@ -635,7 +638,7 @@ async function insertEndpointType(
       db,
       'ERROR',
       isErrorStringPresent ? err.split('Error:')[1] : err,
-      sessionId,
+      sessionPartitionInfo.sessionRef,
       1,
       1
     )
@@ -656,7 +659,7 @@ async function insertEndpointType(
   for (const dtRef of deviceTypeRefs) {
     await setEndpointDefaults(
       db,
-      sessionId,
+      sessionPartitionInfo.sessionRef,
       newEndpointTypeId,
       dtRef,
       doTransaction
@@ -679,7 +682,8 @@ async function duplicateEndpointType(db, endpointTypeId) {
     db,
     `
     SELECT
-      ENDPOINT_TYPE.SESSION_REF,
+      SESSION_PARTITION.SESSION_REF,
+      SESSION_PARTITION.SESSION_PARTITION_ID,
       ENDPOINT_TYPE.NAME,
       ENDPOINT_TYPE_DEVICE.DEVICE_TYPE_REF,
       ENDPOINT_TYPE_DEVICE.DEVICE_IDENTIFIER,
@@ -690,6 +694,10 @@ async function duplicateEndpointType(db, endpointTypeId) {
       ENDPOINT_TYPE_DEVICE
     ON
       ENDPOINT_TYPE.ENDPOINT_TYPE_ID = ENDPOINT_TYPE_DEVICE.ENDPOINT_TYPE_REF
+    INNER JOIN
+      SESSION_PARTITION
+    ON
+      ENDPOINT_TYPE.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
     WHERE
       ENDPOINT_TYPE_DEVICE.ENDPOINT_TYPE_REF = ?`,
     [endpointTypeId]
@@ -699,9 +707,12 @@ async function duplicateEndpointType(db, endpointTypeId) {
     // Enter into the endpoint_type table
     newEndpointTypeId = await dbApi.dbInsert(
       db,
-      `INSERT INTO ENDPOINT_TYPE (SESSION_REF, NAME)
+      `INSERT INTO ENDPOINT_TYPE (SESSION_PARTITION_REF, NAME)
       VALUES (?, ?)`,
-      [endpointTypeDeviceInfo[0].SESSION_REF, endpointTypeDeviceInfo[0].NAME]
+      [
+        endpointTypeDeviceInfo[0].SESSION_PARTITION_ID,
+        endpointTypeDeviceInfo[0].NAME,
+      ]
     )
 
     // Enter into the endpoint_type_device table to establish the endpoint_type
@@ -758,8 +769,12 @@ async function updateEndpointType(db, sessionId, endpointTypeId, changesArray) {
     INNER JOIN
       ENDPOINT_TYPE_DEVICE
     ON ENDPOINT_TYPE.ENDPOINT_TYPE_ID = ENDPOINT_TYPE_DEVICE.ENDPOINT_TYPE_REF
+    INNER JOIN
+      SESSION_PARTITION
+    ON
+      ENDPOINT_TYPE.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
     WHERE
-      ENDPOINT_TYPE_ID = ? AND SESSION_REF = ?
+      ENDPOINT_TYPE.ENDPOINT_TYPE_ID = ? AND SESSION_PARTITION.SESSION_REF = ?
     ORDER BY
       ENDPOINT_TYPE_DEVICE.DEVICE_TYPE_ORDER`,
     [endpointTypeId, sessionId]
@@ -873,7 +888,10 @@ async function updateEndpointType(db, sessionId, endpointTypeId, changesArray) {
 /**
  * Promise to set the default attributes and clusters for a endpoint type.
  * @param {*} db
+ * @param {*} sessionId
  * @param {*} endpointTypeId
+ * @param {*} deviceTypeRef
+ * @param {*} doTransaction
  */
 async function setEndpointDefaults(
   db,
@@ -893,7 +911,20 @@ async function setEndpointDefaults(
   if (pkgs == null || pkgs.length < 1)
     throw new Error('Could not locate package id for a given session.')
 
+  let deviceTypeInfo =
+    await querySession.selectDeviceTypePackageInfoFromDeviceTypeId(
+      db,
+      deviceTypeRef
+    )
+  let endpointTypeCategory =
+    deviceTypeInfo.length > 0 ? deviceTypeInfo[0].category : null
   let packageId = pkgs[0].id
+  for (let i = 0; i < pkgs.length; i++) {
+    if (pkgs[i].category == endpointTypeCategory) {
+      packageId = pkgs[i].id
+      break
+    }
+  }
   let clusters = await queryDeviceType.selectDeviceTypeClustersByDeviceTypeRef(
     db,
     deviceTypeRef
@@ -1216,7 +1247,16 @@ async function resolveNonOptionalAndReportableAttributes(
 async function selectEndpointTypeCount(db, sessionId) {
   let x = await dbApi.dbGet(
     db,
-    'SELECT COUNT(ENDPOINT_TYPE_ID) AS CNT FROM ENDPOINT_TYPE WHERE SESSION_REF = ?',
+    `SELECT
+      COUNT(ENDPOINT_TYPE_ID) AS CNT
+    FROM
+      ENDPOINT_TYPE
+    INNER JOIN
+      SESSION_PARTITION
+    ON
+      SESSION_PARTITION.SESSION_PARTITION_ID = ENDPOINT_TYPE.SESSION_PARTITION_REF
+    WHERE
+      SESSION_PARTITION.SESSION_REF = ?`,
     [sessionId]
   )
   return x['CNT']
@@ -1243,7 +1283,11 @@ SELECT
   COUNT(ENDPOINT_TYPE_ID)
 FROM
   ENDPOINT_TYPE
-WHERE SESSION_REF = ?
+INNER JOIN
+  SESSION_PARTITION
+ON
+  SESSION_PARTITION.SESSION_PARTITION_ID = ENDPOINT_TYPE.SESSION_PARTITION_REF
+WHERE SESSION_PARTITION.SESSION_REF = ?
   AND ENDPOINT_TYPE_ID IN
       (SELECT ENDPOINT_TYPE_REF
        FROM ENDPOINT_TYPE_CLUSTER
@@ -1404,8 +1448,12 @@ JOIN
   ATTRIBUTE AS A ON ETA.ATTRIBUTE_REF = A.ATTRIBUTE_ID
 JOIN
   ENDPOINT_TYPE AS ET ON ETA.ENDPOINT_TYPE_REF = ET.ENDPOINT_TYPE_ID
+INNER JOIN
+  SESSION_PARTITION
+ON
+  ET.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
 WHERE
-  ET.SESSION_REF = ? AND ETA.INCLUDED = 1
+  SESSION_PARTITION.SESSION_REF = ? AND ETA.INCLUDED = 1
 ORDER BY
   CLUSTER_CODE, ATTRIBUTE_CODE
   `,

src-electron/db/query-config.js

@@ -26,15 +26,15 @@ const dbMapping = require('./db-mapping')
 /**
  * Retrieves all the device types in the database.
  *
- * @export
  * @param {*} db
+ * @param {*} packageId
  * @returns Promise that resolves with the rows of device types.
  */
 async function selectAllDeviceTypes(db, packageId) {
   return dbApi
     .dbAll(
       db,
-      'SELECT DEVICE_TYPE_ID, DOMAIN, CODE, PROFILE_ID, NAME, DESCRIPTION, CLASS FROM DEVICE_TYPE WHERE PACKAGE_REF = ? ORDER BY DOMAIN, CODE',
+      'SELECT DEVICE_TYPE_ID, DOMAIN, CODE, PROFILE_ID, NAME, DESCRIPTION, CLASS, PACKAGE_REF FROM DEVICE_TYPE WHERE PACKAGE_REF = ? ORDER BY DOMAIN, CODE',
       [packageId]
     )
     .then((rows) => rows.map(dbMapping.map.deviceType))
@@ -51,7 +51,7 @@ async function selectDeviceTypeById(db, id) {
   return dbApi
     .dbGet(
       db,
-      'SELECT DEVICE_TYPE_ID, DOMAIN, CODE, PROFILE_ID, NAME, DESCRIPTION, CLASS FROM DEVICE_TYPE WHERE DEVICE_TYPE_ID = ?',
+      'SELECT DEVICE_TYPE_ID, DOMAIN, CODE, PROFILE_ID, NAME, DESCRIPTION, CLASS, PACKAGE_REF FROM DEVICE_TYPE WHERE DEVICE_TYPE_ID = ?',
       [id]
     )
     .then(dbMapping.map.deviceType)