Feature Map Attribute conformance and enablement as per the Device type features per endpoint (#1300)

- Creating the feature table and populating it with data from the xml
- Creating the data_type_feature table which references to data type cluster and feature ids. Populating it the same way as the data type attributes and data type commands. For now only adding those features which are of mandatory conformance for now.
- Adding selectDeviceTypeFeaturesByEndpointTypeIdAndClusterId to extract the features per endpoint type and cluster for setting the featureMap attribute appropriately.
- Using the above function from query-config#insertOrUpdateAttributeState to update the featureMap attribute accordingly when its value is 0.
- Start reporting featureMap attribute value warnings for each endpoint based on device type conformance
- Adding unit tests for the device type cluster compliance for feature map attribute

- Adding features in cluster xml as a bitmap such that bitmap name='feature' can be removed from the xml in favor of features which holds the featuremap information as well as defines the feature bitmap such that there is no duplicate information between features and bitmap name=feature

- Showing the correct UI based on category from json files when there is only one zcl and template package and user does not have any packages to select from.

- Making sure the same named feature codes which exist on different bits of the cluster are handled correctly
- Updating the zap schema as per the new changes
- JIRA: ZAPP-1346

Author: brdandu

apack.json

@@ -4,7 +4,7 @@
   "description": "Graphical configuration tool for application and libraries based on Zigbee Cluster Library.",
   "path": [".", "node_modules/.bin/", "ZAP.app/Contents/MacOS"],
   "requiredFeatureLevel": "apack.core:9",
-  "featureLevel": 101,
+  "featureLevel": 102,
   "uc.triggerExtension": "zap",
   "executable": {
     "zap:win32.x86_64": {

docs/zap-schema.svg

@@ -522,6 +522,24 @@ exports.map = {
       deviceVersion: x.DEVICE_VERSION,
     }
   },
+  endpointTypeDeviceExtended: (x) => {
+    if (x == null) return undefined
+    return {
+      id: x.ENDPOINT_TYPE_DEVICE_ID,
+      deviceTypeRef: x.DEVICE_TYPE_REF,
+      endpointTypeRef: x.ENDPOINT_TYPE_REF,
+      endpointTypeId: x.ENDPOINT_TYPE_REF,
+      deviceTypeOrder: x.DEVICE_TYPE_ORDER,
+      deviceIdentifier: x.DEVICE_IDENTIFIER,
+      deviceId: x.DEVICE_IDENTIFIER,
+      deviceVersion: x.DEVICE_VERSION,
+      featureId: x.FEATURE_ID,
+      featureCode: x.FEATURE_CODE,
+      featureName: x.FEATURE_NAME,
+      featureBit: x.FEATURE_BIT,
+      clusterId: x.CLUSTER_REF,
+    }
+  },
   endpointTypeCluster: (x) => {
     if (x == null) return undefined
     return {

src-electron/db/db-mapping.js

@@ -213,6 +213,28 @@ async function insertOrUpdateAttributeState(
       attributeId,
       clusterRef
     )
+  // Looking for the feature map attribute in matter and setting it as per
+  // the device types if default value is 0
+  if (
+    staticAttribute.code == 0xfffc &&
+    staticAttribute.name == 'FeatureMap' &&
+    staticAttribute.defaultValue == 0
+  ) {
+    let featureMapDefaultValue = staticAttribute.defaultValue
+    let mandatoryFeaturesOnEndpointTypeAndCluster =
+      await queryDeviceType.selectDeviceTypeFeaturesByEndpointTypeIdAndClusterId(
+        db,
+        endpointTypeId,
+        clusterRef
+      )
+    let featureMapBitsToBeEnabled =
+      mandatoryFeaturesOnEndpointTypeAndCluster.map((f) => f.featureBit)
+    featureMapBitsToBeEnabled.forEach(
+      (featureBit) =>
+        (featureMapDefaultValue = featureMapDefaultValue | (1 << featureBit))
+    )
+    staticAttribute.defaultValue = featureMapDefaultValue
+  }
   let forcedExternal = await queryUpgrade.getForcedExternalStorage(db)
   staticAttribute.storagePolicy =
     await queryUpgrade.computeStoragePolicyNewConfig(
@@ -314,7 +336,7 @@ async function updateEndpointTypeAttribute(db, id, keyValuePairs) {
   })
   args.push(id)
 
-  let query = `UPDATE ENDPOINT_TYPE_ATTRIBUTE SET 
+  let query = `UPDATE ENDPOINT_TYPE_ATTRIBUTE SET
   ${columns}
 WHERE ENDPOINT_TYPE_ATTRIBUTE_ID = ?`
   return dbApi.dbUpdate(db, query, args)
@@ -508,7 +530,7 @@ INTO ENDPOINT_TYPE_EVENT (
     db,
     `
 UPDATE ENDPOINT_TYPE_EVENT
-SET INCLUDED = ? 
+SET INCLUDED = ?
 WHERE ENDPOINT_TYPE_REF = ?
   AND ENDPOINT_TYPE_CLUSTER_REF = ?
   AND EVENT_REF = ? `,
@@ -1372,17 +1394,17 @@ async function selectEndpointTypeAttributeId(
   let rows = await dbApi.dbAll(
     db,
     `
-SELECT 
+SELECT
   ENDPOINT_TYPE_ATTRIBUTE_ID
-FROM 
+FROM
   ENDPOINT_TYPE_ATTRIBUTE AS ETA
 INNER JOIN
   ATTRIBUTE AS A
 ON
-  ETA.ATTRIBUTE_REF = A.ATTRIBUTE_ID 
+  ETA.ATTRIBUTE_REF = A.ATTRIBUTE_ID
 INNER JOIN
   CLUSTER AS C
-ON 
+ON
   C.CLUSTER_ID = A.CLUSTER_REF
 WHERE
   ETA.ENDPOINT_TYPE_REF = ?

src-electron/db/query-config.js

@@ -312,6 +312,45 @@ WHERE
   )
 }
 
+/**
+ * After loading up device type feature table with the names,
+ * this method links the refererence to actual feature reference.
+ *
+ * @param {*} db
+ * @returns promise of completion
+ */
+async function updateFeatureReferencesForDeviceTypeReferences(db, packageId) {
+  return dbApi.dbUpdate(
+    db,
+    `
+UPDATE
+  DEVICE_TYPE_FEATURE
+SET
+  FEATURE_REF =
+  ( SELECT
+      FEATURE.FEATURE_ID
+    FROM
+      FEATURE
+    WHERE
+      upper(FEATURE.CODE) = upper(DEVICE_TYPE_FEATURE.FEATURE_CODE)
+    AND
+      FEATURE.CLUSTER_REF = (
+        SELECT
+          DEVICE_TYPE_CLUSTER.CLUSTER_REF
+        FROM
+          DEVICE_TYPE_CLUSTER
+        WHERE
+          DEVICE_TYPE_CLUSTER_ID = DEVICE_TYPE_FEATURE.DEVICE_TYPE_CLUSTER_REF
+      )
+    AND
+      FEATURE.PACKAGE_REF = ?
+  )
+WHERE
+  DEVICE_TYPE_FEATURE.FEATURE_REF IS NULL`,
+    [packageId]
+  )
+}
+
 /**
  * This method returns the promise of linking the device type clusters
  * commands and attributes to the correct IDs in the cluster, attribute
@@ -326,7 +365,8 @@ WHERE
 async function updateDeviceTypeEntityReferences(db, packageId) {
   await updateClusterReferencesForDeviceTypeClusters(db, packageId)
   await updateAttributeReferencesForDeviceTypeReferences(db, packageId)
-  return updateCommandReferencesForDeviceTypeReferences(db, packageId)
+  await updateCommandReferencesForDeviceTypeReferences(db, packageId)
+  return updateFeatureReferencesForDeviceTypeReferences(db, packageId)
 }
 
 /**
@@ -355,6 +395,62 @@ async function selectDeviceTypesByEndpointTypeId(db, endpointTypeId) {
   return rows.map(dbMapping.map.endpointTypeDevice)
 }
 
+/**
+ * Retrieves the device type features associated to an endpoint type id and cluster id
+ * Note: Use clusterId as 'all' to get all features for an endpoint type id.
+ * @param {*} db
+ * @param {*} endpointTypeId
+ * @param {*} clusterId
+ * @returns promise with zcl device type feature information based on endpoint type id and cluster id
+ */
+async function selectDeviceTypeFeaturesByEndpointTypeIdAndClusterId(
+  db,
+  endpointTypeId,
+  clusterId
+) {
+  let rows = await dbApi.dbAll(
+    db,
+    `
+  SELECT
+    ETD.ENDPOINT_TYPE_DEVICE_ID,
+    ETD.DEVICE_TYPE_REF,
+    ETD.ENDPOINT_TYPE_REF,
+    ETD.DEVICE_TYPE_ORDER,
+    ETD.DEVICE_IDENTIFIER,
+    ETD.DEVICE_VERSION,
+    FEATURE.FEATURE_ID,
+    FEATURE.NAME AS FEATURE_NAME,
+    FEATURE.CODE AS FEATURE_CODE,
+    FEATURE.BIT AS FEATURE_BIT,
+    DEVICE_TYPE_CLUSTER.CLUSTER_REF
+  FROM
+    ENDPOINT_TYPE_DEVICE AS ETD
+  INNER JOIN
+    DEVICE_TYPE
+  ON
+    ETD.DEVICE_TYPE_REF = DEVICE_TYPE.DEVICE_TYPE_ID
+  INNER JOIN
+    DEVICE_TYPE_CLUSTER
+  ON
+    DEVICE_TYPE_CLUSTER.DEVICE_TYPE_REF = DEVICE_TYPE.DEVICE_TYPE_ID
+  INNER JOIN
+    DEVICE_TYPE_FEATURE
+  ON
+    DEVICE_TYPE_FEATURE.DEVICE_TYPE_CLUSTER_REF = DEVICE_TYPE_CLUSTER.DEVICE_TYPE_CLUSTER_ID
+  INNER JOIN
+    FEATURE
+  ON
+    FEATURE.FEATURE_ID = DEVICE_TYPE_FEATURE.FEATURE_REF
+  WHERE
+    ETD.ENDPOINT_TYPE_REF = ${endpointTypeId}` +
+      (clusterId != 'all'
+        ? ` AND
+        DEVICE_TYPE_CLUSTER.CLUSTER_REF = ${clusterId}`
+        : ``)
+  )
+  return rows.map(dbMapping.map.endpointTypeDeviceExtended)
+}
+
 exports.selectAllDeviceTypes = selectAllDeviceTypes
 exports.selectDeviceTypeById = selectDeviceTypeById
 exports.selectDeviceTypeByCodeAndName = selectDeviceTypeByCodeAndName
@@ -369,3 +465,5 @@ exports.selectDeviceTypeCommandsByDeviceTypeRef =
   selectDeviceTypeCommandsByDeviceTypeRef
 exports.updateDeviceTypeEntityReferences = updateDeviceTypeEntityReferences
 exports.selectDeviceTypesByEndpointTypeId = selectDeviceTypesByEndpointTypeId
+exports.selectDeviceTypeFeaturesByEndpointTypeIdAndClusterId =
+  selectDeviceTypeFeaturesByEndpointTypeIdAndClusterId

src-electron/db/query-device-type.js

@@ -643,6 +643,53 @@ WHERE
   return rows.map(mapFunction)
 }
 
+/**
+ * Given the endpoint type cluster id and attribute code, extract the value
+ * endpoint type attribute information for that attribute within the endpoint
+ * type cluster.
+ * @param {*} db
+ * @param {*} endpointTypeClusterId
+ * @param {*} attributeCode
+ * @param {*} attributeMfgCode
+ */
+async function selectEndpointTypeAttributeFromEndpointTypeClusterId(
+  db,
+  endpointTypeClusterId,
+  attributeCode,
+  attributeMfgCode
+) {
+  let eta = await dbApi.dbGet(
+    db,
+    `
+    SELECT
+      ETC.ENDPOINT_TYPE_REF,
+      ETC.CLUSTER_REF,
+      ETA.INCLUDED,
+      ETA.STORAGE_OPTION,
+      ETA.SINGLETON,
+      ETA.DEFAULT_VALUE
+    FROM
+      ATTRIBUTE AS A
+    INNER JOIN
+      ENDPOINT_TYPE_ATTRIBUTE AS ETA
+    ON
+      ETA.ATTRIBUTE_REF = A.ATTRIBUTE_ID
+    INNER JOIN
+      ENDPOINT_TYPE_CLUSTER AS ETC
+    ON
+      ETA.ENDPOINT_TYPE_CLUSTER_REF = ETC.ENDPOINT_TYPE_CLUSTER_ID
+    WHERE
+      ETC.ENDPOINT_TYPE_CLUSTER_ID = ${endpointTypeClusterId} 
+    AND
+      A.CODE = ${attributeCode}
+    ` +
+      (attributeMfgCode
+        ? ` AND A.MANUFACTURER_CODE = ${attributeMfgCode}`
+        : ` AND A.MANUFACTURER_CODE IS NULL`)
+  )
+  return dbMapping.map.endpointTypeAttribute(eta)
+}
+
 exports.deleteEndpointType = deleteEndpointType
 exports.selectAllEndpointTypes = selectAllEndpointTypes
 exports.selectEndpointTypeIds = selectEndpointTypeIds
@@ -652,13 +699,13 @@ exports.selectAllClustersDetailsFromEndpointTypes =
   selectAllClustersDetailsFromEndpointTypes
 exports.selectEndpointDetailsFromAddedEndpoints =
   selectEndpointDetailsFromAddedEndpoints
-
 exports.selectAllClustersNamesFromEndpointTypes =
   selectAllClustersNamesFromEndpointTypes
 exports.selectAllClustersDetailsIrrespectiveOfSideFromEndpointTypes =
   selectAllClustersDetailsIrrespectiveOfSideFromEndpointTypes
 exports.selectCommandDetailsFromAllEndpointTypeCluster =
   selectCommandDetailsFromAllEndpointTypeCluster
-
 exports.selectClustersAndEndpointDetailsFromEndpointTypes =
   selectClustersAndEndpointDetailsFromEndpointTypes
+exports.selectEndpointTypeAttributeFromEndpointTypeClusterId =
+  selectEndpointTypeAttributeFromEndpointTypeClusterId

src-electron/db/query-endpoint-type.js

@@ -618,6 +618,7 @@ async function insertClusters(db, packageId, data) {
         access: [],
       }
       let pTags = null
+      let pFeatures = null
 
       let i
       for (i = 0; i < lastIdsArray.length; i++) {
@@ -642,16 +643,45 @@ async function insertClusters(db, packageId, data) {
         if ('tags' in data[i]) {
           pTags = insertTags(db, packageId, data[i].tags, lastId)
         }
+
+        if ('features' in data[i]) {
+          pFeatures = insertFeatures(db, packageId, data[i].features, lastId)
+        }
       }
       let pCommand = insertCommands(db, packageId, commands)
       let pAttribute = insertAttributes(db, packageId, attributes)
       let pEvent = insertEvents(db, packageId, events)
       let pArray = [pCommand, pAttribute, pEvent]
       if (pTags != null) pArray.push(pTags)
+      if (pFeatures != null) pArray.push(pFeatures)
       return Promise.all(pArray)
     })
 }
 
+/**
+ * Inserts features into the database.
+ * @param {*} db
+ * @param {*} packageId
+ * @param {*} data
+ * @returns A promise that resolves with array of rowids.
+ */
+async function insertFeatures(db, packageId, data, clusterId) {
+  return dbApi.dbMultiInsert(
+    db,
+    'INSERT INTO FEATURE (PACKAGE_REF, NAME, CODE, BIT, DEFAULT_VALUE, DESCRIPTION, CONFORMANCE, CLUSTER_REF) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+    data.map((feature) => [
+      packageId,
+      feature.name,
+      feature.code,
+      feature.bit,
+      feature.defaultValue,
+      feature.description,
+      feature.conformance,
+      clusterId,
+    ])
+  )
+}
+
 /**
  * Inserts tags into the database.
  * data is an array of objects, containing 'name' and 'description'
@@ -895,6 +925,7 @@ async function insertDeviceTypes(db, packageId, data) {
             let promises = []
             promises.push(insertDeviceTypeAttributes(db, dtClusterRefDataPairs))
             promises.push(insertDeviceTypeCommands(db, dtClusterRefDataPairs))
+            promises.push(insertDeviceTypeFeatures(db, dtClusterRefDataPairs))
             return Promise.all(promises)
           })
         }
@@ -903,6 +934,30 @@ async function insertDeviceTypes(db, packageId, data) {
     })
 }
 
+/**
+ * This handles the loading of device type feature requirements into the database.
+ * There is a need to post-process to attach the actual feature ref after the fact
+ * @param {*} db
+ * @param {*} dtClusterRefDataPairs
+ */
+async function insertDeviceTypeFeatures(db, dtClusterRefDataPairs) {
+  let features = []
+  dtClusterRefDataPairs.map((dtClusterRefDataPair) => {
+    let dtClusterRef = dtClusterRefDataPair.dtClusterRef
+    let clusterData = dtClusterRefDataPair.clusterData
+    if ('features' in clusterData && clusterData.features.length > 0) {
+      clusterData.features.forEach((featureCode) => {
+        features.push([dtClusterRef, featureCode])
+      })
+    }
+  })
+  return dbApi.dbMultiInsert(
+    db,
+    'INSERT INTO DEVICE_TYPE_FEATURE (DEVICE_TYPE_CLUSTER_REF, FEATURE_CODE) VALUES (?, ?)',
+    features
+  )
+}
+
 /**
  * This handles the loading of device type attribute requirements into the database.
  * There is a need to post-process to attach the actual attribute ref after the fact