Implement support for more configurable server endpoints in Matter.framework. (project-chip#31814)

* Implement support for more configurable server endpoints in Matter.framework.

* Public APIs on MTRDeviceController to add/remove an endpoint.
* Internal APIs on MTRDeviceController to query the access grants for a cluster
  path and the declared "minimum privilege needed" to read a given attribute.
* Changes to the controller factory to stop using app/dynamic_server and instead
  use the new infrastructure to expose OTA Provider on endpoint 0.
* Internal APIs on the controller factory to query access grants and declared
  privileges.
* An implemenation of AccessControl::Delegate to do ACL checks using the
  above-mentioned APIs.
* A fix to MTRServerAttribute's setValue for arrays: it was not expecting the
  correct data-value structure for an array.  This requires fixing some tests
  too, to provide the right structures.
* Changes to the MTRServer* data structures to allow passing nil to
  associateWithController, to support the OTA endpoint which is not associated
  with any controller.
* Changes to MTRServerCluster to create an AttributeAccessInterface, the set of
  EmberAfAttributeMetadata needed to represent its attributes, and various other
  things needed to register a cluster with the "ember" bits.
* Changes to MTRServerEndpoint to create an EmberAfEndpointType, a list of
  EmberAfCluster, and various other things needed to register an endpoint with
  the "ember" bits.
* (Re-)addition of MTRIMDispatch to handle command dispatch for OTA and host a
  few other functions the "ember" bits expect to exist.
* Addition of some headers that the "ember" bits expect to exist at specific
  paths and with some specific content: "app/PluginApplicationCallbacks.h" and
  "zap-generated/endpoint_config.h".  Importantly, the latter sets
  FIXED_ENDPOINT_COUNT to 0.
* Addition of unit tests that exercise the non-OTA bits of the above (OTA is
  covered by existing tests), including the ACL checks and so on.
* Including a bunch of src/app and src/app/util files needed for the "ember"
  stuff to work in the framework.
* Turning off the chip_build_controller_dynamic_server bit that we are no longer
  using (and which conflicts with the above bits).
* Configure Darwin to support 254 dynamic endpoints (the maximum that makes
  sense) by default.
* Adjusting include paths for the Xcode darwin-framework-tool project, so that
  it sees the new headers that were added.

* Address review comments.

* Fix test timeout due to resolving IPv4 non-locahost addresses.

* Remove stale comment.

Author: bzbarsky-apple

src/darwin/Framework/CHIP/MTRCallbackBridgeBase.h

@@ -215,6 +215,8 @@ class MTRCallbackBridge : public MTRCallbackBridgeBase {
         }
 
         if (!callbackBridge->mQueue) {
+            ChipLogDetail(Controller, "%s %f seconds: can't dispatch response; no queue", callbackBridge->mCookie.UTF8String,
+                -[callbackBridge->mRequestTime timeIntervalSinceNow]);
             if (!callbackBridge->mKeepAlive) {
                 delete callbackBridge;
             }

src/darwin/Framework/CHIP/MTRDeviceController.h

@@ -22,6 +22,7 @@
 #import <Matter/MTROperationalCertificateIssuer.h>
 
 @class MTRBaseDevice;
+@class MTRServerEndpoint; // Defined in MTRServerEndpoint.h, which imports MTRAccessGrant.h, which imports MTRBaseClusters.h, which imports this file, so we can't import it.
 
 #if MTR_PER_CONTROLLER_STORAGE_ENABLED
 @class MTRDeviceControllerAbstractParameters;
@@ -228,6 +229,29 @@ MTR_AVAILABLE(ios(16.1), macos(13.0), watchos(9.1), tvos(16.1))
 - (NSData * _Nullable)attestationChallengeForDeviceID:(NSNumber *)deviceID
     MTR_AVAILABLE(ios(16.4), macos(13.3), watchos(9.4), tvos(16.4));
 
+/**
+ * Add a server endpoint for this controller.  The endpoint starts off enabled.
+ *
+ * Will fail in the following cases:
+ *
+ * 1) There is already an endpoint defined with the given endpoint id.
+ * 2) There are too many endpoints defined already.
+ */
+- (BOOL)addServerEndpoint:(MTRServerEndpoint *)endpoint MTR_NEWLY_AVAILABLE;
+
+/**
+ * Remove the given server endpoint from this controller.  If the endpoint is
+ * not attached to this controller, will just call the completion and do nothing
+ * else.
+ */
+- (void)removeServerEndpoint:(MTRServerEndpoint *)endpoint queue:(dispatch_queue_t)queue completion:(dispatch_block_t)completion MTR_NEWLY_AVAILABLE;
+
+/**
+ * Remove the given server endpoint without being notified when the removal
+ * completes.
+ */
+- (void)removeServerEndpoint:(MTRServerEndpoint *)endpoint MTR_NEWLY_AVAILABLE;
+
 /**
  * Compute a PASE verifier for the desired setup passcode.
  *

src/darwin/Framework/CHIP/MTRDeviceController.mm

@@ -41,6 +41,7 @@
 #import "MTROperationalCredentialsDelegate.h"
 #import "MTRP256KeypairBridge.h"
 #import "MTRPersistentStorageDelegateBridge.h"
+#import "MTRServerEndpoint_Internal.h"
 #import "MTRSetupPayload.h"
 #import "NSDataSpanConversion.h"
 #import "NSStringSpanConversion.h"
@@ -55,20 +56,23 @@
 
 #include <app-common/zap-generated/cluster-objects.h>
 #include <app/data-model/List.h>
+#include <app/server/Dnssd.h>
 #include <controller/CHIPDeviceController.h>
 #include <controller/CHIPDeviceControllerFactory.h>
 #include <controller/CommissioningWindowOpener.h>
 #include <credentials/FabricTable.h>
 #include <credentials/GroupDataProvider.h>
 #include <credentials/attestation_verifier/DacOnlyPartialAttestationVerifier.h>
 #include <credentials/attestation_verifier/DefaultDeviceAttestationVerifier.h>
+#include <inet/InetInterface.h>
 #include <lib/core/CHIPVendorIdentifiers.hpp>
 #include <platform/LockTracker.h>
 #include <platform/PlatformManager.h>
 #include <setup_payload/ManualSetupPayloadGenerator.h>
 #include <system/SystemClock.h>
 
 #include <atomic>
+#include <dns_sd.h>
 
 #import <os/lock.h>
 
@@ -121,6 +125,9 @@ @implementation MTRDeviceController {
     os_unfair_lock _deviceMapLock; // protects nodeIDToDeviceMap
     MTRCommissionableBrowser * _commissionableBrowser;
     MTRAttestationTrustStoreBridge * _attestationTrustStoreBridge;
+
+    // _serverEndpoints is only touched on the Matter queue.
+    NSMutableArray<MTRServerEndpoint *> * _serverEndpoints;
 }
 
 - (nullable instancetype)initWithParameters:(MTRDeviceControllerAbstractParameters *)parameters error:(NSError * __autoreleasing *)error
@@ -221,6 +228,7 @@ - (instancetype)initWithFactory:(MTRDeviceControllerFactory *)factory
         _factory = factory;
         _deviceMapLock = OS_UNFAIR_LOCK_INIT;
         _nodeIDToDeviceMap = [NSMutableDictionary dictionary];
+        _serverEndpoints = [[NSMutableArray alloc] init];
         _commissionableBrowser = nil;
 
         _deviceControllerDelegateBridge = new MTRDeviceControllerDelegateBridge();
@@ -285,6 +293,11 @@ - (void)shutDownCppController
 {
     assertChipStackLockedByCurrentThread();
 
+    // Shut down all our endpoints.
+    for (MTRServerEndpoint * endpoint in [_serverEndpoints copy]) {
+        [self removeServerEndpointOnMatterQueue:endpoint];
+    }
+
     if (_cppCommissioner) {
         auto * commissionerToShutDown = _cppCommissioner;
         // Flag ourselves as not running before we start shutting down
@@ -947,6 +960,71 @@ - (NSData * _Nullable)attestationChallengeForDeviceID:(NSNumber *)deviceID
     return [self syncRunOnWorkQueueWithReturnValue:block error:nil];
 }
 
+- (BOOL)addServerEndpoint:(MTRServerEndpoint *)endpoint
+{
+    VerifyOrReturnValue([self checkIsRunning], NO);
+
+    if (![_factory addServerEndpoint:endpoint]) {
+        return NO;
+    }
+
+    if (![endpoint associateWithController:self]) {
+        MTR_LOG_ERROR("Failed to associate MTRServerEndpoint with MTRDeviceController");
+        [_factory removeServerEndpoint:endpoint];
+        return NO;
+    }
+
+    [self asyncDispatchToMatterQueue:^() {
+        [self->_serverEndpoints addObject:endpoint];
+        [endpoint registerMatterEndpoint];
+    }
+        errorHandler:^(NSError * error) {
+            MTR_LOG_ERROR("Unexpected failure dispatching to Matter queue on running controller in addServerEndpoint");
+        }];
+    return YES;
+}
+
+- (void)removeServerEndpoint:(MTRServerEndpoint *)endpoint queue:(dispatch_queue_t)queue completion:(dispatch_block_t)completion
+{
+    [self removeServerEndpointInternal:endpoint queue:queue completion:completion];
+}
+
+- (void)removeServerEndpoint:(MTRServerEndpoint *)endpoint
+{
+    [self removeServerEndpointInternal:endpoint queue:nil completion:nil];
+}
+
+- (void)removeServerEndpointInternal:(MTRServerEndpoint *)endpoint queue:(dispatch_queue_t _Nullable)queue completion:(dispatch_block_t _Nullable)completion
+{
+    VerifyOrReturn([self checkIsRunning]);
+
+    // We need to unhook the endpoint from the Matter side before we can start
+    // tearing it down.
+    [self asyncDispatchToMatterQueue:^() {
+        [self removeServerEndpointOnMatterQueue:endpoint];
+        if (queue != nil && completion != nil) {
+            dispatch_async(queue, completion);
+        }
+    }
+        errorHandler:^(NSError * error) {
+            // Error means we got shut down, so the endpoint is removed now.
+            if (queue != nil && completion != nil) {
+                dispatch_async(queue, completion);
+            }
+        }];
+}
+
+- (void)removeServerEndpointOnMatterQueue:(MTRServerEndpoint *)endpoint
+{
+    assertChipStackLockedByCurrentThread();
+
+    [endpoint unregisterMatterEndpoint];
+    [_serverEndpoints removeObject:endpoint];
+    [endpoint invalidate];
+
+    [_factory removeServerEndpoint:endpoint];
+}
+
 - (BOOL)checkForInitError:(BOOL)condition logMsg:(NSString *)logMsg
 {
     if (condition) {
@@ -1230,6 +1308,52 @@ - (void)downloadLogFromNodeWithID:(NSNumber *)nodeID
                              completion:completion];
 }
 
+- (NSArray<MTRAccessGrant *> *)accessGrantsForClusterPath:(MTRClusterPath *)clusterPath
+{
+    assertChipStackLockedByCurrentThread();
+
+    for (MTRServerEndpoint * endpoint in _serverEndpoints) {
+        if ([clusterPath.endpoint isEqual:endpoint.endpointID]) {
+            return [endpoint matterAccessGrantsForCluster:clusterPath.cluster];
+        }
+    }
+
+    // Nothing matched, no grants.
+    return @[];
+}
+
+- (nullable NSNumber *)neededReadPrivilegeForClusterID:(NSNumber *)clusterID attributeID:(NSNumber *)attributeID
+{
+    assertChipStackLockedByCurrentThread();
+
+    for (MTRServerEndpoint * endpoint in _serverEndpoints) {
+        for (MTRServerCluster * cluster in endpoint.serverClusters) {
+            if (![cluster.clusterID isEqual:clusterID]) {
+                continue;
+            }
+
+            for (MTRServerAttribute * attr in cluster.attributes) {
+                if (![attr.attributeID isEqual:attributeID]) {
+                    continue;
+                }
+
+                return @(attr.requiredReadPrivilege);
+            }
+        }
+    }
+
+    return nil;
+}
+
+#ifdef DEBUG
++ (void)forceLocalhostAdvertisingOnly
+{
+    auto interfaceIndex = chip::Inet::InterfaceId::PlatformType(kDNSServiceInterfaceIndexLocalOnly);
+    auto interfaceId = chip::Inet::InterfaceId(interfaceIndex);
+    chip::app::DnssdServer::Instance().SetInterfaceId(interfaceId);
+}
+#endif // DEBUG
+
 @end
 
 /**