Stop relying on an externally-started all-clusters app in Darwin XCTests. (#38120)

Now that we have machinery for starting an app from inside the test, do that for
all the tests that need one.

Author: bzbarsky-apple

.github/workflows/darwin.yaml

@@ -112,7 +112,6 @@ jobs:
               working-directory: src/darwin/Framework
               run: |
                   mkdir -p /tmp/darwin/framework-tests
-                  ../../../out/debug/all-clusters-app/chip-all-clusters-app --interface-id -1 > >(tee /tmp/darwin/framework-tests/all-cluster-app.log) 2> >(tee /tmp/darwin/framework-tests/all-cluster-app-err.log >&2) &
 
                   export TEST_RUNNER_ASAN_OPTIONS=__CURRENT_VALUE__:detect_stack_use_after_return=1
 

src/darwin/Framework/CHIPTests/MTRBackwardsCompatTests.m

@@ -19,41 +19,26 @@
 #import <Matter/Matter.h>
 
 #import "MTRErrorTestUtils.h"
+#import "MTRTestCase+ServerAppRunner.h"
+#import "MTRTestCase.h"
 #import "MTRTestKeys.h"
-#import "MTRTestResetCommissioneeHelper.h"
 #import "MTRTestStorage.h"
 
 #import <math.h> // For INFINITY
 
-// system dependencies
-#import <XCTest/XCTest.h>
-
-// Fixture: chip-all-clusters-app --KVS "$(mktemp -t chip-test-kvs)" --interface-id -1
-
 static const uint16_t kPairingTimeoutInSeconds = 30;
 static const uint16_t kCASESetupTimeoutInSeconds = 30;
-static const uint16_t kTimeoutInSeconds = 3;
 static const uint64_t kDeviceId = 0x12344321;
 static NSString * kOnboardingPayload = @"MT:-24J0AFN00KA0648G00";
 static const uint16_t kLocalPort = 5541;
 static const uint16_t kTestVendorId = 0xFFF1u;
 
-// This test suite reuses a device object to speed up the test process for CI.
-// The following global variable holds the reference to the device object.
-static MTRBaseDevice * sConnectedDevice;
-
 // Singleton controller we use.
 static MTRDeviceController * sController = nil;
 
 // Keys we can use to restart the controller.
 static MTRTestKeys * sTestKeys = nil;
 
-static MTRBaseDevice * GetConnectedDevice(void)
-{
-    XCTAssertNotNil(sConnectedDevice);
-    return sConnectedDevice;
-}
-
 @interface MTRBackwardsCompatTestPairingDelegate : NSObject <MTRDevicePairingDelegate>
 @property (nonatomic, strong) XCTestExpectation * expectation;
 @end
@@ -88,7 +73,7 @@ - (void)onCommissioningComplete:(NSError *)error
 
 @end
 
-@interface MTRBackwardsCompatTests : XCTestCase
+@interface MTRBackwardsCompatTests : MTRTestCase
 @end
 
 @implementation MTRBackwardsCompatTests
@@ -97,6 +82,11 @@ + (void)setUp
 {
     [super setUp];
 
+    BOOL started = [self startAppWithName:@"all-clusters"
+                                arguments:@[]
+                                  payload:kOnboardingPayload];
+    XCTAssertTrue(started);
+
     XCTestExpectation * expectation = [[XCTestExpectation alloc] initWithDescription:@"Pairing Complete"];
 
     __auto_type * factory = [MTRControllerFactory sharedInstance];
@@ -140,16 +130,13 @@ + (void)setUp
             completionHandler:^(MTRBaseDevice * _Nullable device, NSError * _Nullable error) {
                 XCTAssertEqual(error.code, 0);
                 [connectionExpectation fulfill];
-                sConnectedDevice = device;
                 connectionExpectation = nil;
             }];
     XCTAssertEqual([XCTWaiter waitForExpectations:@[ connectionExpectation ] timeout:kCASESetupTimeoutInSeconds], XCTWaiterResultCompleted);
 }
 
 + (void)tearDown
 {
-    ResetCommissionee(GetConnectedDevice(), dispatch_get_main_queue(), nil, kTimeoutInSeconds);
-
     [sController shutdown];
     XCTAssertFalse([sController isRunning]);
     [[MTRControllerFactory sharedInstance] shutdown];

src/darwin/Framework/CHIPTests/MTRCertificateValidityTests.m

@@ -19,13 +19,11 @@
 #import <Matter/Matter.h>
 
 #import "MTRErrorTestUtils.h"
+#import "MTRTestCase+ServerAppRunner.h"
+#import "MTRTestCase.h"
 #import "MTRTestKeys.h"
-#import "MTRTestResetCommissioneeHelper.h"
 #import "MTRTestStorage.h"
 
-// system dependencies
-#import <XCTest/XCTest.h>
-
 static const uint16_t kPairingTimeoutInSeconds = 30;
 static const uint16_t kTimeoutInSeconds = 3;
 static const uint64_t kDeviceId = 0x12341234;
@@ -195,23 +193,16 @@ - (nullable instancetype)initWithRootKey:(MTRTestKeys *)key fabricID:(NSNumber *
 
 @end
 
-@interface MTRCertificateValidityTests : XCTestCase
+@interface MTRCertificateValidityTests : MTRTestCase
 @end
 
-static BOOL sNeedsStackShutdown = YES;
-
 @implementation MTRCertificateValidityTests
 
 + (void)tearDown
 {
     // Global teardown, runs once
-    if (sNeedsStackShutdown) {
-        // We don't need to worry about ResetCommissionee.  If we get here,
-        // we're running only one of our test methods (using
-        // -only-testing:MatterTests/MTROTAProviderTests/testMethodName), since
-        // we did not run test999_TearDown.
-        [self shutdownStack];
-    }
+    [self shutdownStack];
+    [super tearDown];
 }
 
 - (void)setUp
@@ -246,6 +237,11 @@ - (MTRBaseDevice *)commissionDeviceWithPayload:(NSString *)payloadString nodeID:
 
 - (void)initStack:(MTRTestCertificateIssuer *)certificateIssuer
 {
+    BOOL started = [self startAppWithName:@"all-clusters"
+                                arguments:@[]
+                                  payload:kOnboardingPayload];
+    XCTAssertTrue(started);
+
     __auto_type * factory = [MTRDeviceControllerFactory sharedInstance];
     XCTAssertNotNil(factory);
 
@@ -289,8 +285,6 @@ - (void)initStack:(MTRTestCertificateIssuer *)certificateIssuer
 
 + (void)shutdownStack
 {
-    sNeedsStackShutdown = NO;
-
     MTRDeviceController * controller = sController;
     [controller shutdown];
     XCTAssertFalse([controller isRunning]);
@@ -319,8 +313,6 @@ - (void)test001_TestExpiredCertificates
     }];
     [self waitForExpectations:@[ toggleExpectation ] timeout:kTimeoutInSeconds];
 
-    ResetCommissionee(sConnectedDevice, dispatch_get_main_queue(), self, kTimeoutInSeconds);
-
     [[self class] shutdownStack];
 }
 

src/darwin/Framework/CHIPTests/MTRCommissionableBrowserTests.m

@@ -23,8 +23,6 @@
 #import "MTRTestKeys.h"
 #import "MTRTestStorage.h"
 
-// Fixture 1: chip-all-clusters-app --KVS "$(mktemp -t chip-test-kvs)" --interface-id -1
-
 static const uint16_t kTestVendorId = 0xFFF1u;
 static const __auto_type kTestProductIds = @[ @(0x8000u), @(0x8001u), @(0x8002u) ];
 static const __auto_type kTestDiscriminators = @[ @(2000), @(3839u), @(3840u), @(0xb1e) ];
@@ -168,6 +166,7 @@ + (void)setUp
 
     // Start the helper apps our tests use. Note these payloads match kTestDiscriminators etc.
     for (NSString * payload in @[
+             @"MT:-24J0AFN00KA0648G00",
              @"MT:Y.K90SO527JA0648G00",
              @"MT:-24J0AFN00I40648G00",
          ]) {

src/darwin/Framework/CHIPTests/MTRDeviceTests.m

@@ -32,19 +32,15 @@
 #import "MTRDeviceTestDelegate.h"
 #import "MTRDevice_Internal.h"
 #import "MTRErrorTestUtils.h"
+#import "MTRTestCase+ServerAppRunner.h"
+#import "MTRTestCase.h"
 #import "MTRTestDeclarations.h"
 #import "MTRTestKeys.h"
-#import "MTRTestResetCommissioneeHelper.h"
 #import "MTRTestStorage.h"
 
 #import <math.h> // For INFINITY
 #import <os/lock.h>
 
-// system dependencies
-#import <XCTest/XCTest.h>
-
-// Fixture: chip-all-clusters-app --KVS "$(mktemp -t chip-test-kvs)" --interface-id -1
-
 static const uint16_t kPairingTimeoutInSeconds = 30;
 static const uint16_t kTimeoutInSeconds = 3;
 static const uint64_t kDeviceId = 0x12344321;
@@ -128,7 +124,7 @@ - (void)controller:(MTRDeviceController *)controller commissioningComplete:(NSEr
 
 @end
 
-@interface MTRDeviceTests : XCTestCase
+@interface MTRDeviceTests : MTRTestCase
 
 @end
 
@@ -138,6 +134,13 @@ @implementation MTRDeviceTests
 
 + (void)setUp
 {
+    [super setUp];
+
+    BOOL started = [self startAppWithName:@"all-clusters"
+                                arguments:@[]
+                                  payload:kOnboardingPayload];
+    XCTAssertTrue(started);
+
     XCTestExpectation * pairingExpectation = [[XCTestExpectation alloc] initWithDescription:@"Pairing Complete"];
 
     slocalTestStorageEnabledBeforeUnitTest = MTRDeviceControllerLocalTestStorage.localTestStorageEnabled;
@@ -190,8 +193,6 @@ + (void)setUp
 
 + (void)tearDown
 {
-    ResetCommissionee(GetConnectedDevice(), dispatch_get_main_queue(), nil, kTimeoutInSeconds);
-
     // Restore testing setting to previous state, and remove all persisted attributes
     MTRDeviceControllerLocalTestStorage.localTestStorageEnabled = slocalTestStorageEnabledBeforeUnitTest;
     [sController.controllerDataStore clearAllStoredClusterData];

src/darwin/Framework/CHIPTests/MTROTAProviderTests.m

@@ -23,7 +23,6 @@
 #import "MTRTestCase+ServerAppRunner.h"
 #import "MTRTestCase.h"
 #import "MTRTestKeys.h"
-#import "MTRTestResetCommissioneeHelper.h"
 #import "MTRTestStorage.h"
 
 // system dependencies
@@ -521,22 +520,14 @@ - (instancetype)initWithRawImagePath:(NSString *)rawImagePath
 @end
 
 static BOOL sStackInitRan = NO;
-static BOOL sNeedsStackShutdown = YES;
 
-@implementation MTROTAProviderTests {
-    NSMutableSet<NSNumber *> * _commissionedNodeIDs;
-}
+@implementation MTROTAProviderTests
 
 + (void)tearDown
 {
     // Global teardown, runs once
-    if (sNeedsStackShutdown) {
-        // We don't need to worry about ResetCommissionee.  If we get here,
-        // we're running only one of our test methods (using
-        // -only-testing:MatterTests/MTROTAProviderTests/testMethodName), since
-        // we did not run test999_TearDown.
-        [self shutdownStack];
-    }
+    [self shutdownStack];
+    [super tearDown];
 }
 
 - (void)setUp
@@ -549,8 +540,6 @@ - (void)setUp
         [self initStack];
     }
 
-    _commissionedNodeIDs = [[NSMutableSet alloc] init];
-
     XCTAssertNil(sOTAProviderDelegate.queryImageHandler);
     XCTAssertNil(sOTAProviderDelegate.applyUpdateRequestHandler);
     XCTAssertNil(sOTAProviderDelegate.notifyUpdateAppliedHandler);
@@ -576,11 +565,6 @@ - (void)setUp
 
 - (void)tearDown
 {
-    for (NSNumber * nodeID in _commissionedNodeIDs) {
-        __auto_type * device = [MTRBaseDevice deviceWithNodeID:nodeID controller:sController];
-        ResetCommissionee(device, dispatch_get_main_queue(), self, kTimeoutInSeconds);
-    }
-
     if (sController != nil) {
         [sController shutdown];
         XCTAssertFalse([sController isRunning]);
@@ -619,8 +603,6 @@ - (MTRDevice *)commissionDeviceWithPayload:(NSString *)payloadString nodeID:(NSN
 
     [self waitForExpectations:@[ expectation ] timeout:kPairingTimeoutInSeconds];
 
-    [_commissionedNodeIDs addObject:nodeID];
-
     return [MTRDevice deviceWithNodeID:nodeID controller:sController];
 }
 
@@ -645,8 +627,6 @@ - (void)initStack
 
 + (void)shutdownStack
 {
-    sNeedsStackShutdown = NO;
-
     [[MTRDeviceControllerFactory sharedInstance] stopControllerFactory];
 }
 

src/darwin/Framework/CHIPTests/MTROperationalCertificateIssuerTests.m

@@ -18,12 +18,11 @@
 #import <Matter/Matter.h>
 
 #import "MTRErrorTestUtils.h"
+#import "MTRTestCase+ServerAppRunner.h"
+#import "MTRTestCase.h"
 #import "MTRTestKeys.h"
 #import "MTRTestStorage.h"
 
-// system dependencies
-#import <XCTest/XCTest.h>
-
 static const uint16_t kPairingTimeoutInSeconds = 30;
 static const uint64_t kDeviceId = 0x12344321;
 static NSString * kOnboardingPayload = @"MT:-24J0AFN00KA0648G00";
@@ -110,7 +109,7 @@ - (void)issueOperationalCertificateForRequest:(MTROperationalCSRInfo *)csrInfo
 
 @end
 
-@interface MTROperationalCertificateIssuerTests : XCTestCase
+@interface MTROperationalCertificateIssuerTests : MTRTestCase
 @end
 
 @implementation MTROperationalCertificateIssuerTests
@@ -123,6 +122,11 @@ - (void)setUp
 
 - (void)testFailedCertificateIssuance
 {
+    BOOL started = [self startAppWithName:@"all-clusters"
+                                arguments:@[]
+                                  payload:kOnboardingPayload];
+    XCTAssertTrue(started);
+
     XCTestExpectation * expectation = [self expectationWithDescription:@"Pairing Complete"];
 
     __auto_type * factory = [MTRDeviceControllerFactory sharedInstance];

src/darwin/Framework/CHIPTests/MTRPerControllerStorageTests.m

@@ -334,6 +334,21 @@ @implementation MTRPerControllerStorageTests {
     BOOL _localTestStorageEnabledBeforeUnitTest;
 }
 
++ (void)setUp
+{
+    // Start one instance of all-clusters-app that the various tests below can
+    // then use, so we're not doing quite as much startup logging.  Though it's
+    // not clear that the ResetCommissionee logging is really shorter, so maybe
+    // we should start these on a per-test basis, e.g. in
+    // commissionWithController:newNodeID:onboardingPayload:
+    [super setUp];
+
+    BOOL started = [self startAppWithName:@"all-clusters"
+                                arguments:@[]
+                                  payload:kOnboardingPayload];
+    XCTAssertTrue(started);
+}
+
 - (void)setUp
 {
     // Set detectLeaks true first, in case our superclass wants to do something