Start the helper apps for the MTRDevice pool test as part of the test.

We're actually hitting random failures where the apps get started, but over 15
minutes pass before the test tries to commission them, which causes them to
close their commissioning windows, which makes the test fail.

The fix is to just start the apps right when we're about to need them.

Author: bzbarsky-apple

.github/workflows/darwin.yaml

@@ -115,11 +115,6 @@ jobs:
                   echo "This is a simple log" > /tmp/darwin/framework-tests/end_user_support_log.txt
                   ../../../out/debug/chip-all-clusters-app --interface-id -1 --end_user_support_log /tmp/darwin/framework-tests/end_user_support_log.txt > >(tee /tmp/darwin/framework-tests/all-cluster-app.log) 2> >(tee /tmp/darwin/framework-tests/all-cluster-app-err.log >&2) &
                   ../../../out/debug/chip-all-clusters-app --interface-id -1 --dac_provider ../../../credentials/development/commissioner_dut/struct_cd_origin_pid_vid_correct/test_case_vector.json --product-id 32768 --discriminator 3839 --secured-device-port 5539 --KVS /tmp/chip-all-clusters-app-kvs2 > >(tee /tmp/darwin/framework-tests/all-cluster-app-origin-vid.log) 2> >(tee /tmp/darwin/framework-tests/all-cluster-app-origin-vid-err.log >&2) &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 101 --passcode 1001 --KVS /tmp/chip-all-clusters-app-kvs101 --secured-device-port 5531 &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 102 --passcode 1002 --KVS /tmp/chip-all-clusters-app-kvs102 --secured-device-port 5532 &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 103 --passcode 1003 --KVS /tmp/chip-all-clusters-app-kvs103 --secured-device-port 5533 &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 104 --passcode 1004 --KVS /tmp/chip-all-clusters-app-kvs104 --secured-device-port 5534 &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 105 --passcode 1005 --KVS /tmp/chip-all-clusters-app-kvs105 --secured-device-port 5535 &
 
                   export TEST_RUNNER_ASAN_OPTIONS=__CURRENT_VALUE__:detect_stack_use_after_return=1
 

src/darwin/Framework/CHIPTests/MTRPerControllerStorageTests.m

@@ -29,6 +29,7 @@
 #import "MTRTestKeys.h"
 #import "MTRTestPerControllerStorage.h"
 #import "MTRTestResetCommissioneeHelper.h"
+#import "MTRTestServerAppRunner.h"
 
 static const uint16_t kPairingTimeoutInSeconds = 10;
 static const uint16_t kTimeoutInSeconds = 3;
@@ -2147,7 +2148,7 @@ - (void)testSetMRPParametersWithRunningController
 }
 
 // TODO: This might also want to go in a separate test file, with some shared setup for commissioning devices per test
-- (void)doTestSubscriptionPoolWithSize:(NSInteger)subscriptionPoolSize
+- (void)doTestSubscriptionPoolWithSize:(NSInteger)subscriptionPoolSize deviceOnboardingPayloads:(NSDictionary<NSNumber *, NSString *> *)deviceOnboardingPayloads
 {
     __auto_type * factory = [MTRDeviceControllerFactory sharedInstance];
     XCTAssertNotNil(factory);
@@ -2183,15 +2184,7 @@ - (void)doTestSubscriptionPoolWithSize:(NSInteger)subscriptionPoolSize
 
     XCTAssertEqualObjects(controller.controllerNodeID, nodeID);
 
-    // QRCodes generated for discriminators 101~105 and passcodes 1001~1005
     NSArray<NSNumber *> * orderedDeviceIDs = @[ @(101), @(102), @(103), @(104), @(105) ];
-    NSDictionary<NSNumber *, NSString *> * deviceOnboardingPayloads = @{
-        @(101) : @"MT:00000EBQ15IZC900000",
-        @(102) : @"MT:00000MNY16-AD900000",
-        @(103) : @"MT:00000UZ427GOD900000",
-        @(104) : @"MT:00000CQM00Z.D900000",
-        @(105) : @"MT:00000K0V01FDE900000",
-    };
 
     // Commission 5 devices
     for (NSNumber * deviceID in orderedDeviceIDs) {
@@ -2272,8 +2265,27 @@ - (void)doTestSubscriptionPoolWithSize:(NSInteger)subscriptionPoolSize
 
 - (void)testSubscriptionPool
 {
-    [self doTestSubscriptionPoolWithSize:1];
-    [self doTestSubscriptionPoolWithSize:2];
+    // QRCodes generated for discriminators 1111~1115 and passcodes 1001~1005
+    NSDictionary<NSNumber *, NSString *> * deviceOnboardingPayloads = @{
+        @(101) : @"MT:00000UZ427U0D900000",
+        @(102) : @"MT:00000CQM00BED900000",
+        @(103) : @"MT:00000K0V01TRD900000",
+        @(104) : @"MT:00000SC11293E900000",
+        @(105) : @"MT:00000-O913RGE900000",
+    };
+
+    // Start our helper apps.
+    __auto_type * sortedKeys = [[deviceOnboardingPayloads allKeys] sortedArrayUsingSelector:@selector(compare:)];
+    for (NSNumber * deviceID in sortedKeys) {
+        __auto_type * appRunner = [[MTRTestServerAppRunner alloc] initWithAppName:@"all-clusters"
+                                                                        arguments:@[]
+                                                                          payload:deviceOnboardingPayloads[deviceID]
+                                                                         testcase:self];
+        XCTAssertNotNil(appRunner);
+    }
+
+    [self doTestSubscriptionPoolWithSize:1 deviceOnboardingPayloads:deviceOnboardingPayloads];
+    [self doTestSubscriptionPoolWithSize:2 deviceOnboardingPayloads:deviceOnboardingPayloads];
 }
 
 @end

src/darwin/Framework/CHIPTests/TestHelpers/MTRTestServerAppRunner.m

@@ -90,7 +90,7 @@ - (instancetype)initWithAppName:(NSString *)name arguments:(NSArray<NSString *>
 
     [testcase launchTask:_appTask];
 
-    NSLog(@"Started %@ with arguments %@ stdout=%@ and stderr=%@", name, allArguments, outFile, errorFile);
+    NSLog(@"Started chip-%@-app with arguments %@ stdout=%@ and stderr=%@", name, allArguments, outFile, errorFile);
 
     return self;
 #endif // HAVE_NSTASK