Start the helper apps for the MTRDevice pool test as part of the test.

We're actually hitting random failures where the apps get started, but over 15
minutes pass before the test tries to commission them, which causes them to
close their commissioning windows, which makes the test fail.

The fix is to just start the apps right when we're about to need them.

Author: bzbarsky-apple

.github/workflows/darwin.yaml

@@ -97,10 +97,10 @@ jobs:
                       bootstrap-logs-framework-${{ matrix.options.flavor }}
             - name: Build example All Clusters Server
               run: |
-                  scripts/examples/gn_build_example.sh examples/all-clusters-app/linux out/debug chip_config_network_layer_ble=false
+                  scripts/examples/gn_build_example.sh examples/all-clusters-app/linux out/debug/all-clusters-app chip_config_network_layer_ble=false
             - name: Build example OTA Provider
               run: |
-                  scripts/examples/gn_build_example.sh examples/ota-provider-app/linux out/debug chip_config_network_layer_ble=false
+                  scripts/examples/gn_build_example.sh examples/ota-provider-app/linux out/debug/ota-provider-app chip_config_network_layer_ble=false
             - name: Build example OTA Requestor
               run: |
                   scripts/examples/gn_build_example.sh examples/ota-requestor-app/linux out/debug/ota-requestor-app chip_config_network_layer_ble=false non_spec_compliant_ota_action_delay_floor=0
@@ -113,13 +113,8 @@ jobs:
               run: |
                   mkdir -p /tmp/darwin/framework-tests
                   echo "This is a simple log" > /tmp/darwin/framework-tests/end_user_support_log.txt
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --end_user_support_log /tmp/darwin/framework-tests/end_user_support_log.txt > >(tee /tmp/darwin/framework-tests/all-cluster-app.log) 2> >(tee /tmp/darwin/framework-tests/all-cluster-app-err.log >&2) &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --dac_provider ../../../credentials/development/commissioner_dut/struct_cd_origin_pid_vid_correct/test_case_vector.json --product-id 32768 --discriminator 3839 --secured-device-port 5539 --KVS /tmp/chip-all-clusters-app-kvs2 > >(tee /tmp/darwin/framework-tests/all-cluster-app-origin-vid.log) 2> >(tee /tmp/darwin/framework-tests/all-cluster-app-origin-vid-err.log >&2) &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 101 --passcode 1001 --KVS /tmp/chip-all-clusters-app-kvs101 --secured-device-port 5531 &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 102 --passcode 1002 --KVS /tmp/chip-all-clusters-app-kvs102 --secured-device-port 5532 &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 103 --passcode 1003 --KVS /tmp/chip-all-clusters-app-kvs103 --secured-device-port 5533 &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 104 --passcode 1004 --KVS /tmp/chip-all-clusters-app-kvs104 --secured-device-port 5534 &
-                  ../../../out/debug/chip-all-clusters-app --interface-id -1 --discriminator 105 --passcode 1005 --KVS /tmp/chip-all-clusters-app-kvs105 --secured-device-port 5535 &
+                  ../../../out/debug/all-clusters-app/chip-all-clusters-app --interface-id -1 --end_user_support_log /tmp/darwin/framework-tests/end_user_support_log.txt > >(tee /tmp/darwin/framework-tests/all-cluster-app.log) 2> >(tee /tmp/darwin/framework-tests/all-cluster-app-err.log >&2) &
+                  ../../../out/debug/all-clusters-app/chip-all-clusters-app --interface-id -1 --dac_provider ../../../credentials/development/commissioner_dut/struct_cd_origin_pid_vid_correct/test_case_vector.json --product-id 32768 --discriminator 3839 --secured-device-port 5539 --KVS /tmp/chip-all-clusters-app-kvs2 > >(tee /tmp/darwin/framework-tests/all-cluster-app-origin-vid.log) 2> >(tee /tmp/darwin/framework-tests/all-cluster-app-origin-vid-err.log >&2) &
 
                   export TEST_RUNNER_ASAN_OPTIONS=__CURRENT_VALUE__:detect_stack_use_after_return=1
 

src/darwin/Framework/CHIPTests/MTRCommissionableBrowserTests.m

@@ -34,13 +34,8 @@
 static const uint16_t kTestProductId2 = 0x8001u;
 static const uint16_t kTestDiscriminator1 = 3840u;
 static const uint16_t kTestDiscriminator2 = 3839u;
-static const uint16_t kTestDiscriminator3 = 101u;
-static const uint16_t kTestDiscriminator4 = 102u;
-static const uint16_t kTestDiscriminator5 = 103u;
-static const uint16_t kTestDiscriminator6 = 104u;
-static const uint16_t kTestDiscriminator7 = 105u;
 static const uint16_t kDiscoverDeviceTimeoutInSeconds = 10;
-static const uint16_t kExpectedDiscoveredDevicesCount = 7;
+static const uint16_t kExpectedDiscoveredDevicesCount = 2;
 
 // Singleton controller we use.
 static MTRDeviceController * sController = nil;
@@ -102,7 +97,7 @@ - (void)controller:(MTRDeviceController *)controller didFindCommissionableDevice
     XCTAssertEqual(instanceName.length, 16); // The  instance name is random, so just ensure the len is right.
     XCTAssertEqualObjects(vendorId, @(kTestVendorId));
     XCTAssertTrue([productId isEqual:@(kTestProductId1)] || [productId isEqual:@(kTestProductId2)]);
-    XCTAssertTrue([discriminator isEqual:@(kTestDiscriminator1)] || [discriminator isEqual:@(kTestDiscriminator2)] || [discriminator isEqual:@(kTestDiscriminator3)] || [discriminator isEqual:@(kTestDiscriminator4)] || [discriminator isEqual:@(kTestDiscriminator5)] || [discriminator isEqual:@(kTestDiscriminator6)] || [discriminator isEqual:@(kTestDiscriminator7)]);
+    XCTAssertTrue([discriminator isEqual:@(kTestDiscriminator1)] || [discriminator isEqual:@(kTestDiscriminator2)]);
     XCTAssertEqual(commissioningMode, YES);
 
     NSLog(@"Found Device (%@) with discriminator: %@ (vendor: %@, product: %@)", instanceName, discriminator, vendorId, productId);

src/darwin/Framework/CHIPTests/MTRPerControllerStorageTests.m

@@ -29,6 +29,7 @@
 #import "MTRTestKeys.h"
 #import "MTRTestPerControllerStorage.h"
 #import "MTRTestResetCommissioneeHelper.h"
+#import "MTRTestServerAppRunner.h"
 
 static const uint16_t kPairingTimeoutInSeconds = 10;
 static const uint16_t kTimeoutInSeconds = 3;
@@ -2147,7 +2148,7 @@ - (void)testSetMRPParametersWithRunningController
 }
 
 // TODO: This might also want to go in a separate test file, with some shared setup for commissioning devices per test
-- (void)doTestSubscriptionPoolWithSize:(NSInteger)subscriptionPoolSize
+- (void)doTestSubscriptionPoolWithSize:(NSInteger)subscriptionPoolSize deviceOnboardingPayloads:(NSDictionary<NSNumber *, NSString *> *)deviceOnboardingPayloads
 {
     __auto_type * factory = [MTRDeviceControllerFactory sharedInstance];
     XCTAssertNotNil(factory);
@@ -2183,15 +2184,7 @@ - (void)doTestSubscriptionPoolWithSize:(NSInteger)subscriptionPoolSize
 
     XCTAssertEqualObjects(controller.controllerNodeID, nodeID);
 
-    // QRCodes generated for discriminators 101~105 and passcodes 1001~1005
     NSArray<NSNumber *> * orderedDeviceIDs = @[ @(101), @(102), @(103), @(104), @(105) ];
-    NSDictionary<NSNumber *, NSString *> * deviceOnboardingPayloads = @{
-        @(101) : @"MT:00000EBQ15IZC900000",
-        @(102) : @"MT:00000MNY16-AD900000",
-        @(103) : @"MT:00000UZ427GOD900000",
-        @(104) : @"MT:00000CQM00Z.D900000",
-        @(105) : @"MT:00000K0V01FDE900000",
-    };
 
     // Commission 5 devices
     for (NSNumber * deviceID in orderedDeviceIDs) {
@@ -2272,8 +2265,27 @@ - (void)doTestSubscriptionPoolWithSize:(NSInteger)subscriptionPoolSize
 
 - (void)testSubscriptionPool
 {
-    [self doTestSubscriptionPoolWithSize:1];
-    [self doTestSubscriptionPoolWithSize:2];
+    // QRCodes generated for discriminators 1111~1115 and passcodes 1001~1005
+    NSDictionary<NSNumber *, NSString *> * deviceOnboardingPayloads = @{
+        @(101) : @"MT:00000UZ427U0D900000",
+        @(102) : @"MT:00000CQM00BED900000",
+        @(103) : @"MT:00000K0V01TRD900000",
+        @(104) : @"MT:00000SC11293E900000",
+        @(105) : @"MT:00000-O913RGE900000",
+    };
+
+    // Start our helper apps.
+    __auto_type * sortedKeys = [[deviceOnboardingPayloads allKeys] sortedArrayUsingSelector:@selector(compare:)];
+    for (NSNumber * deviceID in sortedKeys) {
+        __auto_type * appRunner = [[MTRTestServerAppRunner alloc] initWithAppName:@"all-clusters"
+                                                                        arguments:@[]
+                                                                          payload:deviceOnboardingPayloads[deviceID]
+                                                                         testcase:self];
+        XCTAssertNotNil(appRunner);
+    }
+
+    [self doTestSubscriptionPoolWithSize:1 deviceOnboardingPayloads:deviceOnboardingPayloads];
+    [self doTestSubscriptionPoolWithSize:2 deviceOnboardingPayloads:deviceOnboardingPayloads];
 }
 
 @end

src/darwin/Framework/CHIPTests/TestHelpers/MTRTestServerAppRunner.m

@@ -90,7 +90,7 @@ - (instancetype)initWithAppName:(NSString *)name arguments:(NSArray<NSString *>
 
     [testcase launchTask:_appTask];
 
-    NSLog(@"Started %@ with arguments %@ stdout=%@ and stderr=%@", name, allArguments, outFile, errorFile);
+    NSLog(@"Started chip-%@-app with arguments %@ stdout=%@ and stderr=%@", name, allArguments, outFile, errorFile);
 
     return self;
 #endif // HAVE_NSTASK