From 4e3feaad2add441f054ba0ac5f225cf53612382f Mon Sep 17 00:00:00 2001
From: kkewwei <kkewwei@163.com>
Date: Sat, 21 Sep 2024 13:30:01 +0800
Subject: [PATCH] Fix Flaky Test
 SpecificClusterManagerNodesIT.testElectOnlyBetweenClusterManagerNodes

Signed-off-by: kkewwei <kewei.11@bytedance.com>
Signed-off-by: kkewwei <kkewwei@163.com>
---
 .../cluster/SpecificClusterManagerNodesIT.java | 18 ++++++++++++------
 .../opensearch/test/InternalTestCluster.java   |  3 ++-
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/SpecificClusterManagerNodesIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/SpecificClusterManagerNodesIT.java
index 713873bb222e2..8e3426b9cae26 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/SpecificClusterManagerNodesIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/SpecificClusterManagerNodesIT.java
@@ -44,6 +44,7 @@
 import org.opensearch.test.OpenSearchIntegTestCase.Scope;
 
 import java.io.IOException;
+import java.util.function.Supplier;
 
 import static org.opensearch.test.NodeRoles.clusterManagerNode;
 import static org.opensearch.test.NodeRoles.dataOnlyNode;
@@ -254,9 +255,9 @@ public void testElectOnlyBetweenClusterManagerNodes() throws Exception {
         logger.info("--> closing cluster-manager node (1)");
         client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(clusterManagerNodeName)).get();
         // removing the cluster-manager from the voting configuration immediately triggers the cluster-manager to step down
-        assertBusy(() -> {
-            assertThat(
-                internalCluster().nonClusterManagerClient()
+        Supplier<String> getClusterManagerIfElected = () -> {
+            try {
+                return internalCluster().nonClusterManagerClient()
                     .admin()
                     .cluster()
                     .prepareState()
@@ -265,9 +266,14 @@ public void testElectOnlyBetweenClusterManagerNodes() throws Exception {
                     .getState()
                     .nodes()
                     .getClusterManagerNode()
-                    .getName(),
-                equalTo(nextClusterManagerEligableNodeName)
-            );
+                    .getName();
+            } catch (ClusterManagerNotDiscoveredException e) {
+                logger.debug("failed to get cluster-manager name", e);
+                return null;
+            }
+        };
+        assertBusy(() -> {
+            assertThat(getClusterManagerIfElected.get(), equalTo(nextClusterManagerEligableNodeName));
             assertThat(
                 internalCluster().clusterManagerClient()
                     .admin()
diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
index 7b2c653e9bdb2..59ffa8577a456 100644
--- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
@@ -89,6 +89,7 @@
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.indices.breaker.CircuitBreakerService;
 import org.opensearch.core.util.FileSystemUtils;
+import org.opensearch.discovery.ClusterManagerNotDiscoveredException;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
 import org.opensearch.env.ShardLockObtainFailedException;
@@ -2171,7 +2172,7 @@ public String getClusterManagerName(@Nullable String viaNode) {
             return client.admin().cluster().prepareState().get().getState().nodes().getClusterManagerNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
-            throw new RuntimeException("Can't get cluster-manager node " + e.getMessage(), e);
+            throw new ClusterManagerNotDiscoveredException(e);
         }
     }