Test the Cluster Shard Allocation Explain API with closed indices (#38631)

This pull request modifies the `ClusterAllocationExplainIT` test suite so that it always runs the tests with opened and closed indices. The only test that was not adapted for closed indices is `testAllocationFilteringOnIndexCreation` because we don't allow to directly create indices in the closed state. Relates to #33888

Author: tlrx

rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml

@@ -53,3 +53,47 @@
  - match: { primary: false }
  - is_true: cluster_info
  - is_true: can_allocate
+
+
+---
+"Cluster shard allocation explanation test with a closed index":
+ - skip:
+ version: " - 7.99.99"
+ reason: closed indices are replicated starting version 8.0.0
+
+ - do:
+ indices.create:
+ index: test_closed
+ body: { "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 0 } }
+
+ - match: { acknowledged: true }
+
+ - do:
+ cluster.health:
+ index: test_closed
+ wait_for_status: green
+
+ - do:
+ indices.close:
+ index: test_closed
+
+ - match: { acknowledged: true }
+
+ - do:
+ cluster.health:
+ index: test_closed
+ wait_for_status: green
+
+ - do:
+ cluster.allocation_explain:
+ body: { "index": "test_closed", "shard": 0, "primary": true }
+
+ - match: { current_state: "started" }
+ - is_true: current_node.id
+ - match: { index: "test_closed" }
+ - match: { shard: 0 }
+ - match: { primary: true }
+ - is_true: can_remain_on_current_node
+ - is_true: can_rebalance_cluster
+ - is_true: can_rebalance_to_other_node
+ - is_true: rebalance_explanation

server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java

@@ -19,9 +19,12 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -32,8 +35,10 @@
 import org.elasticsearch.cluster.routing.allocation.MoveDecision;
 import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -52,6 +57,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -70,8 +76,7 @@ public void testUnassignedPrimaryWithExistingIndex() throws Exception {
  logger.info("--> starting 2 nodes");
  internalCluster().startNodes(2);
 
- logger.info("--> creating an index with 1 primary, 0 replicas");
- createIndexAndIndexData(1, 0);
+ prepareIndex(1, 0);
 
  logger.info("--> stopping the node with the primary");
  internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName()));
@@ -149,8 +154,7 @@ public void testUnassignedReplicaDelayedAllocation() throws Exception {
  logger.info("--> starting 3 nodes");
  internalCluster().startNodes(3);
 
- logger.info("--> creating an index with 1 primary, 1 replica");
- createIndexAndIndexData(1, 1);
+ prepareIndex(1, 1);
  logger.info("--> stopping the node with the replica");
  internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode().getName()));
  ensureStableCluster(2);
@@ -268,8 +272,7 @@ public void testUnassignedReplicaWithPriorCopy() throws Exception {
  logger.info("--> starting 3 nodes");
  List<String> nodes = internalCluster().startNodes(3);
 
- logger.info("--> creating an index with 1 primary and 1 replica");
- createIndexAndIndexData(1, 1);
+ prepareIndex(1, 1);
  String primaryNodeName = primaryNodeName();
  nodes.remove(primaryNodeName);
 
@@ -390,7 +393,8 @@ public void testAllocationFilteringOnIndexCreation() throws Exception {
  internalCluster().startNodes(2);
 
  logger.info("--> creating an index with 1 primary, 0 replicas, with allocation filtering so the primary can't be assigned");
- createIndexAndIndexData(1, 0, Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(),
+ prepareIndex(IndexMetaData.State.OPEN, 1, 0,
+ Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(),
  ActiveShardCount.NONE);
 
  boolean includeYesDecisions = randomBoolean();
@@ -481,8 +485,7 @@ public void testAllocationFilteringPreventsShardMove() throws Exception {
  logger.info("--> starting 2 nodes");
  internalCluster().startNodes(2);
 
- logger.info("--> creating an index with 1 primary and 0 replicas");
- createIndexAndIndexData(1, 0);
+ prepareIndex(1, 0);
 
  logger.info("--> setting up allocation filtering to prevent allocation to both nodes");
  client().admin().indices().prepareUpdateSettings("idx").setSettings(
@@ -591,8 +594,7 @@ public void testRebalancingNotAllowed() throws Exception {
  internalCluster().startNode();
  ensureStableCluster(1);
 
- logger.info("--> creating an index with 5 shards, all allocated to the single node");
- createIndexAndIndexData(5, 0);
+ prepareIndex(5, 0);
 
  logger.info("--> disabling rebalancing on the index");
  client().admin().indices().prepareUpdateSettings("idx").setSettings(
@@ -704,8 +706,7 @@ public void testWorseBalance() throws Exception {
  internalCluster().startNode();
  ensureStableCluster(1);
 
- logger.info("--> creating an index with 5 shards, all allocated to the single node");
- createIndexAndIndexData(5, 0);
+ prepareIndex(5, 0);
 
  logger.info("--> setting balancing threshold really high, so it won't be met");
  client().admin().cluster().prepareUpdateSettings().setTransientSettings(
@@ -808,8 +809,7 @@ public void testBetterBalanceButCannotAllocate() throws Exception {
  String firstNode = internalCluster().startNode();
  ensureStableCluster(1);
 
- logger.info("--> creating an index with 5 shards, all allocated to the single node");
- createIndexAndIndexData(5, 0);
+ prepareIndex(5, 0);
 
  logger.info("--> setting up allocation filtering to only allow allocation to the current node");
  client().admin().indices().prepareUpdateSettings("idx").setSettings(
@@ -918,9 +918,9 @@ public void testAssignedReplicaOnSpecificNode() throws Exception {
  logger.info("--> starting 3 nodes");
  List<String> nodes = internalCluster().startNodes(3);
 
- logger.info("--> creating an index with 1 primary and 2 replicas");
  String excludedNode = nodes.get(randomIntBetween(0, 2));
- createIndexAndIndexData(1, 2, Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(),
+ prepareIndex(randomIndexState(), 1, 2,
+ Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(),
  ActiveShardCount.from(2));
 
  boolean includeYesDecisions = randomBoolean();
@@ -1019,8 +1019,7 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception {
  final String replicaNode = internalCluster().startNode();
  final String primaryNode = internalCluster().startNode();
 
- logger.info("--> creating an index with 1 primary and 1 replica");
- createIndexAndIndexData(1, 1,
+ prepareIndex(IndexMetaData.State.OPEN, 1, 1,
  Settings.builder()
  .put("index.routing.allocation.include._name", primaryNode)
  .put("index.routing.allocation.exclude._name", masterNode)
@@ -1037,8 +1036,22 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception {
  logger.info("--> stop node with the replica shard");
  internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));
 
- logger.info("--> index more data, now the replica is stale");
- indexData();
+ final IndexMetaData.State indexState = randomIndexState();
+ if (indexState == IndexMetaData.State.OPEN) {
+ logger.info("--> index more data, now the replica is stale");
+ indexData();
+ } else {
+ logger.info("--> close the index, now the replica is stale");
+ assertAcked(client().admin().indices().prepareClose("idx"));
+
+ final ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth("idx")
+ .setTimeout(TimeValue.timeValueSeconds(30))
+ .setWaitForActiveShards(ActiveShardCount.ONE)
+ .setWaitForNoInitializingShards(true)
+ .setWaitForEvents(Priority.LANGUID)
+ .get();
+ assertThat(clusterHealthResponse.getStatus().value(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW.value()));
+ }
 
  logger.info("--> stop the node with the primary");
  internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
@@ -1147,21 +1160,39 @@ private ClusterAllocationExplanation runExplain(boolean primary, String nodeId,
  return explanation;
  }
 
- private void createIndexAndIndexData(int numPrimaries, int numReplicas) {
- createIndexAndIndexData(numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL);
+ private void prepareIndex(final int numPrimaries, final int numReplicas) {
+ prepareIndex(randomIndexState(), numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL);
  }
 
- private void createIndexAndIndexData(int numPrimaries, int numReplicas, Settings settings, ActiveShardCount activeShardCount) {
- client().admin().indices().prepareCreate("idx")
+ private void prepareIndex(final IndexMetaData.State state, final int numPrimaries, final int numReplicas,
+ final Settings settings, final ActiveShardCount activeShardCount) {
+
+ logger.info("--> creating a {} index with {} primary, {} replicas", state, numPrimaries, numReplicas);
+ assertAcked(client().admin().indices().prepareCreate("idx")
  .setSettings(Settings.builder()
  .put("index.number_of_shards", numPrimaries)
  .put("index.number_of_replicas", numReplicas)
  .put(settings))
  .setWaitForActiveShards(activeShardCount)
- .get();
+ .get());
+
  if (activeShardCount != ActiveShardCount.NONE) {
  indexData();
  }
+ if (state == IndexMetaData.State.CLOSE) {
+ assertAcked(client().admin().indices().prepareClose("idx"));
+
+ final ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth("idx")
+ .setTimeout(TimeValue.timeValueSeconds(30))
+ .setWaitForActiveShards(activeShardCount)
+ .setWaitForEvents(Priority.LANGUID)
+ .get();
+ assertThat(clusterHealthResponse.getStatus().value(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW.value()));
+ }
+ }
+
+ private static IndexMetaData.State randomIndexState() {
+ return randomFrom(IndexMetaData.State.values());
  }
 
  private void indexData() {