Fix SingleNodeDiscoveryStabilisesEvenWhenDisrupted (elastic#91325)

By increasing the voting duration in case of high delays, to avoid the possible endless repetition of voting rounds. Fixes elastic#89867

Author: kingherc

server/src/main/java/org/elasticsearch/cluster/coordination/ElectionSchedulerFactory.java

@@ -77,15 +77,15 @@ public class ElectionSchedulerFactory {
  ELECTION_MAX_TIMEOUT_SETTING_KEY,
  TimeValue.timeValueSeconds(10),
  TimeValue.timeValueMillis(200),
- TimeValue.timeValueSeconds(300),
+ TimeValue.timeValueSeconds(600),
  Property.NodeScope
  );
 
  public static final Setting<TimeValue> ELECTION_DURATION_SETTING = Setting.timeSetting(
  ELECTION_DURATION_SETTING_KEY,
  TimeValue.timeValueMillis(500),
  TimeValue.timeValueMillis(1),
- TimeValue.timeValueSeconds(300),
+ TimeValue.timeValueSeconds(600),
  Property.NodeScope
  );
 

server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java

@@ -1904,23 +1904,28 @@ public void testSingleNodeDiscoveryWithQuorum() {
  }
  }
 
- @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/89867")
  public void testSingleNodeDiscoveryStabilisesEvenWhenDisrupted() {
- try (
- Cluster cluster = new Cluster(
- 1,
- randomBoolean(),
- Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE).build()
- )
- ) {
-
- // A cluster using single-node discovery should not apply any timeouts to joining or cluster state publication. There are no
- // other options, so there's no point in failing and retrying from scratch no matter how badly disrupted we are and we may as
- // well just wait.
+ // A cluster using single-node discovery should not apply any timeouts to joining or cluster state publication. There are no
+ // other options, so there's no point in failing and retrying from scratch no matter how badly disrupted we are and we may as
+ // well just wait.
+
+ // larger variability is are good for checking that we don't time out, but smaller variability also tightens up the time bound
+ // within which we expect to converge, so use a mix of both
+ final long delayVariabilityMillis = randomLongBetween(DEFAULT_DELAY_VARIABILITY, TimeValue.timeValueMinutes(10).millis());
+
+ Settings.Builder settings = Settings.builder()
+ .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE);
+
+ // If the delay variability is high, set election duration accordingly, to avoid the possible endless repetition of voting rounds.
+ // Note that elections could take even longer than the delay variability, but this seems to be long enough to avoid bad collisions.
+ if (ElectionSchedulerFactory.ELECTION_DURATION_SETTING.getDefault(Settings.EMPTY).getMillis() < delayVariabilityMillis) {
+ settings = settings.put(
+ ElectionSchedulerFactory.ELECTION_DURATION_SETTING.getKey(),
+ TimeValue.timeValueMillis(delayVariabilityMillis)
+ );
+ }
 
- // larger variability is are good for checking that we don't time out, but smaller variability also tightens up the time bound
- // within which we expect to converge, so use a mix of both
- final long delayVariabilityMillis = randomLongBetween(DEFAULT_DELAY_VARIABILITY, TimeValue.timeValueMinutes(10).millis());
+ try (Cluster cluster = new Cluster(1, randomBoolean(), settings.build())) {
  if (randomBoolean()) {
  cluster.runRandomly(true, false, delayVariabilityMillis);
  }

server/src/test/java/org/elasticsearch/cluster/coordination/ElectionSchedulerFactoryTests.java

@@ -192,9 +192,9 @@ public void testSettingsValidation() {
  }
 
  {
- final Settings settings = Settings.builder().put(ELECTION_MAX_TIMEOUT_SETTING.getKey(), "301s").build();
+ final Settings settings = Settings.builder().put(ELECTION_MAX_TIMEOUT_SETTING.getKey(), "601s").build();
  IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ELECTION_MAX_TIMEOUT_SETTING.get(settings));
- assertThat(e.getMessage(), is("failed to parse value [301s] for setting [cluster.election.max_timeout], must be <= [300s]"));
+ assertThat(e.getMessage(), is("failed to parse value [601s] for setting [cluster.election.max_timeout], must be <= [600s]"));
  }
 
  {