Readd index.lifecycle.skip setting (#128736) (#128901)

We want to be able to skip specific indices in ILM again for #109206. This is essentially just a revert of #34823. (cherry picked from commit 269fbbc) # Conflicts: #	server/src/main/java/org/elasticsearch/TransportVersions.java

Author: nielsbauman

docs/changelog/128736.yaml

@@ -0,0 +1,5 @@
+pr: 128736
+summary: Add `index.lifecycle.skip` index-scoped setting to instruct ILM to skip processing specific indices
+area: ILM+SLM
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -233,6 +233,7 @@ static TransportVersion def(int id) {
  public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES_ADDED_8_19 = def(8_841_0_40);
  public static final TransportVersion DATA_STREAM_OPTIONS_API_REMOVE_INCLUDE_DEFAULTS_8_19 = def(8_841_0_41);
  public static final TransportVersion JOIN_ON_ALIASES_8_19 = def(8_841_0_42);
+ public static final TransportVersion ILM_ADD_SKIP_SETTING_8_19 = def(8_841_0_43);
  /*
  * STOP! READ THIS FIRST! No, really,
  * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java

@@ -55,6 +55,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
  private static final ParseField REPOSITORY_NAME = new ParseField("repository_name");
  private static final ParseField SHRINK_INDEX_NAME = new ParseField("shrink_index_name");
  private static final ParseField SNAPSHOT_NAME = new ParseField("snapshot_name");
+ private static final ParseField SKIP_NAME = new ParseField("skip");
 
  public static final ConstructingObjectParser<IndexLifecycleExplainResponse, Void> PARSER = new ConstructingObjectParser<>(
  "index_lifecycle_explain_response",
@@ -78,7 +79,8 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
  (String) a[18],
  (BytesReference) a[11],
  (BytesReference) a[21],
- (PhaseExecutionInfo) a[12]
+ (PhaseExecutionInfo) a[12],
+ Objects.requireNonNullElse((Boolean) a[22], false)
  // a[13] == "age"
  // a[20] == "time_since_index_creation"
  )
@@ -118,6 +120,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
  builder.copyCurrentStructure(p);
  return BytesReference.bytes(builder);
  }, PREVIOUS_STEP_INFO_FIELD);
+ PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), SKIP_NAME);
  }
 
  private final String index;
@@ -140,6 +143,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
  private final String repositoryName;
  private final String snapshotName;
  private final String shrinkIndexName;
+ private final boolean skip;
 
  Supplier<Long> nowSupplier = System::currentTimeMillis; // Can be changed for testing
 
@@ -162,7 +166,8 @@ public static IndexLifecycleExplainResponse newManagedIndexResponse(
  String shrinkIndexName,
  BytesReference stepInfo,
  BytesReference previousStepInfo,
- PhaseExecutionInfo phaseExecutionInfo
+ PhaseExecutionInfo phaseExecutionInfo,
+ boolean skip
  ) {
  return new IndexLifecycleExplainResponse(
  index,
@@ -184,7 +189,8 @@ public static IndexLifecycleExplainResponse newManagedIndexResponse(
  shrinkIndexName,
  stepInfo,
  previousStepInfo,
- phaseExecutionInfo
+ phaseExecutionInfo,
+ skip
  );
  }
 
@@ -209,7 +215,8 @@ public static IndexLifecycleExplainResponse newUnmanagedIndexResponse(String ind
  null,
  null,
  null,
- null
+ null,
+ false
  );
  }
 
@@ -233,7 +240,8 @@ private IndexLifecycleExplainResponse(
  String shrinkIndexName,
  BytesReference stepInfo,
  BytesReference previousStepInfo,
- PhaseExecutionInfo phaseExecutionInfo
+ PhaseExecutionInfo phaseExecutionInfo,
+ boolean skip
  ) {
  if (managedByILM) {
  if (policyName == null) {
@@ -301,6 +309,7 @@ private IndexLifecycleExplainResponse(
  this.repositoryName = repositoryName;
  this.snapshotName = snapshotName;
  this.shrinkIndexName = shrinkIndexName;
+ this.skip = skip;
  }
 
  public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
@@ -333,6 +342,11 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
  } else {
  previousStepInfo = null;
  }
+ if (in.getTransportVersion().onOrAfter(TransportVersions.ILM_ADD_SKIP_SETTING_8_19)) {
+ skip = in.readBoolean();
+ } else {
+ skip = false;
+ }
  } else {
  policyName = null;
  lifecycleDate = null;
@@ -352,6 +366,7 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
  snapshotName = null;
  shrinkIndexName = null;
  indexCreationDate = null;
+ skip = false;
  }
  }
 
@@ -382,6 +397,9 @@ public void writeTo(StreamOutput out) throws IOException {
  if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
  out.writeOptionalBytesReference(previousStepInfo);
  }
+ if (out.getTransportVersion().onOrAfter(TransportVersions.ILM_ADD_SKIP_SETTING_8_19)) {
+ out.writeBoolean(skip);
+ }
  }
  }
 
@@ -481,6 +499,10 @@ public String getShrinkIndexName() {
  return shrinkIndexName;
  }
 
+ public boolean getSkip() {
+ return skip;
+ }
+
  @Override
  public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
  builder.startObject();
@@ -564,6 +586,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
  if (phaseExecutionInfo != null) {
  builder.field(PHASE_EXECUTION_INFO.getPreferredName(), phaseExecutionInfo);
  }
+ builder.field(SKIP_NAME.getPreferredName(), skip);
  }
  builder.endObject();
  return builder;
@@ -591,7 +614,8 @@ public int hashCode() {
  shrinkIndexName,
  stepInfo,
  previousStepInfo,
- phaseExecutionInfo
+ phaseExecutionInfo,
+ skip
  );
  }
 
@@ -623,7 +647,8 @@ public boolean equals(Object obj) {
  && Objects.equals(shrinkIndexName, other.shrinkIndexName)
  && Objects.equals(stepInfo, other.stepInfo)
  && Objects.equals(previousStepInfo, other.previousStepInfo)
- && Objects.equals(phaseExecutionInfo, other.phaseExecutionInfo);
+ && Objects.equals(phaseExecutionInfo, other.phaseExecutionInfo)
+ && Objects.equals(skip, other.skip);
  }
 
  @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java

@@ -23,6 +23,7 @@ public class LifecycleSettings {
  public static final String LIFECYCLE_STEP_MASTER_TIMEOUT = "indices.lifecycle.step.master_timeout";
  public static final String LIFECYCLE_STEP_WAIT_TIME_THRESHOLD = "index.lifecycle.step.wait_time_threshold";
  public static final String LIFECYCLE_ROLLOVER_ONLY_IF_HAS_DOCUMENTS = "indices.lifecycle.rollover.only_if_has_documents";
+ public static final String LIFECYCLE_SKIP = "index.lifecycle.skip";
 
  public static final String SLM_HISTORY_INDEX_ENABLED = "slm.history_index_enabled";
  public static final String SLM_RETENTION_SCHEDULE = "slm.retention_schedule";
@@ -82,6 +83,12 @@ public class LifecycleSettings {
  Setting.Property.NodeScope,
  Setting.Property.DeprecatedWarning
  );
+ public static final Setting<Boolean> LIFECYCLE_SKIP_SETTING = Setting.boolSetting(
+ LIFECYCLE_SKIP,
+ false,
+ Setting.Property.Dynamic,
+ Setting.Property.IndexScope
+ );
 
  public static final Setting<Boolean> SLM_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(
  SLM_HISTORY_INDEX_ENABLED,

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java

@@ -74,7 +74,8 @@ private static IndexLifecycleExplainResponse randomManagedIndexExplainResponse()
  stepNull ? null : randomAlphaOfLength(10),
  randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
  randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
- randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo("")
+ randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo(""),
+ randomBoolean()
  );
  }
 
@@ -101,7 +102,8 @@ public void testInvalidStepDetails() {
  randomBoolean() ? null : randomAlphaOfLength(10),
  randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
  randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
- randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo("")
+ randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo(""),
+ randomBoolean()
  )
  );
  assertThat(exception.getMessage(), startsWith("managed index response must have complete step details"));
@@ -135,7 +137,8 @@ public void testIndexAges() {
  null,
  null,
  null,
- null
+ null,
+ false
  );
  assertThat(managedExplainResponse.getLifecycleDate(), is(notNullValue()));
  Long now = 1_000_000L;
@@ -196,9 +199,10 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp
  BytesReference stepInfo = instance.getStepInfo();
  BytesReference previousStepInfo = instance.getPreviousStepInfo();
  PhaseExecutionInfo phaseExecutionInfo = instance.getPhaseExecutionInfo();
+ boolean skip = instance.getSkip();
 
  if (managed) {
- switch (between(0, 15)) {
+ switch (between(0, 16)) {
  case 0 -> index += randomAlphaOfLengthBetween(1, 5);
  case 1 -> policy += randomAlphaOfLengthBetween(1, 5);
  case 2 -> {
@@ -257,6 +261,7 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp
  case 13 -> repositoryName = randomValueOtherThan(repositoryName, () -> randomAlphaOfLengthBetween(5, 10));
  case 14 -> snapshotName = randomValueOtherThan(snapshotName, () -> randomAlphaOfLengthBetween(5, 10));
  case 15 -> shrinkIndexName = randomValueOtherThan(shrinkIndexName, () -> randomAlphaOfLengthBetween(5, 10));
+ case 16 -> skip = skip == false;
  default -> throw new AssertionError("Illegal randomisation branch");
  }
 
@@ -279,7 +284,8 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp
  shrinkIndexName,
  stepInfo,
  previousStepInfo,
- phaseExecutionInfo
+ phaseExecutionInfo,
+ skip
  );
  } else {
  return switch (between(0, 1)) {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

@@ -128,6 +128,7 @@ public List<Setting<?>> getSettings() {
  LifecycleSettings.LIFECYCLE_STEP_MASTER_TIMEOUT_SETTING,
  LifecycleSettings.LIFECYCLE_STEP_WAIT_TIME_THRESHOLD_SETTING,
  LifecycleSettings.LIFECYCLE_ROLLOVER_ONLY_IF_HAS_DOCUMENTS_SETTING,
+ LifecycleSettings.LIFECYCLE_SKIP_SETTING,
  RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING,
  IlmHealthIndicatorService.MAX_TIME_ON_ACTION_SETTING,
  IlmHealthIndicatorService.MAX_TIME_ON_STEP_SETTING,

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.xpack.core.ilm.ClusterStateActionStep;
 import org.elasticsearch.xpack.core.ilm.ClusterStateWaitStep;
 import org.elasticsearch.xpack.core.ilm.ErrorStep;
+import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep;
 import org.elasticsearch.xpack.core.ilm.Step;
 import org.elasticsearch.xpack.core.ilm.Step.StepKey;
@@ -176,6 +177,10 @@ boolean isReadyToTransitionToThisPhase(final String policy, final IndexMetadata
  */
  void runPeriodicStep(String policy, Metadata metadata, IndexMetadata indexMetadata) {
  String index = indexMetadata.getIndex().getName();
+ if (LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(indexMetadata.getSettings())) {
+ logger.info("[{}] skipping policy [{}] because [{}] is true", index, policy, LifecycleSettings.LIFECYCLE_SKIP);
+ return;
+ }
  LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
  final Step currentStep;
  try {
@@ -303,6 +308,10 @@ void onErrorMaybeRetryFailedStep(String policy, StepKey currentStep, IndexMetada
  */
  void maybeRunAsyncAction(ClusterState currentState, IndexMetadata indexMetadata, String policy, StepKey expectedStepKey) {
  String index = indexMetadata.getIndex().getName();
+ if (LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(indexMetadata.getSettings())) {
+ logger.info("[{}] skipping policy [{}] because [{}] is true", index, policy, LifecycleSettings.LIFECYCLE_SKIP);
+ return;
+ }
  LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
  final Step currentStep;
  try {
@@ -383,6 +392,10 @@ public void onFailure(Exception e) {
  */
  void runPolicyAfterStateChange(String policy, IndexMetadata indexMetadata) {
  String index = indexMetadata.getIndex().getName();
+ if (LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(indexMetadata.getSettings())) {
+ logger.info("[{}] skipping policy [{}] because [{}] is true", index, policy, LifecycleSettings.LIFECYCLE_SKIP);
+ return;
+ }
  LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
  final StepKey currentStepKey = Step.getCurrentStepKey(lifecycleState);
  if (busyIndices.contains(Tuple.tuple(indexMetadata.getIndex(), currentStepKey))) {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java

@@ -477,6 +477,7 @@ private static IndexMetadata.Builder removePolicyForIndex(IndexMetadata indexMet
 
  notChanged &= Strings.isNullOrEmpty(newSettings.remove(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey()));
  notChanged &= Strings.isNullOrEmpty(newSettings.remove(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.getKey()));
+ notChanged &= Strings.isNullOrEmpty(newSettings.remove(LifecycleSettings.LIFECYCLE_SKIP_SETTING.getKey()));
  notChanged &= Strings.isNullOrEmpty(newSettings.remove(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.getKey()));
  long newSettingsVersion = notChanged ? indexMetadata.getSettingsVersion() : 1 + indexMetadata.getSettingsVersion();
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java

@@ -188,7 +188,8 @@ static IndexLifecycleExplainResponse getIndexLifecycleExplainResponse(
  lifecycleState.shrinkIndexName(),
  stepInfoBytes,
  previousStepInfoBytes,
- phaseExecutionInfo
+ phaseExecutionInfo,
+ LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(idxSettings)
  );
  } else {
  indexResponse = null;