[ML] Automatically rollover legacy .ml-anomalies indices

docs/changelog/120913.yaml

@@ -0,0 +1,5 @@
+pr: 120913
+summary: Automatically rollover legacy .ml-anomalies indices
+area: Machine Learning
+type: upgrade
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java

@@ -38,6 +38,10 @@ public static String jobResultsIndexPrefix() {
  return AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX;
  }
 
+ public static String jobResultsIndexPattern() {
+ return AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*";
+ }
+
  /**
  * The name of the alias pointing to the indices where the job's results are stored
  * @param jobId Job Id
@@ -47,15 +51,26 @@ public static String jobResultsAliasedName(String jobId) {
  return AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + jobId;
  }
 
+ /**
+ * Extract the job Id from the alias name.
+ * If not an results index alias null is returned
+ * @param jobResultsAliasedName The alias
+ * @return The job Id
+ */
+ public static String jobIdFromAlias(String jobResultsAliasedName) {
+ if (jobResultsAliasedName.length() < AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()) {
+ return null;
+ }
+ return jobResultsAliasedName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length());
+ }
+
  /**
  * The name of the alias pointing to the write index for a job
  * @param jobId Job Id
  * @return The write alias
  */
  public static String resultsWriteAlias(String jobId) {
- // ".write" rather than simply "write" to avoid the danger of clashing
- // with the read alias of a job whose name begins with "write-"
- return AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + ".write-" + jobId;
+ return AnomalyDetectorsIndexFields.RESULTS_INDEX_WRITE_PREFIX + jobId;
  }
 
  /**

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java

@@ -11,6 +11,9 @@ public final class AnomalyDetectorsIndexFields {
  public static final String STATE_INDEX_PREFIX = ".ml-state";
 
  public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-";
+ // ".write" rather than simply "write" to avoid the danger of clashing
+ // with the read alias of a job whose name begins with "write-"
+ public static final String RESULTS_INDEX_WRITE_PREFIX = RESULTS_INDEX_PREFIX + ".write-";
  public static final String RESULTS_INDEX_DEFAULT = "shared";
 
  private AnomalyDetectorsIndexFields() {}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java

@@ -31,6 +31,8 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -64,27 +66,24 @@ public final class MlIndexAndAlias {
  */
  public static final String BWC_MAPPINGS_VERSION = "8.11.0";
 
- private static final Logger logger = LogManager.getLogger(MlIndexAndAlias.class);
+ public static final String FIRST_INDEX_SIX_DIGIT_SUFFIX = "-000001";
 
- static final Comparator<String> INDEX_NAME_COMPARATOR = new Comparator<>() {
-
- private final Predicate<String> HAS_SIX_DIGIT_SUFFIX = Pattern.compile("\\d{6}").asMatchPredicate();
-
- @Override
- public int compare(String index1, String index2) {
- String[] index1Parts = index1.split("-");
- String index1Suffix = index1Parts[index1Parts.length - 1];
- boolean index1HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index1Suffix);
- String[] index2Parts = index2.split("-");
- String index2Suffix = index2Parts[index2Parts.length - 1];
- boolean index2HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index2Suffix);
- if (index1HasSixDigitsSuffix && index2HasSixDigitsSuffix) {
- return index1Suffix.compareTo(index2Suffix);
- } else if (index1HasSixDigitsSuffix != index2HasSixDigitsSuffix) {
- return Boolean.compare(index1HasSixDigitsSuffix, index2HasSixDigitsSuffix);
- } else {
- return index1.compareTo(index2);
- }
+ private static final Logger logger = LogManager.getLogger(MlIndexAndAlias.class);
+ private static final Predicate<String> HAS_SIX_DIGIT_SUFFIX = Pattern.compile("\\d{6}").asMatchPredicate();
+
+ static final Comparator<String> INDEX_NAME_COMPARATOR = (index1, index2) -> {
+ String[] index1Parts = index1.split("-");
+ String index1Suffix = index1Parts[index1Parts.length - 1];
+ boolean index1HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index1Suffix);
+ String[] index2Parts = index2.split("-");
+ String index2Suffix = index2Parts[index2Parts.length - 1];
+ boolean index2HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index2Suffix);
+ if (index1HasSixDigitsSuffix && index2HasSixDigitsSuffix) {
+ return index1Suffix.compareTo(index2Suffix);
+ } else if (index1HasSixDigitsSuffix != index2HasSixDigitsSuffix) {
+ return Boolean.compare(index1HasSixDigitsSuffix, index2HasSixDigitsSuffix);
+ } else {
+ return index1.compareTo(index2);
  }
  };
 
@@ -126,7 +125,7 @@ public static void createIndexAndAliasIfNecessary(
  String legacyIndexWithoutSuffix = indexPatternPrefix;
  String indexPattern = indexPatternPrefix + "*";
  // The initial index name must be suitable for rollover functionality.
- String firstConcreteIndex = indexPatternPrefix + "-000001";
+ String firstConcreteIndex = indexPatternPrefix + FIRST_INDEX_SIX_DIGIT_SUFFIX;
  String[] concreteIndexNames = resolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandHidden(), indexPattern);
  Optional<String> indexPointedByCurrentWriteAlias = clusterState.getMetadata().hasAlias(alias)
  ? clusterState.getMetadata().getIndicesLookup().get(alias).getIndices().stream().map(Index::getName).findFirst()
@@ -384,6 +383,10 @@ public static boolean hasIndexTemplate(ClusterState state, String templateName)
  return state.getMetadata().templatesV2().containsKey(templateName);
  }
 
+ public static boolean has6DigitSuffix(String indexName) {
+ return HAS_SIX_DIGIT_SUFFIX.test(indexName);
+ }
+
  /**
  * Returns the latest index. Latest is the index with the highest
  * 6 digit suffix.
@@ -395,4 +398,11 @@ public static String latestIndex(String[] concreteIndices) {
  ? concreteIndices[0]
  : Arrays.stream(concreteIndices).max(MlIndexAndAlias.INDEX_NAME_COMPARATOR).get();
  }
+
+ /**
+ * True if the version is read *and* write compatible not just read only compatible
+ */
+ public static boolean indexIsReadWriteCompatibleInV9(IndexVersion version) {
+ return version.onOrAfter(IndexVersions.V_8_0_0);
+ }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.indices.TestIndexNameExpressionResolver;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -365,8 +366,20 @@ public void testIndexNameComparator() {
  }
 
  public void testLatestIndex() {
- var names = new String[] { "index-000001", "index-000002", "index-000003" };
- assertThat(MlIndexAndAlias.latestIndex(names), equalTo("index-000003"));
+ {
+ var names = new String[] { "index-000001", "index-000002", "index-000003" };
+ assertThat(MlIndexAndAlias.latestIndex(names), equalTo("index-000003"));
+ }
+ {
+ var names = new String[] { "index", "index-000001", "index-000002" };
+ assertThat(MlIndexAndAlias.latestIndex(names), equalTo("index-000002"));
+ }
+ }
+
+ public void testIndexIsReadWriteCompatibleInV9() {
+ assertTrue(MlIndexAndAlias.indexIsReadWriteCompatibleInV9(IndexVersion.current()));
+ assertTrue(MlIndexAndAlias.indexIsReadWriteCompatibleInV9(IndexVersions.V_8_0_0));
+ assertFalse(MlIndexAndAlias.indexIsReadWriteCompatibleInV9(IndexVersions.V_7_17_0));
  }
 
  private void createIndexAndAliasIfNecessary(ClusterState clusterState) {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

@@ -1240,7 +1240,8 @@ public Collection<?> createComponents(PluginServices services) {
  ),
  indexNameExpressionResolver,
  client
- )
+ ),
+ new MlAnomaliesIndexUpdate(indexNameExpressionResolver, client)
  )
  );
  clusterService.addListener(mlAutoUpdateService);