Add selector syntax to index expressions

docs/reference/ml/anomaly-detection/apis/put-job.asciidoc

@@ -557,4 +557,3 @@ The API returns the following results:
 // TESTRESPONSE[s/"job_version" : "8.4.0"/"job_version" : $body.job_version/]
 // TESTRESPONSE[s/1656087283340/$body.$_path/]
 // TESTRESPONSE[s/"superuser"/"_es_test_root"/]
-// TESTRESPONSE[s/"ignore_throttled" : true/"ignore_throttled" : true,"failure_store":"exclude"/]

modules/data-streams/build.gradle

@@ -20,6 +20,7 @@ restResources {
 
 dependencies {
  testImplementation project(path: ':test:test-clusters')
+ testImplementation project(":modules:mapper-extras")
  internalClusterTestImplementation project(":modules:mapper-extras")
 }
 
@@ -70,4 +71,16 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
  task.skipTest("data_stream/200_rollover_failure_store/Lazily roll over a data stream's failure store after a shard failure", "Configuring the failure store via data stream templates is not supported anymore.")
  task.skipTest("data_stream/200_rollover_failure_store/Don't roll over a data stream's failure store when conditions aren't met", "Configuring the failure store via data stream templates is not supported anymore.")
  task.skipTest("data_stream/200_rollover_failure_store/Roll over a data stream's failure store with conditions", "Configuring the failure store via data stream templates is not supported anymore.")
+
+ task.skipTest("data_stream/200_rollover_failure_store/Rolling over a failure store on a data stream without the failure store enabled should work", "Rolling over a data stream using target_failure_store is no longer supported.")
+ task.skipTest("data_stream/200_rollover_failure_store/Rolling over an uninitialized failure store should initialize it", "Rolling over a data stream using target_failure_store is no longer supported.")
+
+ task.skipTest("data_stream/210_rollover_failure_store/A failure store marked for lazy rollover should only be rolled over when there is a failure", "Rolling over a data stream using target_failure_store is no longer supported.")
+ task.skipTest("data_stream/210_rollover_failure_store/Don't roll over a data stream's failure store when conditions aren't met", "Rolling over a data stream using target_failure_store is no longer supported.")
+ task.skipTest("data_stream/210_rollover_failure_store/Rolling over a failure store on a data stream without the failure store enabled should work", "Rolling over a data stream using target_failure_store is no longer supported.")
+ task.skipTest("data_stream/210_rollover_failure_store/Rolling over an uninitialized failure store should initialize it", "Rolling over a data stream using target_failure_store is no longer supported.")
+ task.skipTest("data_stream/210_rollover_failure_store/Roll over a data stream's failure store with conditions", "Rolling over a data stream using target_failure_store is no longer supported.")
+ task.skipTest("data_stream/210_rollover_failure_store/Lazily roll over a data stream's failure store after an ingest failure", "Rolling over a data stream using target_failure_store is no longer supported.")
+ task.skipTest("data_stream/210_rollover_failure_store/Lazily roll over a data stream's failure store after a shard failure", "Rolling over a data stream using target_failure_store is no longer supported.")
+ task.skipTest("data_stream/210_rollover_failure_store/Roll over a data stream's failure store without conditions", "Rolling over a data stream using target_failure_store is no longer supported.")
 })

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java

@@ -31,11 +31,13 @@
 import org.elasticsearch.action.datastreams.CreateDataStreamAction;
 import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamAlias;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.index.Index;
@@ -136,10 +138,7 @@ public void setup() throws Exception {
  assertTrue(response.isAcknowledged());
 
  // Initialize the failure store.
- RolloverRequest rolloverRequest = new RolloverRequest("with-fs", null);
- rolloverRequest.setIndicesOptions(
- IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
- );
+ RolloverRequest rolloverRequest = new RolloverRequest("with-fs::failures", null);
  response = client.execute(RolloverAction.INSTANCE, rolloverRequest).get();
  assertTrue(response.isAcknowledged());
 
@@ -345,7 +344,7 @@ public void testFailureStoreSnapshotAndRestore() throws Exception {
  .cluster()
  .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
  .setWaitForCompletion(true)
- .setIndices(dataStreamName)
+ .setIndices(IndexNameExpressionResolver.combineSelector(dataStreamName, IndexComponentSelector.ALL_APPLICABLE))
  .setIncludeGlobalState(false)
  .get();
 

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java

@@ -20,11 +20,12 @@
 import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.action.datastreams.CreateDataStreamAction;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.core.Strings;
@@ -194,9 +195,9 @@ public void testRejectionFromFailureStore() throws IOException {
  createDataStream();
 
  // Initialize failure store.
- var rolloverRequest = new RolloverRequest(dataStream, null);
- rolloverRequest.setIndicesOptions(
- IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
+ var rolloverRequest = new RolloverRequest(
+  IndexNameExpressionResolver.combineSelector(dataStream, IndexComponentSelector.FAILURES),
+ null
  );
  var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet();
  var failureStoreIndex = rolloverResponse.getNewIndex();

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java

@@ -60,7 +60,7 @@ public void setup() throws IOException {
 
  assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME)));
  // Initialize the failure store.
- assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store")));
+ assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "::failures/_rollover")));
  ensureGreen(DATA_STREAM_NAME);
 
  final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME));

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.action.datastreams.DataStreamsStatsAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -102,10 +103,11 @@ protected ClusterBlockException checkRequestBlock(
 
  @Override
  protected String[] resolveConcreteIndexNames(ClusterState clusterState, DataStreamsStatsAction.Request request) {
- return DataStreamsActionUtil.resolveConcreteIndexNames(
+ return DataStreamsActionUtil.resolveConcreteIndexNamesWithSelector(
  indexNameExpressionResolver,
  clusterState,
  request.indices(),
+ IndexComponentSelector.ALL_APPLICABLE,
  request.indicesOptions()
  ).toArray(String[]::new);
  }
@@ -163,13 +165,17 @@ protected DataStreamsStatsAction.DataStreamShardStats readShardResult(StreamInpu
  request.indicesOptions(),
  request.indices()
  );
- for (String abstractionName : abstractionNames) {
- IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName);
+ for (String abstraction : abstractionNames) {
+ IndexAbstraction indexAbstraction = indicesLookup.get(abstraction);
  assert indexAbstraction != null;
  if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
  DataStream dataStream = (DataStream) indexAbstraction;
  AggregatedStats stats = aggregatedDataStreamsStats.computeIfAbsent(dataStream.getName(), s -> new AggregatedStats());
- dataStream.getIndices().stream().map(Index::getName).forEach(index -> {
+ dataStream.getBackingIndices().getIndices().stream().map(Index::getName).forEach(index -> {
+ stats.backingIndices.add(index);
+ allBackingIndices.add(index);
+ });
+ dataStream.getFailureIndices().getIndices().stream().map(Index::getName).forEach(index -> {
  stats.backingIndices.add(index);
  allBackingIndices.add(index);
  });