Downsampling: copyindex.hidden setting from source (#89177)

Rollup indices are initially created as hidden (index.hidden: true). At the end of the rollup process, we must set this setting to the value the source index has

Author: csoulios

x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/TransportRollupAction.java

@@ -267,6 +267,14 @@ protected void masterOperation(
  if (sourceIndexMetadata.getNumberOfReplicas() > 0) {
  settings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, sourceIndexMetadata.getNumberOfReplicas());
  }
+ // Setting index.hidden has been initially set to true. We revert this to the value of the source index
+ if (sourceIndexMetadata.isHidden() == false) {
+ if (sourceIndexMetadata.getSettings().keySet().contains(IndexMetadata.SETTING_INDEX_HIDDEN)) {
+ settings.put(IndexMetadata.SETTING_INDEX_HIDDEN, false);
+ } else {
+ settings.putNull(IndexMetadata.SETTING_INDEX_HIDDEN);
+ }
+ }
  UpdateSettingsRequest updateSettingsReq = new UpdateSettingsRequest(settings.build(), rollupIndexName);
  updateSettingsReq.setParentTask(parentTask);
  client.admin().indices().updateSettings(updateSettingsReq, ActionListener.wrap(updateSettingsResponse -> {
@@ -545,7 +553,7 @@ private void createRollupIndex(
  * case rollup will fail.
  */
  Settings.builder()
- .put(IndexMetadata.INDEX_HIDDEN_SETTING.getKey(), true)
+ .put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
  .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, sourceIndexMetadata.getNumberOfShards())
  .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
  .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")

x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/RollupActionSingleNodeTests.java

@@ -165,23 +165,25 @@ public void setup() {
  * check that the value of the label (last value) matches the value
  * of the corresponding metric which uses a last_value metric type.
  */
+ Settings.Builder settings = Settings.builder()
+ .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
+ .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
+ .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
+ .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1))
+ .put(
+ IndexSettings.TIME_SERIES_START_TIME.getKey(),
+ DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(Instant.ofEpochMilli(startTime).toEpochMilli())
+ )
+ .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z");
+
+ if (randomBoolean()) {
+ settings.put(IndexMetadata.SETTING_INDEX_HIDDEN, randomBoolean());
+ }
  assertAcked(
  client().admin()
  .indices()
  .prepareCreate(sourceIndex)
- .setSettings(
- Settings.builder()
- .put("index.number_of_shards", numOfShards)
- .put("index.number_of_replicas", numOfReplicas)
- .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
- .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1))
- .put(
- IndexSettings.TIME_SERIES_START_TIME.getKey(),
- DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(Instant.ofEpochMilli(startTime).toEpochMilli())
- )
- .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z")
- .build()
- )
+ .setSettings(settings.build())
  .setMapping(
  FIELD_TIMESTAMP,
  "type=date",
@@ -769,7 +771,10 @@ private void assertRollupIndexSettings(String sourceIndex, String rollupIndex, G
  );
 
  assertEquals(sourceIndex, indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_ROLLUP_SOURCE_NAME_KEY));
- assertEquals(indexSettingsResp.getSetting(sourceIndex, "index.mode"), indexSettingsResp.getSetting(rollupIndex, "index.mode"));
+ assertEquals(
+ indexSettingsResp.getSetting(sourceIndex, IndexSettings.MODE.getKey()),
+ indexSettingsResp.getSetting(rollupIndex, IndexSettings.MODE.getKey())
+ );
 
  assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()));
  assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()));
@@ -784,11 +789,11 @@ private void assertRollupIndexSettings(String sourceIndex, String rollupIndex, G
  indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_END_TIME.getKey()),
  indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_END_TIME.getKey())
  );
- assertNotNull(indexSettingsResp.getSetting(sourceIndex, "index.routing_path"));
- assertNotNull(indexSettingsResp.getSetting(rollupIndex, "index.routing_path"));
+ assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()));
+ assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()));
  assertEquals(
- indexSettingsResp.getSetting(sourceIndex, "index.routing_path"),
- indexSettingsResp.getSetting(rollupIndex, "index.routing_path")
+ indexSettingsResp.getSetting(sourceIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()),
+ indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey())
  );
 
  assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS));
@@ -804,7 +809,11 @@ private void assertRollupIndexSettings(String sourceIndex, String rollupIndex, G
  indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS),
  indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)
  );
- assertEquals("true", indexSettingsResp.getSetting(rollupIndex, "index.blocks.write"));
+ assertEquals("true", indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_BLOCKS_WRITE));
+ assertEquals(
+ indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_HIDDEN),
+ indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_INDEX_HIDDEN)
+ );
  }
 
  private AggregationBuilder buildAggregations(
@@ -869,9 +878,9 @@ private String createDataStream() throws Exception {
  String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault());
  Template indexTemplate = new Template(
  Settings.builder()
- .put("index.number_of_shards", numOfShards)
- .put("index.number_of_replicas", numOfReplicas)
- .put("index.mode", "time_series")
+ .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
+ .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
+ .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
  .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1))
  .build(),
  new CompressedXContent("""