Fix issues with ignore_missing_component_templates (#95527)

Author: eyalkoren

docs/changelog/95527.yaml

@@ -0,0 +1,5 @@
+pr: 95527
+summary: Allow deletion of component templates that are specified in the `ignore_missing_component_templates` array
+area: Data streams
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -198,6 +198,23 @@ public List<String> composedOf() {
  return componentTemplates;
  }
 
+ /**
+ * Returns the <b>required</b> component templates, i.e. such that are not allowed to be missing, as in
+ * {@link #ignoreMissingComponentTemplates}.
+ * @return a list of required component templates
+ */
+ public List<String> getRequiredComponentTemplates() {
+ if (componentTemplates == null) {
+ return List.of();
+ }
+ if (ignoreMissingComponentTemplates == null) {
+ return componentTemplates;
+ }
+ return componentTemplates.stream()
+ .filter(componentTemplate -> ignoreMissingComponentTemplates.contains(componentTemplate) == false)
+ .toList();
+ }
+
  @Nullable
  public Long priority() {
  return priority;

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -383,7 +383,7 @@ public void removeComponentTemplate(
  ClusterState state,
  final ActionListener<AcknowledgedResponse> listener
  ) {
- validateNotInUse(state.metadata(), names);
+ validateCanBeRemoved(state.metadata(), names);
  taskQueue.submitTask("remove-component-template [" + String.join(",", names) + "]", new TemplateClusterStateUpdateTask(listener) {
  @Override
  public ClusterState execute(ClusterState currentState) {
@@ -394,7 +394,7 @@ public ClusterState execute(ClusterState currentState) {
 
  // Exposed for ReservedComponentTemplateAction
  public static ClusterState innerRemoveComponentTemplate(ClusterState currentState, String... names) {
- validateNotInUse(currentState.metadata(), names);
+ validateCanBeRemoved(currentState.metadata(), names);
 
  final Set<String> templateNames = new HashSet<>();
  if (names.length > 1) {
@@ -439,10 +439,11 @@ public static ClusterState innerRemoveComponentTemplate(ClusterState currentStat
  }
 
  /**
- * Validates that the given component template is not used by any index
- * templates, throwing an error if it is still in use
+ * Validates that the given component template can be removed, throwing an error if it cannot.
+ * A component template should not be removed if it is <b>required</b> by any index templates,
+ * that is- it is used AND NOT specified as {@code ignore_missing_component_templates}.
  */
- static void validateNotInUse(Metadata metadata, String... templateNameOrWildcard) {
+ static void validateCanBeRemoved(Metadata metadata, String... templateNameOrWildcard) {
  final Predicate<String> predicate;
  if (templateNameOrWildcard.length > 1) {
  predicate = name -> Arrays.asList(templateNameOrWildcard).contains(name);
@@ -456,7 +457,10 @@ static void validateNotInUse(Metadata metadata, String... templateNameOrWildcard
  .collect(Collectors.toSet());
  final Set<String> componentsBeingUsed = new HashSet<>();
  final List<String> templatesStillUsing = metadata.templatesV2().entrySet().stream().filter(e -> {
- Set<String> intersecting = Sets.intersection(new HashSet<>(e.getValue().composedOf()), matchingComponentTemplates);
+ Set<String> intersecting = Sets.intersection(
+ new HashSet<>(e.getValue().getRequiredComponentTemplates()),
+ matchingComponentTemplates
+ );
  if (intersecting.size() > 0) {
  componentsBeingUsed.addAll(intersecting);
  return true;

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -1666,6 +1666,40 @@ public void testRemoveComponentTemplateInUse() throws Exception {
  );
  }
 
+ public void testRemoveRequiredAndNonRequiredComponents() throws Exception {
+ ComposableIndexTemplate composableIndexTemplate = new ComposableIndexTemplate(
+ Collections.singletonList("pattern"),
+ null,
+ List.of("required1", "non-required", "required2"),
+ null,
+ null,
+ null,
+ null,
+ null,
+ Collections.singletonList("non-required")
+ );
+ ComponentTemplate ct = new ComponentTemplate(new Template(null, new CompressedXContent("{}"), null), null, null);
+
+ final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
+ ClusterState clusterState = service.addComponentTemplate(ClusterState.EMPTY_STATE, false, "required1", ct);
+ clusterState = service.addComponentTemplate(clusterState, false, "required2", ct);
+ clusterState = service.addComponentTemplate(clusterState, false, "non-required", ct);
+ clusterState = service.addIndexTemplateV2(clusterState, false, "composable-index-template", composableIndexTemplate);
+
+ final ClusterState cs = clusterState;
+ Exception e = expectThrows(IllegalArgumentException.class, () -> innerRemoveComponentTemplate(cs, "required*"));
+ assertThat(
+ e.getMessage(),
+ containsString(
+ "component templates [required1, required2] cannot be removed as they are still in use by index templates "
+ + "[composable-index-template]"
+ )
+ );
+
+ // This removal should succeed
+ innerRemoveComponentTemplate(cs, "non-required*");
+ }
+
  /**
  * Tests that we check that settings/mappings/etc are valid even after template composition,
  * when adding/updating a composable index template

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java

@@ -327,7 +327,7 @@ private void addComposableTemplatesIfMissing(ClusterState state) {
  * Returns true if the cluster state contains all of the component templates needed by the composable template
  */
  private static boolean componentTemplatesExist(ClusterState state, ComposableIndexTemplate indexTemplate) {
- return state.metadata().componentTemplates().keySet().containsAll(indexTemplate.composedOf());
+ return state.metadata().componentTemplates().keySet().containsAll(indexTemplate.getRequiredComponentTemplates());
  }
 
  private void putLegacyTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) {

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackRegistryWithNonRequiredTemplates.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.stack;
+
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
+
+import java.util.Map;
+
+class StackRegistryWithNonRequiredTemplates extends StackTemplateRegistry {
+ StackRegistryWithNonRequiredTemplates(
+ Settings nodeSettings,
+ ClusterService clusterService,
+ ThreadPool threadPool,
+ Client client,
+ NamedXContentRegistry xContentRegistry
+ ) {
+ super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
+ }
+
+ @Override
+ protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
+ return parseComposableTemplates(
+ new IndexTemplateConfig("syslog", "/non-required-template.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE)
+ );
+ }
+}

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -270,6 +271,53 @@ public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception {
  assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size())));
  }
 
+ public void testMissingNonRequiredTemplates() throws Exception {
+ StackRegistryWithNonRequiredTemplates registryWithNonRequiredTemplate = new StackRegistryWithNonRequiredTemplates(
+ Settings.EMPTY,
+ clusterService,
+ threadPool,
+ client,
+ NamedXContentRegistry.EMPTY
+ );
+
+ DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
+ DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
+
+ ClusterChangedEvent event = createClusterChangedEvent(
+ Collections.singletonMap(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, null),
+ nodes
+ );
+
+ final AtomicInteger calledTimes = new AtomicInteger(0);
+ client.setVerifier((action, request, listener) -> {
+ if (action instanceof PutComponentTemplateAction) {
+ // Ignore such
+ return AcknowledgedResponse.TRUE;
+ } else if (action instanceof PutLifecycleAction) {
+ // Ignore such
+ return AcknowledgedResponse.TRUE;
+ } else if (action instanceof PutComposableIndexTemplateAction) {
+ calledTimes.incrementAndGet();
+ assertThat(request, instanceOf(PutComposableIndexTemplateAction.Request.class));
+ PutComposableIndexTemplateAction.Request putComposableTemplateRequest = (PutComposableIndexTemplateAction.Request) request;
+ assertThat(putComposableTemplateRequest.name(), equalTo("syslog"));
+ ComposableIndexTemplate composableIndexTemplate = putComposableTemplateRequest.indexTemplate();
+ assertThat(composableIndexTemplate.composedOf(), hasSize(2));
+ assertThat(composableIndexTemplate.composedOf().get(0), equalTo("logs-settings"));
+ assertThat(composableIndexTemplate.composedOf().get(1), equalTo("syslog@custom"));
+ assertThat(composableIndexTemplate.getIgnoreMissingComponentTemplates(), hasSize(1));
+ assertThat(composableIndexTemplate.getIgnoreMissingComponentTemplates().get(0), equalTo("syslog@custom"));
+ return AcknowledgedResponse.TRUE;
+ } else {
+ fail("client called with unexpected request:" + request.toString());
+ return null;
+ }
+ });
+
+ registryWithNonRequiredTemplate.clusterChanged(event);
+ assertBusy(() -> assertThat(calledTimes.get(), equalTo(1)));
+ }
+
  @TestLogging(value = "org.elasticsearch.xpack.core.template:DEBUG", reason = "test")
  public void testSameOrHigherVersionTemplateNotUpgraded() {
  DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);

x-pack/plugin/stack/src/test/resources/non-required-template.json

@@ -0,0 +1,16 @@
+{
+ "index_patterns": ["syslog-*-*"],
+ "priority": 100,
+ "data_stream": {},
+ "composed_of": [
+ "logs-settings",
+ "syslog@custom"
+ ],
+ "ignore_missing_component_templates": ["syslog@custom"],
+ "allow_auto_create": true,
+ "_meta": {
+ "description": "default logs template installed by x-pack",
+ "managed": true
+ },
+ "version": ${xpack.stack.template.version}
+}