Merge mappings for composable index templates

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml

@@ -200,3 +200,89 @@
  index: eggplant
 
  - match: {eggplant.settings.index.number_of_shards: "3"}
+
+---
+"Index template mapping merging":
+ - skip:
+ version: " - 7.99.99"
+ reason: "index template v2 mapping merging not yet backported to 7.9"
+ features: allowed_warnings
+
+ - do:
+ cluster.put_component_template:
+ name: red
+ body:
+ template:
+ mappings:
+ properties:
+ object1.red:
+ type: keyword
+ object2.red:
+ type: keyword
+
+ - do:
+ cluster.put_component_template:
+ name: blue
+ body:
+ template:
+ mappings:
+ properties:
+ object2.red:
+ type: text
+ object1.blue:
+ type: text
+ object2.blue:
+ type: text
+
+ - do:
+ allowed_warnings:
+ - "index template [my-template] has index patterns [baz*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
+ indices.put_index_template:
+ name: blue
+ body:
+ index_patterns: ["purple-index"]
+ composed_of: ["red", "blue"]
+ template:
+ mappings:
+ properties:
+ object2.blue:
+ type: integer
+ object1.purple:
+ type: integer
+ object2.purple:
+ type: integer
+ nested:
+ type: nested
+ include_in_root: true
+
+ - do:
+ indices.create:
+ index: purple-index
+ body:
+ mappings:
+ properties:
+ object2.purple:
+ type: double
+ object3.purple:
+ type: double
+ nested:
+ type: nested
+ include_in_root: false
+ include_in_parent: true
+
+ - do:
+ indices.get:
+ index: purple-index
+
+ - match: {purple-index.mappings.properties.object1.properties.red: {type: keyword}}
+ - match: {purple-index.mappings.properties.object1.properties.blue: {type: text}}
+ - match: {purple-index.mappings.properties.object1.properties.purple: {type: integer}}
+
+ - match: {purple-index.mappings.properties.object2.properties.red: {type: text}}
+ - match: {purple-index.mappings.properties.object2.properties.blue: {type: integer}}
+ - match: {purple-index.mappings.properties.object2.properties.purple: {type: double}}
+
+ - match: {purple-index.mappings.properties.object3.properties.purple: {type: double}}
+
+ - is_false: purple-index.mappings.properties.nested.include_in_root
+ - is_true: purple-index.mappings.properties.nested.include_in_parent

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -28,22 +28,22 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.AliasValidator;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -58,7 +58,6 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.resolveV2Mappings;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template;
@@ -172,10 +171,6 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri
  final AliasValidator aliasValidator) throws Exception {
  Settings settings = resolveSettings(simulatedState.metadata(), matchingTemplate);
 
- // empty request mapping as the user can't specify any explicit mappings via the simulate api
- Map<String, Object> mappings = resolveV2Mappings("{}", simulatedState, matchingTemplate, xContentRegistry);
- String mappingsJson = Strings.toString(XContentFactory.jsonBuilder().map(mappings));
-
  List<Map<String, AliasMetadata>> resolvedAliases = MetadataIndexTemplateService.resolveAliases(simulatedState.metadata(),
  matchingTemplate);
 
@@ -200,8 +195,27 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri
  // the context is only used for validation so it's fine to pass fake values for the
  // shard id and the current timestamp
  tempIndexService.newQueryShardContext(0, null, () -> 0L, null)));
+ Map<String, AliasMetadata> aliasesByName = aliases.stream().collect(
+ Collectors.toMap(AliasMetadata::getAlias, Function.identity()));
 
- return new Template(settings, mappingsJson == null ? null : new CompressedXContent(mappingsJson),
- aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity())));
+ // empty request mapping as the user can't specify any explicit mappings via the simulate api
+ List<Map<String, Object>> mappings = MetadataCreateIndexService.collectV2Mappings(
+ "{}", simulatedState, matchingTemplate, xContentRegistry);
+
+ CompressedXContent mergedMapping = indicesService.<CompressedXContent, Exception>withTempIndexService(indexMetadata,
+ tempIndexService -> {
+ MapperService mapperService = tempIndexService.mapperService();
+ for (Map<String, Object> mapping : mappings) {
+ if (!mapping.isEmpty()) {
+ assert mapping.size() == 1 : mapping;
+ mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MapperService.MergeReason.INDEX_TEMPLATE);
+ }
+ }
+
+ DocumentMapper documentMapper = mapperService.documentMapper();
+ return documentMapper != null ? documentMapper.mappingSource() : null;
+ });
+
+ return new Template(settings, mergedMapping, aliasesByName);
  }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -27,7 +27,6 @@
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 
@@ -244,28 +243,24 @@ public TimestampField(StreamInput in) throws IOException {
  }
 
  /**
- * Force fully inserts the timestamp field mapping into the provided mapping.
- * Existing mapping definitions for the timestamp field will be completely overwritten.
- * Takes into account if the name of the timestamp field is nested.
- *
- * @param mappings The mapping to update
+ * Creates a map representing the full timestamp field mapping, taking into
+ * account if the timestamp field is nested under object mappers (its path
+ * contains dots).
  */
- public void insertTimestampFieldMapping(Map<String, Object> mappings) {
- assert mappings.containsKey("_doc");
-
+ public Map<String, Object> getTimestampFieldMapping() {
  String mappingPath = convertFieldPathToMappingPath(name);
  String parentObjectFieldPath = "_doc." + mappingPath.substring(0, mappingPath.lastIndexOf('.'));
  String leafFieldName = mappingPath.substring(mappingPath.lastIndexOf('.') + 1);
 
- Map<String, Object> changes = new HashMap<>();
- Map<String, Object> current = changes;
+ Map<String, Object> result = new HashMap<>();
+ Map<String, Object> current = result;
  for (String key : parentObjectFieldPath.split("\\.")) {
  Map<String, Object> map = new HashMap<>();
  current.put(key, map);
  current = map;
  }
  current.put(leafFieldName, fieldMapping);
- XContentHelper.update(mappings, changes, false);
+ return result;
  }
 
  @Override