Revert "Add mappings for enrich fields (#96056)" (#98683)

We implemented a mapping change to enrich indices to support enrich in ESQL. Unfortunately, that change does not interact well with object fields. While we are actively addressing this issue, a comprehensive solution will take some time. To mitigate the impact of this bug, this PR will revert the mapping change for 8.9 and 8.10. The reason is that enrich for ESQL isn't required until 8.11. Relates #98019

Author: dnhatn

docs/changelog/98683.yaml

@@ -0,0 +1,5 @@
+pr: 98683
+summary: Revert "Add mappings for enrich fields"
+area: Ingest Node
+type: bug
+issues: []

x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java

@@ -39,15 +39,15 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.ReindexRequest;
 import org.elasticsearch.index.reindex.ScrollableHitSource;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.TaskCancelledException;
+import org.elasticsearch.xcontent.ObjectPath;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -57,7 +57,6 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -181,9 +180,9 @@ static void validateMappings(
  }
  // Validate the key and values
  try {
- validateAndGetMappingTypeAndFormat(mapping, policy.getMatchField(), true);
+ validateField(mapping, policy.getMatchField(), true);
  for (String valueFieldName : policy.getEnrichFields()) {
- validateAndGetMappingTypeAndFormat(mapping, valueFieldName, false);
+ validateField(mapping, valueFieldName, false);
  }
  } catch (ElasticsearchException e) {
  throw new ElasticsearchException(
@@ -195,64 +194,11 @@ static void validateMappings(
  }
  }
 
- private record MappingTypeAndFormat(String type, String format) {
-
- }
-
- private static MappingTypeAndFormat validateAndGetMappingTypeAndFormat(
- String fieldName,
- EnrichPolicy policy,
- boolean strictlyRequired,
- List<Map<String, Object>> sourceMappings
- ) {
- var fieldMappings = sourceMappings.stream()
- .map(mapping -> validateAndGetMappingTypeAndFormat(mapping, fieldName, strictlyRequired))
- .filter(Objects::nonNull)
- .toList();
- Set<String> types = fieldMappings.stream().map(tf -> tf.type).collect(Collectors.toSet());
- if (types.size() > 1) {
- if (strictlyRequired) {
- throw new ElasticsearchException(
- "Multiple distinct mapping types for field '{}' - indices({}) types({})",
- fieldName,
- Strings.collectionToCommaDelimitedString(policy.getIndices()),
- Strings.collectionToCommaDelimitedString(types)
- );
- }
- return null;
- }
- if (types.isEmpty()) {
- return null;
- }
- Set<String> formats = fieldMappings.stream().map(tf -> tf.format).filter(Objects::nonNull).collect(Collectors.toSet());
- if (formats.size() > 1) {
- if (strictlyRequired) {
- throw new ElasticsearchException(
- "Multiple distinct formats specified for field '{}' - indices({}) format entries({})",
- policy.getMatchField(),
- Strings.collectionToCommaDelimitedString(policy.getIndices()),
- Strings.collectionToCommaDelimitedString(formats)
- );
- }
- return null;
- }
- return new MappingTypeAndFormat(Iterables.get(types, 0), formats.isEmpty() ? null : Iterables.get(formats, 0));
- }
-
- @SuppressWarnings("unchecked")
- private static <T> T extractValues(Map<String, Object> properties, String path) {
- return (T) properties.get(path);
- }
-
- private static MappingTypeAndFormat validateAndGetMappingTypeAndFormat(
- Map<String, Object> properties,
- String fieldName,
- boolean fieldRequired
- ) {
+ private static void validateField(Map<?, ?> properties, String fieldName, boolean fieldRequired) {
  assert Strings.isEmpty(fieldName) == false : "Field name cannot be null or empty";
  String[] fieldParts = fieldName.split("\\.");
  StringBuilder parent = new StringBuilder();
- Map<String, Object> currentField = properties;
+ Map<?, ?> currentField = properties;
  boolean onRoot = true;
  for (String fieldPart : fieldParts) {
  // Ensure that the current field is of object type only (not a nested type or a non compound field)
@@ -265,7 +211,7 @@ private static MappingTypeAndFormat validateAndGetMappingTypeAndFormat(
  type
  );
  }
- Map<String, Object> currentProperties = extractValues(currentField, "properties");
+ Map<?, ?> currentProperties = ((Map<?, ?>) currentField.get("properties"));
  if (currentProperties == null) {
  if (fieldRequired) {
  throw new ElasticsearchException(
@@ -274,10 +220,10 @@ private static MappingTypeAndFormat validateAndGetMappingTypeAndFormat(
  onRoot ? "root" : parent.toString()
  );
  } else {
- return null;
+ return;
  }
  }
- currentField = extractValues(currentProperties, fieldPart);
+ currentField = ((Map<?, ?>) currentProperties.get(fieldPart));
  if (currentField == null) {
  if (fieldRequired) {
  throw new ElasticsearchException(
@@ -287,7 +233,7 @@ private static MappingTypeAndFormat validateAndGetMappingTypeAndFormat(
  onRoot ? "root" : parent.toString()
  );
  } else {
- return null;
+ return;
  }
  }
  if (onRoot) {
@@ -297,70 +243,95 @@ private static MappingTypeAndFormat validateAndGetMappingTypeAndFormat(
  }
  parent.append(fieldPart);
  }
- if (currentField == null) {
- return null;
+ }
+
+ private XContentBuilder resolveEnrichMapping(final EnrichPolicy enrichPolicy, final List<Map<String, Object>> mappings) {
+ if (EnrichPolicy.MATCH_TYPE.equals(enrichPolicy.getType())) {
+ return createEnrichMappingBuilder((builder) -> builder.field("type", "keyword").field("doc_values", false));
+ } else if (EnrichPolicy.RANGE_TYPE.equals(enrichPolicy.getType())) {
+ return createRangeEnrichMappingBuilder(enrichPolicy, mappings);
+ } else if (EnrichPolicy.GEO_MATCH_TYPE.equals(enrichPolicy.getType())) {
+ return createEnrichMappingBuilder((builder) -> builder.field("type", "geo_shape"));
+ } else {
+ throw new ElasticsearchException("Unrecognized enrich policy type [{}]", enrichPolicy.getType());
  }
- final String type = (String) currentField.getOrDefault("type", "object");
- final String format = (String) currentField.get("format");
- return new MappingTypeAndFormat(type, format);
  }
 
- static final Set<String> RANGE_TYPES = Set.of("integer_range", "float_range", "long_range", "double_range", "ip_range", "date_range");
+ private XContentBuilder createRangeEnrichMappingBuilder(EnrichPolicy enrichPolicy, List<Map<String, Object>> mappings) {
+ String matchFieldPath = "properties." + enrichPolicy.getMatchField().replace(".", ".properties.");
+ List<Map<String, String>> matchFieldMappings = mappings.stream()
+ .map(map -> ObjectPath.<Map<String, String>>eval(matchFieldPath, map))
+ .filter(Objects::nonNull)
+ .toList();
+
+ Set<String> types = matchFieldMappings.stream().map(map -> map.get("type")).collect(Collectors.toSet());
+ if (types.size() == 1) {
+ String type = types.iterator().next();
+ if (type == null) {
+ // when no type is defined in a field mapping then it is of type object:
+ throw new ElasticsearchException(
+ "Field '{}' has type [object] which doesn't appear to be a range type",
+ enrichPolicy.getMatchField(),
+ type
+ );
+ }
 
- static Map<String, Object> mappingForMatchField(EnrichPolicy policy, List<Map<String, Object>> sourceMappings) {
- MappingTypeAndFormat typeAndFormat = validateAndGetMappingTypeAndFormat(policy.getMatchField(), policy, true, sourceMappings);
- if (typeAndFormat == null) {
- throw new ElasticsearchException(
- "Match field '{}' doesn't have a correct mapping type for policy type '{}'",
- policy.getMatchField(),
- policy.getType()
- );
- }
- return switch (policy.getType()) {
- case EnrichPolicy.MATCH_TYPE -> Map.of("type", "keyword", "doc_values", false);
- case EnrichPolicy.GEO_MATCH_TYPE -> Map.of("type", "geo_shape");
- case EnrichPolicy.RANGE_TYPE -> {
- if (RANGE_TYPES.contains(typeAndFormat.type) == false) {
+ switch (type) {
+ case "integer_range":
+ case "float_range":
+ case "long_range":
+ case "double_range":
+ case "ip_range":
+ return createEnrichMappingBuilder((builder) -> builder.field("type", type).field("doc_values", false));
+
+ // date_range types mappings allow for the format to be specified, should be preserved in the created index
+ case "date_range":
+ Set<String> formatEntries = matchFieldMappings.stream().map(map -> map.get("format")).collect(Collectors.toSet());
+ if (formatEntries.size() == 1) {
+ return createEnrichMappingBuilder((builder) -> {
+ builder.field("type", type).field("doc_values", false);
+ String format = formatEntries.iterator().next();
+ if (format != null) {
+ builder.field("format", format);
+ }
+ return builder;
+ });
+ }
+ if (formatEntries.isEmpty()) {
+ // no format specify rely on default
+ return createEnrichMappingBuilder((builder) -> builder.field("type", type).field("doc_values", false));
+ }
  throw new ElasticsearchException(
- "Field '{}' has type [{}] which doesn't appear to be a range type",
- policy.getMatchField(),
- typeAndFormat.type
+ "Multiple distinct date format specified for match field '{}' - indices({}) format entries({})",
+ enrichPolicy.getMatchField(),
+ Strings.collectionToCommaDelimitedString(enrichPolicy.getIndices()),
+ (formatEntries.contains(null) ? "(DEFAULT), " : "") + Strings.collectionToCommaDelimitedString(formatEntries)
  );
- }
- Map<String, Object> mapping = Maps.newMapWithExpectedSize(3);
- mapping.put("type", typeAndFormat.type);
- mapping.put("doc_values", false);
- if (typeAndFormat.format != null) {
- mapping.put("format", typeAndFormat.format);
- }
- yield mapping;
- }
- default -> throw new ElasticsearchException("Unrecognized enrich policy type [{}]", policy.getType());
- };
- }
 
- private XContentBuilder createEnrichMapping(List<Map<String, Object>> sourceMappings) {
- Map<String, Map<String, Object>> fieldMappings = new HashMap<>();
- Map<String, Object> mappingForMatchField = mappingForMatchField(policy, sourceMappings);
- for (String enrichField : policy.getEnrichFields()) {
- if (enrichField.equals(policy.getMatchField())) {
- mappingForMatchField = new HashMap<>(mappingForMatchField);
- mappingForMatchField.remove("doc_values"); // enable doc_values
- } else {
- var typeAndFormat = validateAndGetMappingTypeAndFormat(enrichField, policy, false, sourceMappings);
- if (typeAndFormat != null) {
- Map<String, Object> mapping = Maps.newMapWithExpectedSize(3);
- mapping.put("type", typeAndFormat.type);
- if (typeAndFormat.format != null) {
- mapping.put("format", typeAndFormat.format);
- }
- mapping.put("index", false); // disable index
- fieldMappings.put(enrichField, mapping);
- }
+ default:
+ throw new ElasticsearchException(
+ "Field '{}' has type [{}] which doesn't appear to be a range type",
+ enrichPolicy.getMatchField(),
+ type
+ );
  }
  }
- fieldMappings.put(policy.getMatchField(), mappingForMatchField);
+ if (types.isEmpty()) {
+ throw new ElasticsearchException(
+ "No mapping type found for match field '{}' - indices({})",
+ enrichPolicy.getMatchField(),
+ Strings.collectionToCommaDelimitedString(enrichPolicy.getIndices())
+ );
+ }
+ throw new ElasticsearchException(
+ "Multiple distinct mapping types for match field '{}' - indices({}) types({})",
+ enrichPolicy.getMatchField(),
+ Strings.collectionToCommaDelimitedString(enrichPolicy.getIndices()),
+ Strings.collectionToCommaDelimitedString(types)
+ );
+ }
 
+ private XContentBuilder createEnrichMappingBuilder(CheckedFunction<XContentBuilder, XContentBuilder, IOException> matchFieldMapping) {
  // Enable _source on enrich index. Explicitly mark key mapping type.
  try {
  XContentBuilder builder = JsonXContent.contentBuilder();
@@ -376,7 +347,9 @@ private XContentBuilder createEnrichMapping(List<Map<String, Object>> sourceMapp
  builder.endObject();
  builder.startObject("properties");
  {
- builder.mapContents(fieldMappings);
+ builder.startObject(policy.getMatchField());
+ matchFieldMapping.apply(builder);
+ builder.endObject();
  }
  builder.endObject();
  builder.startObject("_meta");
@@ -407,7 +380,7 @@ private void prepareAndCreateEnrichIndex(List<Map<String, Object>> mappings) {
  .put("index.warmer.enabled", false)
  .build();
  CreateIndexRequest createEnrichIndexRequest = new CreateIndexRequest(enrichIndexName, enrichIndexSettings);
- createEnrichIndexRequest.mapping(createEnrichMapping(mappings));
+ createEnrichIndexRequest.mapping(resolveEnrichMapping(policy, mappings));
  logger.debug("Policy [{}]: Creating new enrich index [{}]", policyName, enrichIndexName);
  enrichOriginClient().admin()
  .indices()