Encorporating feedback

Author: eyalkoren

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml

@@ -111,7 +111,6 @@ Test Elastic Agent log ECS mappings:
  "stream": "stderr",
  "data_stream": {
  "namespace": "default",
- "type": "logs",
  "dataset": "kubernetes.container_logs"
  },
  "host": {
@@ -276,7 +275,7 @@ Test general mockup ECS mappings:
  "data_stream": {
  "dataset": "nginx.access",
  "namespace": "production",
- "type": "logs"
+ "custom": "whatever"
  },
  "structured_data": {
  "key1": "value1",
@@ -312,6 +311,10 @@ Test general mockup ECS mappings:
  fields:
  - field: 'start_timestamp'
  - field: 'start-timestamp'
+ script_fields:
+ data_stream_type:
+ script:
+ source: "doc['data_stream.type'].value"
  - length: { hits.hits: 1 }
  # the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type
  - length: { hits.hits.0._ignored: 2 }
@@ -320,6 +323,8 @@ Test general mockup ECS mappings:
  - match: { hits.hits.0.ignored_field_values.start_timestamp.0: 'not a date' }
  # "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp"
  - match: { hits.hits.0.fields.start-timestamp.0: 'not a date' }
+ # verify that data_stream.type has the correct constant_keyword value
+ - match: { hits.hits.0.fields.data_stream_type.0: 'logs' }
  - match: { hits.hits.0._ignored.1: 'vulnerability.textual_score' }
 
  - do:
@@ -381,6 +386,8 @@ Test general mockup ECS mappings:
  - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" }
  - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" }
  - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" }
+ # not one of the three data_stream fields that are explicitly mapped to constant_keyword
+ - match: { .$idx0name.mappings.properties.data_stream.properties.custom.type: "keyword" }
  - match: { .$idx0name.mappings.properties.structured_data.type: "flattened" }
  - match: { .$idx0name.mappings.properties.exports.type: "flattened" }
  - match: { .$idx0name.mappings.properties.top_level_imports.type: "flattened" }

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1841,7 +1841,6 @@ protected static boolean isXPackTemplate(String name) {
  case "data-streams-mappings":
  case "ecs-dynamic-mappings":
  case "ecs-data-stream-mappings":
- case "string-to-keyword-dynamic-mapping":
  return true;
  default:
  return false;

x-pack/plugin/core/src/main/resources/ecs-data-stream-mappings.json

@@ -7,7 +7,11 @@
  "mapping": {
  "type": "constant_keyword"
  },
- "path_match": "data_stream.*"
+ "path_match": [
+ "data_stream.dataset",
+ "data_stream.namespace",
+ "data_stream.type"
+ ]
  }
  }
  ]

x-pack/plugin/core/src/main/resources/ecs-dynamic-mappings.json

@@ -176,6 +176,15 @@
  "*imports"
  ]
  }
+ },
+ {
+ "all_strings_to_keywords": {
+ "mapping": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "match_mapping_type": "string"
+ }
  }
  ]
  }

x-pack/plugin/core/src/main/resources/logs-template.json

@@ -4,11 +4,10 @@
  "data_stream": {},
  "composed_of": [
  "logs-mappings",
- "ecs-dynamic-mappings",
  "ecs-data-stream-mappings",
  "logs-settings",
  "logs@custom",
- "string-to-keyword-dynamic-mapping"
+ "ecs-dynamic-mappings"
  ],
  "ignore_missing_component_templates": ["logs@custom"],
  "allow_auto_create": true,

x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java

@@ -60,9 +60,6 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
  // ECS dynamic mappings for the data_stream fields
  public static final String ECS_DATA_STREAM_MAPPINGS_COMPONENT_TEMPLATE_NAME = "ecs-data-stream-mappings";
 
- // Default mapping of input string fields to keyword
- public static final String STRING_TO_KEYWORD_DYNAMIC_TEMPLATE_NAME = "string-to-keyword-dynamic-mapping";
-
  //////////////////////////////////////////////////////////
  // Built in ILM policies for users to use
  //////////////////////////////////////////////////////////
@@ -176,12 +173,6 @@ protected List<LifecyclePolicy> getPolicyConfigs() {
  REGISTRY_VERSION,
  TEMPLATE_VERSION_VARIABLE
  ),
- new IndexTemplateConfig(
- STRING_TO_KEYWORD_DYNAMIC_TEMPLATE_NAME,
- "/string-to-keyword-dynamic-mapping.json",
- REGISTRY_VERSION,
- TEMPLATE_VERSION_VARIABLE
- ),
  new IndexTemplateConfig(
  LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME,
  "/logs-settings.json",

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java

@@ -373,7 +373,6 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
  versions.put(StackTemplateRegistry.DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
  versions.put(StackTemplateRegistry.ECS_DYNAMIC_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
  versions.put(StackTemplateRegistry.ECS_DATA_STREAM_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
- versions.put(StackTemplateRegistry.STRING_TO_KEYWORD_DYNAMIC_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
  versions.put(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
  versions.put(StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
  versions.put(StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
@@ -411,10 +410,6 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
  StackTemplateRegistry.ECS_DATA_STREAM_MAPPINGS_COMPONENT_TEMPLATE_NAME,
  StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
  );
- versions.put(
- StackTemplateRegistry.STRING_TO_KEYWORD_DYNAMIC_TEMPLATE_NAME,
- StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
- );
  versions.put(
  StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME,
  StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)