Skip to content

Commit 09db486

Browse files
eyalkoreneyalkoren
authored
Adjust ECS dynamic templates to support subobjects: false (#96712)
1 parent 13fb935 commit 09db486

File tree

3 files changed

+253
-25
lines changed

3 files changed

+253
-25
lines changed

‎docs/changelog/96712.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 96712
2+
summary: "Adjust ECS dynamic templates to support `subobjects: false`"
3+
area: Data streams
4+
type: enhancement
5+
issues: []

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
---
2+
Test flattened document with subobjects-false:
3+
# NOTE: this doesn't work. In order to run this test set "subobjects: false" through logs-mappings.json
4+
- do:
5+
cluster.put_component_template:
6+
name: logs-test-subobjects-mappings
7+
body:
8+
template:
9+
settings:
10+
mapping:
11+
ignore_malformed: true
12+
mappings:
13+
subobjects: false
14+
date_detection: false
15+
properties:
16+
data_stream.type:
17+
type: constant_keyword
18+
value: logs
19+
data_stream.dataset:
20+
type: constant_keyword
21+
data_stream.namespace:
22+
type: constant_keyword
23+
24+
- do:
25+
indices.put_index_template:
26+
name: logs-ecs-test-template
27+
body:
28+
priority: 200
29+
data_stream: {}
30+
index_patterns:
31+
- logs-*-*
32+
composed_of:
33+
- logs-test-subobjects-mappings
34+
- ecs@dynamic_templates
35+
36+
- do:
37+
indices.create_data_stream:
38+
name: logs-ecs-test-subobjects
39+
- is_true: acknowledged
40+
41+
- do:
42+
indices.get_data_stream:
43+
name: logs-ecs-test-subobjects
44+
- set: { data_streams.0.indices.0.index_name: idx0name }
45+
46+
- do:
47+
index:
48+
index: logs-ecs-test-subobjects
49+
refresh: true
50+
body: >
51+
{
52+
"@timestamp": "2023-06-12",
53+
"start_timestamp": "2023-06-08",
54+
"location" : "POINT (-71.34 41.12)",
55+
"test": "flattened",
56+
"test.start_timestamp": "not a date",
57+
"test.start-timestamp": "not a date",
58+
"registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"],
59+
"process.title": "ssh",
60+
"process.executable": "/usr/bin/ssh",
61+
"process.name": "ssh",
62+
"process.command_line": "/usr/bin/ssh -l user 10.0.0.16",
63+
"process.working_directory": "/home/ekoren",
64+
"process.io.text": "test",
65+
"url.path": "/page",
66+
"url.full": "https://mydomain.com/app/page",
67+
"url.original": "https://mydomain.com/app/original",
68+
"email.message_id": "81ce15$8r2j59@mail01.example.com",
69+
"parent.url.path": "/page",
70+
"parent.url.full": "https://mydomain.com/app/page",
71+
"parent.url.original": "https://mydomain.com/app/original",
72+
"parent.body.content": "Some content",
73+
"parent.file.path": "/path/to/my/file",
74+
"parent.file.target_path": "/path/to/my/file",
75+
"parent.registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"],
76+
"error.stack_trace": "co.elastic.test.TestClass error:\n at co.elastic.test.BaseTestClass",
77+
"error.message": "Error occurred",
78+
"file.path": "/path/to/my/file",
79+
"file.target_path": "/path/to/my/file",
80+
"os.full": "Mac OS Mojave",
81+
"user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15",
82+
"user.full_name": "John Doe",
83+
"vulnerability.score.base": 5.5,
84+
"vulnerability.score.temporal": 5.5,
85+
"vulnerability.score.version": "2.0",
86+
"vulnerability.textual_score": "bad",
87+
"host.cpu.usage": 0.68,
88+
"geo.location": [-73.614830, 45.505918],
89+
"data_stream.dataset": "nginx.access",
90+
"data_stream.namespace": "production",
91+
"data_stream.custom": "whatever",
92+
"structured_data": {"key1": "value1", "key2": ["value2", "value3"]},
93+
"exports": {"key": "value"},
94+
"top_level_imports": {"key": "value"},
95+
"nested.imports": {"key": "value"},
96+
"numeric_as_string": "42",
97+
"socket.ip": "127.0.0.1",
98+
"socket.remote_ip": "187.8.8.8"
99+
}
100+
- match: {result: "created"}
101+
102+
- do:
103+
search:
104+
index: logs-ecs-test-subobjects
105+
body:
106+
query:
107+
term:
108+
test:
109+
value: 'flattened'
110+
fields:
111+
- field: 'data_stream.type'
112+
- field: 'location'
113+
- field: 'geo.location'
114+
- field: 'test.start-timestamp'
115+
- field: 'test.start_timestamp'
116+
- field: 'vulnerability.textual_score'
117+
- length: { hits.hits: 1 }
118+
# verify that data_stream.type has the correct constant_keyword value
119+
- match: { hits.hits.0.fields.data_stream\.type.0: 'logs' }
120+
# verify geo_point subfields evaluation
121+
- match: { hits.hits.0.fields.location.0.type: 'Point' }
122+
- length: { hits.hits.0.fields.location.0.coordinates: 2 }
123+
- match: { hits.hits.0.fields.location.0.coordinates.0: -71.34 }
124+
- match: { hits.hits.0.fields.location.0.coordinates.1: 41.12 }
125+
- match: { hits.hits.0.fields.geo\.location.0.type: 'Point' }
126+
- length: { hits.hits.0.fields.geo\.location.0.coordinates: 2 }
127+
- match: { hits.hits.0.fields.geo\.location.0.coordinates.0: -73.614830 }
128+
- match: { hits.hits.0.fields.geo\.location.0.coordinates.1: 45.505918 }
129+
# "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp"
130+
# TODO: uncomment once https://github.com/elastic/elasticsearch/issues/96700 gets resolved
131+
# - match: { hits.hits.0.fields.test\.start-timestamp.0: 'not a date' }
132+
- length: { hits.hits.0._ignored: 2 }
133+
- match: { hits.hits.0._ignored.0: 'vulnerability.textual_score' }
134+
# the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type
135+
- match: { hits.hits.0._ignored.1: 'test.start_timestamp' }
136+
- length: { hits.hits.0.ignored_field_values.test\.start_timestamp: 1 }
137+
# TODO: uncomment once https://github.com/elastic/elasticsearch/issues/96700 gets resolved
138+
# - match: { hits.hits.0.ignored_field_values.test\.start_timestamp.0: 'not a date' }
139+
- length: { hits.hits.0.ignored_field_values.vulnerability\.textual_score: 1 }
140+
- match: { hits.hits.0.ignored_field_values.vulnerability\.textual_score.0: 'bad' }
141+
142+
- do:
143+
indices.get_mapping:
144+
index: logs-ecs-test-subobjects
145+
- match: { .$idx0name.mappings.properties.error\.message.type: "match_only_text" }
146+
- match: { .$idx0name.mappings.properties.registry\.data\.strings.type: "wildcard" }
147+
- match: { .$idx0name.mappings.properties.parent\.registry\.data\.strings.type: "wildcard" }
148+
- match: { .$idx0name.mappings.properties.process\.io\.text.type: "wildcard" }
149+
- match: { .$idx0name.mappings.properties.email\.message_id.type: "wildcard" }
150+
- match: { .$idx0name.mappings.properties.url\.path.type: "wildcard" }
151+
- match: { .$idx0name.mappings.properties.parent\.url\.path.type: "wildcard" }
152+
- match: { .$idx0name.mappings.properties.url\.full.type: "wildcard" }
153+
- match: { .$idx0name.mappings.properties.url\.full.fields.text.type: "match_only_text" }
154+
- match: { .$idx0name.mappings.properties.parent\.url\.full.type: "wildcard" }
155+
- match: { .$idx0name.mappings.properties.parent\.url\.full.fields.text.type: "match_only_text" }
156+
- match: { .$idx0name.mappings.properties.url\.original.type: "wildcard" }
157+
- match: { .$idx0name.mappings.properties.url\.original.fields.text.type: "match_only_text" }
158+
- match: { .$idx0name.mappings.properties.parent\.url\.original.type: "wildcard" }
159+
- match: { .$idx0name.mappings.properties.parent\.url\.original.fields.text.type: "match_only_text" }
160+
- match: { .$idx0name.mappings.properties.parent\.body\.content.type: "wildcard" }
161+
- match: { .$idx0name.mappings.properties.parent\.body\.content.fields.text.type: "match_only_text" }
162+
- match: { .$idx0name.mappings.properties.process\.command_line.type: "wildcard" }
163+
- match: { .$idx0name.mappings.properties.process\.command_line.fields.text.type: "match_only_text" }
164+
- match: { .$idx0name.mappings.properties.error\.stack_trace.type: "wildcard" }
165+
- match: { .$idx0name.mappings.properties.error\.stack_trace.fields.text.type: "match_only_text" }
166+
- match: { .$idx0name.mappings.properties.file\.path.type: "keyword" }
167+
- match: { .$idx0name.mappings.properties.file\.path.fields.text.type: "match_only_text" }
168+
- match: { .$idx0name.mappings.properties.parent\.file\.path.type: "keyword" }
169+
- match: { .$idx0name.mappings.properties.parent\.file\.path.fields.text.type: "match_only_text" }
170+
- match: { .$idx0name.mappings.properties.file\.target_path.type: "keyword" }
171+
- match: { .$idx0name.mappings.properties.file\.target_path.fields.text.type: "match_only_text" }
172+
- match: { .$idx0name.mappings.properties.parent\.file\.target_path.type: "keyword" }
173+
- match: { .$idx0name.mappings.properties.parent\.file\.target_path.fields.text.type: "match_only_text" }
174+
- match: { .$idx0name.mappings.properties.os\.full.type: "keyword" }
175+
- match: { .$idx0name.mappings.properties.os\.full.fields.text.type: "match_only_text" }
176+
- match: { .$idx0name.mappings.properties.user_agent\.original.type: "keyword" }
177+
- match: { .$idx0name.mappings.properties.user_agent\.original.fields.text.type: "match_only_text" }
178+
- match: { .$idx0name.mappings.properties.process\.title.type: "keyword" }
179+
- match: { .$idx0name.mappings.properties.process\.title.fields.text.type: "match_only_text" }
180+
- match: { .$idx0name.mappings.properties.process\.executable.type: "keyword" }
181+
- match: { .$idx0name.mappings.properties.process\.executable.fields.text.type: "match_only_text" }
182+
- match: { .$idx0name.mappings.properties.process\.name.type: "keyword" }
183+
- match: { .$idx0name.mappings.properties.process\.name.fields.text.type: "match_only_text" }
184+
- match: { .$idx0name.mappings.properties.process\.working_directory.type: "keyword" }
185+
- match: { .$idx0name.mappings.properties.process\.working_directory.fields.text.type: "match_only_text" }
186+
- match: { .$idx0name.mappings.properties.user\.full_name.type: "keyword" }
187+
- match: { .$idx0name.mappings.properties.user\.full_name.fields.text.type: "match_only_text" }
188+
- match: { .$idx0name.mappings.properties.start_timestamp.type: "date" }
189+
- match: { .$idx0name.mappings.properties.test\.start_timestamp.type: "date" }
190+
# testing the default mapping of string input fields to keyword if not matching any pattern
191+
- match: { .$idx0name.mappings.properties.test\.start-timestamp.type: "keyword" }
192+
- match: { .$idx0name.mappings.properties.vulnerability\.score\.base.type: "float" }
193+
- match: { .$idx0name.mappings.properties.vulnerability\.score\.temporal.type: "float" }
194+
- match: { .$idx0name.mappings.properties.vulnerability\.score\.version.type: "keyword" }
195+
- match: { .$idx0name.mappings.properties.vulnerability\.textual_score.type: "float" }
196+
- match: { .$idx0name.mappings.properties.host\.cpu\.usage.type: "scaled_float" }
197+
- match: { .$idx0name.mappings.properties.host\.cpu\.usage.scaling_factor: 1000 }
198+
- match: { .$idx0name.mappings.properties.location.type: "geo_point" }
199+
- match: { .$idx0name.mappings.properties.geo\.location.type: "geo_point" }
200+
- match: { .$idx0name.mappings.properties.data_stream\.dataset.type: "constant_keyword" }
201+
- match: { .$idx0name.mappings.properties.data_stream\.namespace.type: "constant_keyword" }
202+
- match: { .$idx0name.mappings.properties.data_stream\.type.type: "constant_keyword" }
203+
# not one of the three data_stream fields that are explicitly mapped to constant_keyword
204+
- match: { .$idx0name.mappings.properties.data_stream\.custom.type: "keyword" }
205+
- match: { .$idx0name.mappings.properties.structured_data.type: "flattened" }
206+
- match: { .$idx0name.mappings.properties.exports.type: "flattened" }
207+
- match: { .$idx0name.mappings.properties.top_level_imports.type: "flattened" }
208+
- match: { .$idx0name.mappings.properties.nested\.imports.type: "flattened" }
209+
# verifying the default mapping for strings into keyword, overriding the automatic numeric string detection
210+
- match: { .$idx0name.mappings.properties.numeric_as_string.type: "keyword" }
211+
- match: { .$idx0name.mappings.properties.socket\.ip.type: "ip" }
212+
- match: { .$idx0name.mappings.properties.socket\.remote_ip.type: "ip" }
213+

‎x-pack/plugin/core/src/main/resources/ecs-dynamic-mappings.json‎

Lines changed: 35 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,20 @@
1616
"mapping": {
1717
"type": "match_only_text"
1818
},
19-
"match": "message"
19+
"path_match": [
20+
"message",
21+
"*.message"
22+
]
2023
}
2124
},
2225
{
2326
"ecs_ip": {
2427
"mapping": {
2528
"type": "ip"
2629
},
27-
"match": [
30+
"path_match": [
2831
"ip",
32+
"*.ip",
2933
"*_ip"
3034
],
3135
"match_mapping_type": "string"
@@ -72,8 +76,8 @@
7276
"type": "wildcard"
7377
},
7478
"match": [
75-
"command_line",
76-
"stack_trace"
79+
"*command_line",
80+
"*stack_trace"
7781
]
7882
}
7983
},
@@ -105,12 +109,12 @@
105109
},
106110
"type": "keyword"
107111
},
108-
"match": [
109-
"title",
110-
"executable",
111-
"name",
112-
"working_directory",
113-
"full_name"
112+
"path_match": [
113+
"*.title",
114+
"*.executable",
115+
"*.name",
116+
"*.working_directory",
117+
"*.full_name"
114118
]
115119
}
116120
},
@@ -119,20 +123,23 @@
119123
"mapping": {
120124
"type": "date"
121125
},
122-
"match": [
126+
"path_match": [
123127
"timestamp",
128+
"*.timestamp",
124129
"*_timestamp",
125-
"not_after",
126-
"not_before",
127-
"accessed",
130+
"*.not_after",
131+
"*.not_before",
132+
"*.accessed",
128133
"created",
129-
"installed",
130-
"creation_date",
131-
"ctime",
132-
"mtime",
134+
"*.created",
135+
"*.installed",
136+
"*.creation_date",
137+
"*.ctime",
138+
"*.mtime",
133139
"ingested",
134-
"start",
135-
"end"
140+
"*.ingested",
141+
"*.start",
142+
"*.end"
136143
]
137144
}
138145
},
@@ -154,25 +161,28 @@
154161
"type": "scaled_float",
155162
"scaling_factor": 1000
156163
},
157-
"match": "usage"
164+
"path_match": "*.usage"
158165
}
159166
},
160167
{
161168
"ecs_geo_point": {
162169
"mapping": {
163170
"type": "geo_point"
164171
},
165-
"match": "location"
172+
"path_match": [
173+
"location",
174+
"*.location"
175+
]
166176
}
167177
},
168178
{
169179
"ecs_flattened": {
170180
"mapping": {
171181
"type": "flattened"
172182
},
173-
"match": [
174-
"structured_data",
175-
"exports",
183+
"path_match": [
184+
"*structured_data",
185+
"*exports",
176186
"*imports"
177187
]
178188
}

0 commit comments

Comments
 (0)