Resolve review comments.

1. Update descriptions of config parameters. 2. Add saved search in dashboard. 3. Update query parameter in data collection, moved it to the request body. 4. Preserve event.original value from message field. 5. Update readme.

Author: mohitjha-elastic

packages/elastic_security/_dev/build/docs/README.md

@@ -2,13 +2,14 @@
 
 ## Overview
 
-Elastic Security Alerts are triggered when detection rules identify suspicious or malicious activity. They provide detailed context like rule name, impacted entities, timestamps, and other necessary details. Alerts can be investigated in Kibana using tools like Timeline. They support custom actions such as notifications or automated responses. These alerts help prioritize and manage security threats efficiently.
+Elastic Security is a free and open solution that helps detect, investigate, and respond to threats using data from endpoints, cloud, and network sources. It offers SIEM and endpoint protection with powerful search, correlation, and visualization features in Kibana.
+It enables security teams to streamline investigations and strengthen their overall security posture.
 
 ## Data streams
 
-This integration collects the following logs:
+The Elastic Security integration collects the following events:
 
-`alert`: - Retrieve alerts from Elastic Instance.
+`alert`: - Retrieve alerts from Elasticsearch Instance using Elasticsearch [_search](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-search) API.
 
 ## Requirements
 
@@ -26,17 +27,21 @@ Elastic Agent must be installed. For more details, check the Elastic Agent [inst
 
 ### To collect data from the Elastic API:
 
-To collect data from the Elastic API, you will need the following information:
+You will need the following information:
 
 1. The URL for the Elasticsearch instance.
-2. Authentication credentials such as username, password, API key, or bearer token depend on the selected authentication type.
+2. Authentication credentials such as username, password, API key, or bearer token depending on the selected authentication type.
+
+Note:
+1. Users must have read index privileges on the `.alerts-security.alerts*` indices to access and query security alerts.
+2. To learn how to create authentication credentials and use the appropriate authentication type, refer to the Elasticsearch Authentication [Documentation](https://www.elastic.co/docs/api/doc/elasticsearch/authentication).
 
 ### Enable the integration in Elastic
 
 1. In Kibana navigate to **Management** > **Integrations**.
 2. In the search top bar, type **Elastic Security**.
 3. Select the **Elastic Security** integration and add it.
-4. Add all the required integration configuration parameters such as username, password, API key, or bearer token depend on the selected authentication type to enable data collection.
+4. Add all the required integration configuration parameters such as username, password, API key, or bearer token depending on the selected authentication type to enable data collection.
 5. Select "Save and continue" to save the integration.
 
 ## Logs reference

packages/elastic_security/_dev/deploy/docker/files/config.yml

@@ -1,12 +1,12 @@
 rules:
- - path: /.internal.alerts-security.alerts*/_search
+ - path: /.alerts-security.alerts*/_search
  methods: ['GET']
  query_params:
  size: 1
- q: "{q:@timestamp:{2025.*}}"
  request_headers:
  Authorization:
  - "Apikey xxxx"
+ request_body: /.*2025.*/
  responses:
  - status_code: 200
  body: |-
@@ -1133,14 +1133,14 @@ rules:
  }
  }
  `}}
- - path: /.internal.alerts-security.alerts*/_search
+ - path: /.alerts-security.alerts*/_search
  methods: ['GET']
  query_params:
  size: 1
- q: "{q:@timestamp:{2060.*}}"
  request_headers:
  Authorization:
  - "Apikey xxxx"
+ request_body: /.*2060.*/
  responses:
  - status_code: 200
  body: |-
@@ -1149,8 +1149,8 @@ rules:
  "hits": {
  "hits": [
  {
- "_index": "abcd-1234",
- "_id": "fghiabcd",
+ "_index": "xyz-1234",
+ "_id": "xyz1234",
  "_source": {
  "kibana.alert.start": "2024-06-09T13:56:03.235Z",
  "kibana.alert.last_detected": "2024-06-09T13:56:03.235Z",
@@ -2267,14 +2267,14 @@ rules:
  }
  }
  `}}
- - path: /.internal.alerts-security.alerts*/_search
+ - path: /.alerts-security.alerts*/_search
  methods: ['GET']
  query_params:
  size: 1
- q: "{q:@timestamp:{2070.*}}"
  request_headers:
  Authorization:
  - "Apikey xxxx"
+ request_body: /.*2070.*/
  responses:
  - status_code: 200
  body: |-

packages/elastic_security/data_stream/alert/_dev/test/pipeline/test-alert.log

@@ -8,6 +8,5 @@ data_stream:
  initial_interval: 24h
  batch_size: 1
  preserve_original_event: true
- preserve_duplicate_custom_fields: true
 assert:
  hit_count: 2

packages/elastic_security/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json

@@ -20,7 +20,7 @@ state:
  user: {{username}}
  password: {{password}}
  bearer_token: {{bearer_token}}
- index: .internal.alerts-security.alerts*
+ index: .alerts-security.alerts*
  batch_size: {{batch_size}}
  initial_interval: {{initial_interval}}
 redact:
@@ -38,8 +38,17 @@ program: |
  state.url.trim_right("/") + "/" + state.index + "/_search?" + {
  "sort": ["@timestamp:asc"],
  "size": [string(state.batch_size)],
- "q": ["@timestamp:{" + state.start_time + " TO now}"],
- }.format_query()
+ }.format_query(),
+ {
+ "query": {
+ "range": {
+ "@timestamp": {
+ "gt": state.start_time,
+ "lte": "now"
+ }
+ }
+ }
+ }.encode_json()
  ).with({
  "Header":{
  "Content-Type": ["application/json"],

packages/elastic_security/data_stream/alert/_dev/test/system/test-default-config.yml

@@ -1,10 +1,15 @@
 ---
 description: Pipeline for processing Alert logs.
 processors:
- - fail:
+ - terminate:
  tag: data_collection_error
  if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null
- message: error message set and no data to process.
+ description: error message set and no data to process.
+ - set:
+ field: event.original
+ tag: set_event_original
+ copy_from: message
+ ignore_empty_value: true
  - json:
  field: message
  tag: json_message
@@ -14,6 +19,19 @@ processors:
  - append:
  field: error.message
  value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+ - script:
+ lang: painless
+ description: Script to remove message.
+ tag: script_remove_message
+ if: ctx.event?.original != null && ctx.message != null
+ source: |-
+ if(ctx.event.original == ctx.message) {
+ ctx.remove('message');
+ }
+ on_failure:
+ - append:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
  - remove:
  field:
  - organization
@@ -25,24 +43,13 @@ processors:
  description: >-
  Removes the fields added by Agentless as metadata,
  as they can collide with ECS fields.
- - rename:
- field: message
- tag: rename_message_to_event_original
- target_field: event.original
- ignore_missing: true
- description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.
- if: ctx.event?.original == null
- - remove:
- field: message
- tag: remove_message
- ignore_missing: true
- description: The `message` field is no longer required if the document has an `event.original` field.
- if: ctx.event?.original != null
  - dot_expander:
  field: "*"
+ tag: dot_expander_source_fields
  ignore_failure: true
  - uppercase:
  field: host.mac
+ tag: uppercase_host_mac
  ignore_missing: true
  - convert:
  field: user.id
@@ -54,11 +61,6 @@ processors:
  tag: convert_group_id
  type: string
  ignore_missing: true
- - append:
- field: tags
- value: preserve_original_event
- allow_duplicates: false
- if: ctx.error?.message != null
  - set:
  field: event.kind
  tag: set_event_kind

packages/elastic_security/data_stream/alert/agent/stream/cel.yml.hbs

@@ -6,6 +6,7 @@
  fields:
  - name: id
  type: keyword
+# Kibana-specific fields
 - name: kibana
  type: group
  fields:
@@ -261,6 +262,7 @@
  type: keyword
  - name: version
  type: keyword
+# Endpoint-specific fields
 - name: Endpoint
  type: group
  fields:
@@ -731,8 +733,6 @@
  fields:
  - name: ancestry
  type: keyword
- - name: args
- type: keyword
  - name: args_count
  type: long
  - name: command_line

packages/elastic_security/data_stream/alert/elasticsearch/ingest_pipeline/default.yml

@@ -20,11 +20,11 @@ streams:
  - text: Bearer Auth
  value: bearer_auth
  default: api_auth
- description: Provide the type of authentication to be used for the request.
+ description: Type of authentication to be used for the Elasticsearch API requests. See [documentation](https://www.elastic.co/docs/api/doc/elasticsearch/authentication) for details.
  - name: url
  type: url
  title: URL
- description: URL of Elasticsearch instance.
+ description: URL of the Elasticsearch instance. Example `https://<host>:<port>`.
  required: true
  show_user: true
  - name: username
@@ -65,7 +65,7 @@ streams:
  required: true
  show_user: true
  default: 24h
- description: How far back to pull the logs from Elasticsearch API. Supported units for this parameter are h/m/s.
+ description: How far back to pull the events from Elasticsearch API. Supported units for this parameter are h/m/s. Example `72h`.
  - name: interval
  type: text
  title: Interval