sentinel_one: Add configuration option to filter results by Site IDs, fix error handling for CEL, improve UI layout, and readme document

Add a configuration option to filter results by Site IDs for the application data stream. Fix error handling for CEL code in the application risk data stream. Update the configuration parameter descriptions and the README document.

Author: brijesh-elastic

packages/sentinel_one/_dev/build/docs/README.md

@@ -25,12 +25,28 @@ To collect data from SentinelOne APIs, you must have an API token. To create an
  ![SentinelOne generate MFA Code ](../img/sentinel-one-mfa-code.png)
  6. You will see the API token on the screen.
 
-## Note
+**Permissions Required for the Role Attached to the User**
+
+| **Data Stream** | **Permission** |
+|-------------------|---------------------------------|
+| Activity | Activity -> view |
+| Agent | Endpoints -> view |
+| Alert | STAR Rule Alerts -> view |
+| Application | Applications -> view |
+| Application Risk | Applications -> viewRisks |
+| Group | Groups -> view |
+| Threat | Threats -> view |
 
-The API token generated by the user is time-limited. To rotate a new token, log in with the dedicated admin account.
+## Note
 
 The **alert** data stream depends on STAR Custom Rules. STAR Custom Rules are supported in Cloud environments, but are not supported in on-premises environments. Because of this, the **alert** data stream is not supported in on-premises environments.
 
+## Troubleshooting
+
+- The API token generated by the user is time-limited. The user must reconfigure a new API token before it expires.
+ - For console users, the default expiration time limit is 30 days.
+ - For service users, the expiration time limit is the same as the duration specified while generating the API token.
+
 ## Alert severity mapping
 
 The values used in `event.severity` are consistent with Elastic Detection Rules.

packages/sentinel_one/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.41.0"
+ changes:
+ - description: Add configuration option to filter results by Site IDs in the application data stream, improve UI layout, and readme document.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/15446
+ - description: 'Enhanced error handling in the CEL program for API calls to prevent "no such key: batch_size" errors in the application risk data stream.'
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/15446
 - version: "1.40.0"
  changes:
  - description: Update deprecated visualization charts and implemented best practices in the existing dashboards.

packages/sentinel_one/data_stream/application/_dev/test/policy/test-all.expected

@@ -32,6 +32,7 @@ inputs:
  state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory?" + {
  "skipCount": ["true"],
  "limit": [string(state.batch_size)],
+ ?"siteIds": state.?site_ids.optMap(v, [string(v)]),
  ?"cursor": state.?next_page.token.optMap(v, [v]),
  }.format_query()
  ).with(
@@ -56,7 +57,7 @@ inputs:
  "error": {
  "code": string(resp.StatusCode),
  "id": string(resp.Status),
- "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory" + (
+ "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory: " + (
  (size(resp.Body) != 0) ?
  string(resp.Body)
  :
@@ -81,6 +82,7 @@ inputs:
  "applicationName": [string(state.worklist.data[0].applicationName)],
  "applicationVendor": [string(state.worklist.data[0].applicationVendor)],
  "limit": [string(state.batch_size)],
+ ?"siteIds": state.?site_ids.optMap(v, [string(v)]),
  ?"cursor": state.?next_chain.token.optMap(v, [v]),
  }.format_query()
  ).with(
@@ -115,7 +117,7 @@ inputs:
  "error": {
  "code": string(resp.StatusCode),
  "id": string(resp.Status),
- "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory/endpoints" + (
+ "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory/endpoints: " + (
  (size(resp.Body) != 0) ?
  string(resp.Body)
  :
@@ -226,6 +228,7 @@ inputs:
  state:
  api_token: ${SECRET_0}
  batch_size: 100
+ site_ids: "123"
  tags:
  - preserve_original_event
  - preserve_duplicate_custom_fields

packages/sentinel_one/data_stream/application/_dev/test/policy/test-all.yml

@@ -85,6 +85,7 @@ data_stream:
  vars:
  interval: 30s
  batch_size: 100
+ site_ids: 123
  tags:
  - forwarded
  - sentinel_one-application

packages/sentinel_one/data_stream/application/_dev/test/policy/test-default.expected

@@ -21,6 +21,7 @@ inputs:
  state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory?" + {
  "skipCount": ["true"],
  "limit": [string(state.batch_size)],
+ ?"siteIds": state.?site_ids.optMap(v, [string(v)]),
  ?"cursor": state.?next_page.token.optMap(v, [v]),
  }.format_query()
  ).with(
@@ -45,7 +46,7 @@ inputs:
  "error": {
  "code": string(resp.StatusCode),
  "id": string(resp.Status),
- "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory" + (
+ "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory: " + (
  (size(resp.Body) != 0) ?
  string(resp.Body)
  :
@@ -70,6 +71,7 @@ inputs:
  "applicationName": [string(state.worklist.data[0].applicationName)],
  "applicationVendor": [string(state.worklist.data[0].applicationVendor)],
  "limit": [string(state.batch_size)],
+ ?"siteIds": state.?site_ids.optMap(v, [string(v)]),
  ?"cursor": state.?next_chain.token.optMap(v, [v]),
  }.format_query()
  ).with(
@@ -104,7 +106,7 @@ inputs:
  "error": {
  "code": string(resp.StatusCode),
  "id": string(resp.Status),
- "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory/endpoints" + (
+ "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory/endpoints: " + (
  (size(resp.Body) != 0) ?
  string(resp.Body)
  :

packages/sentinel_one/data_stream/application/agent/stream/cel.yml.hbs

@@ -17,6 +17,9 @@ resource.url: {{url}}
 state:
  api_token: {{api_token}}
  batch_size: {{batch_size}}
+{{#if site_ids }}
+ site_ids: !!str {{site_ids}}
+{{/if}}
 redact:
  fields:
  - api_token
@@ -31,6 +34,7 @@ program: |-
  state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory?" + {
  "skipCount": ["true"],
  "limit": [string(state.batch_size)],
+ ?"siteIds": state.?site_ids.optMap(v, [string(v)]),
  ?"cursor": state.?next_page.token.optMap(v, [v]),
  }.format_query()
  ).with(
@@ -55,7 +59,7 @@ program: |-
  "error": {
  "code": string(resp.StatusCode),
  "id": string(resp.Status),
- "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory" + (
+ "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory: " + (
  (size(resp.Body) != 0) ?
  string(resp.Body)
  :
@@ -80,6 +84,7 @@ program: |-
  "applicationName": [string(state.worklist.data[0].applicationName)],
  "applicationVendor": [string(state.worklist.data[0].applicationVendor)],
  "limit": [string(state.batch_size)],
+ ?"siteIds": state.?site_ids.optMap(v, [string(v)]),
  ?"cursor": state.?next_chain.token.optMap(v, [v]),
  }.format_query()
  ).with(
@@ -114,7 +119,7 @@ program: |-
  "error": {
  "code": string(resp.StatusCode),
  "id": string(resp.Status),
- "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory/endpoints" + (
+ "message": "GET " + state.url.trim_right("/") + "/web/api/v2.1/application-management/inventory/endpoints: " + (
  (size(resp.Body) != 0) ?
  string(resp.Body)
  :

packages/sentinel_one/data_stream/application/manifest.yml

@@ -23,6 +23,21 @@ streams:
  multi: false
  required: true
  show_user: false
+ - name: site_ids
+ type: text
+ title: Site IDs
+ multi: false
+ required: false
+ show_user: false
+ description: Comma separated list of Site IDs to filter by. Example - "225494730938493804,225494730938493915".
+ - name: http_client_timeout
+ type: text
+ title: HTTP Client Timeout
+ description: Duration before declaring that the HTTP client connection has timed out. Supported time units are ns, us, ms, s, m, h.
+ multi: false
+ required: true
+ show_user: false
+ default: 30s
  - name: enable_request_tracer
  type: bool
  title: Enable request tracing
@@ -55,14 +70,6 @@ streams:
  description: Preserve sentinel_one.application fields that were copied to Elastic Common Schema (ECS) fields.
  type: bool
  multi: false
- - name: http_client_timeout
- type: text
- title: HTTP Client Timeout
- description: Duration before declaring that the HTTP client connection has timed out. Supported time units are ns, us, ms, s, m, h.
- multi: false
- required: true
- show_user: false
- default: 30s
  - name: processors
  type: yaml
  title: Processors

packages/sentinel_one/data_stream/application_risk/_dev/test/policy/test-all.expected

@@ -25,7 +25,7 @@ inputs:
  request("GET",
  state.url.trim_right("/") + "/web/api/v2.1/application-management/risks?" + {
  ?"cursor": state.?next.page.optMap(v, [v]),
- ?"siteids": state.?site_ids.orValue(null) != null ? optional.of([string(state.site_ids)]) : optional.none(),
+ ?"siteIds": state.?site_ids.optMap(v, [string(v)]),
  "limit": [string(state.batch_size)],
  }.format_query()
  ).with({
@@ -39,7 +39,7 @@ inputs:
  }),
  "api_token": state.api_token,
  "batch_size": state.batch_size,
- ?"site_ids": state.?site_ids.orValue(null) != null ? optional.of(state.site_ids) : optional.none(),
+ ?"site_ids": state.?site_ids,
  "next": {
  ?"page": body.?pagination.?nextCursor.orValue(null) != null ? optional.of(body.pagination.nextCursor) : optional.none(),
  },
@@ -60,6 +60,9 @@ inputs:
  },
  },
  "want_more": false,
+ "api_token": state.api_token,
+ "batch_size": state.batch_size,
+ ?"site_ids": state.?site_ids,
  }
  )
  publisher_pipeline.disable_host: true
@@ -155,7 +158,7 @@ inputs:
  state:
  api_token: ${SECRET_0}
  batch_size: 100
- site_ids: null
+ site_ids: "123"
  tags:
  - preserve_original_event
  - preserve_duplicate_custom_fields

packages/sentinel_one/data_stream/application_risk/_dev/test/policy/test-all.yml

@@ -85,6 +85,7 @@ data_stream:
  vars:
  interval: 30s
  batch_size: 100
+ site_ids: 123
  tags:
  - forwarded
  - sentinel_one-application_risk

packages/sentinel_one/data_stream/application_risk/_dev/test/policy/test-default.expected

@@ -14,7 +14,7 @@ inputs:
  request("GET",
  state.url.trim_right("/") + "/web/api/v2.1/application-management/risks?" + {
  ?"cursor": state.?next.page.optMap(v, [v]),
- ?"siteids": state.?site_ids.orValue(null) != null ? optional.of([string(state.site_ids)]) : optional.none(),
+ ?"siteIds": state.?site_ids.optMap(v, [string(v)]),
  "limit": [string(state.batch_size)],
  }.format_query()
  ).with({
@@ -28,7 +28,7 @@ inputs:
  }),
  "api_token": state.api_token,
  "batch_size": state.batch_size,
- ?"site_ids": state.?site_ids.orValue(null) != null ? optional.of(state.site_ids) : optional.none(),
+ ?"site_ids": state.?site_ids,
  "next": {
  ?"page": body.?pagination.?nextCursor.orValue(null) != null ? optional.of(body.pagination.nextCursor) : optional.none(),
  },
@@ -49,6 +49,9 @@ inputs:
  },
  },
  "want_more": false,
+ "api_token": state.api_token,
+ "batch_size": state.batch_size,
+ ?"site_ids": state.?site_ids,
  }
  )
  publisher_pipeline.disable_host: true
@@ -65,7 +68,6 @@ inputs:
  state:
  api_token: ${SECRET_0}
  batch_size: 1000
- site_ids: null
  tags:
  - forwarded
  - sentinel_one-application_risk