okta: record the okta domain in events from input

This records the Okta Domain value from the Okta API URL provided by the configuration. This is chosen over using the okta_domain value since the URL is always present, so this simplifies the logic. The URL is passed outside the event.original to avoid collision. This means that it does not end up in the event.original. The actual Okta Domain is obtain from the URL in the ingest pipeline and then placed gingerly in host.name if possible, falling back to the same locations used in the entityanalytics_okta integration in order to harmonise the two integrations. The work to do this is done last in the pipeline to allow an unlikely failure to not interfere with other parts of the pipeline, but to provide an informative error in that case that that does happen.

Author: efd6

packages/okta/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.7.0"
+ changes:
+ - description: Retain Okta Domain value in `host.name` where possible.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/13721
 - version: "3.6.0"
  changes:
  - description: Set `user.name` from Okta `actor.alternateId` field without modification.

packages/okta/data_stream/system/_dev/test/pipeline/test-okta-system-events.json

@@ -0,0 +1,8 @@
+{
+"events": [
+{
+"message": "{\"actor\":{\"alternateId\":\"username@elastic.co\",\"detailEntry\":null,\"displayName\":\"xxxxxx\",\"id\":\"00u1abvz4pYqdM8ms4x6\",\"type\":\"User\"},\"authenticationContext\":{\"authenticationProvider\":null,\"authenticationStep\":0,\"credentialProvider\":null,\"credentialType\":null,\"externalSessionId\":\"102nZHzd6OHSfGG51vsoc22gw\",\"interface\":null,\"issuer\":null},\"client\":{\"device\":\"Computer\",\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"id\":null,\"ipAddress\":\"175.16.199.1\",\"userAgent\":{\"browser\":\"FIREFOX\",\"os\":\"Mac OS X\",\"rawUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0\"},\"zone\":\"null\"},\"debugContext\":{\"debugData\":{\"authnRequestId\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"requestId\":\"XkccyyMli2Uay2I93ZgRzQAAB0c\",\"requestUri\":\"/login/signout\",\"threatSuspected\":\"false\",\"url\":\"/login/signout?message=login_page_messages.session_has_expired\"}},\"displayMessage\":\"User logout from Okta\",\"eventType\":\"user.session.end\",\"legacyEventType\":\"core.user_auth.logout_success\",\"outcome\":{\"reason\":null,\"result\":\"SUCCESS\"},\"published\":\"2020-02-14T22:18:51.843Z\",\"request\":{\"ipChain\":[{\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"ip\":\"175.16.199.1\",\"source\":null,\"version\":\"V4\"}]},\"securityContext\":{\"asNumber\":null,\"asOrg\":null,\"domain\":null,\"isProxy\":null,\"isp\":null},\"severity\":\"INFO\",\"target\":null,\"transaction\":{\"detail\":{},\"id\":\"XkccyyMli2Uay2I93ZgRzQAAB0c\",\"type\":\"WEB\"},\"uuid\":\"faf7398a-4f77-11ea-97fb-5925e98228bd\",\"version\":\"0\"}",
+"okta_url": "https://trial-xxxxxxx-admin.okta.com/api/v1/logs"
+}
+]
+}

packages/okta/data_stream/system/_dev/test/pipeline/test-okta-system-events.json-expected.json

@@ -0,0 +1,156 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2020-02-14T22:18:51.843Z",
+ "client": {
+ "geo": {
+ "city_name": "Dublin",
+ "country_name": "United States",
+ "location": {
+ "lat": 37.7201,
+ "lon": -121.919
+ },
+ "region_name": "California"
+ },
+ "ip": "175.16.199.1",
+ "user": {
+ "full_name": "xxxxxx",
+ "id": "00u1abvz4pYqdM8ms4x6",
+ "name": "username@elastic.co"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "user.session.end",
+ "category": [
+ "authentication",
+ "session"
+ ],
+ "id": "faf7398a-4f77-11ea-97fb-5925e98228bd",
+ "kind": "event",
+ "original": "{\"actor\":{\"alternateId\":\"username@elastic.co\",\"detailEntry\":null,\"displayName\":\"xxxxxx\",\"id\":\"00u1abvz4pYqdM8ms4x6\",\"type\":\"User\"},\"authenticationContext\":{\"authenticationProvider\":null,\"authenticationStep\":0,\"credentialProvider\":null,\"credentialType\":null,\"externalSessionId\":\"102nZHzd6OHSfGG51vsoc22gw\",\"interface\":null,\"issuer\":null},\"client\":{\"device\":\"Computer\",\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"id\":null,\"ipAddress\":\"175.16.199.1\",\"userAgent\":{\"browser\":\"FIREFOX\",\"os\":\"Mac OS X\",\"rawUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0\"},\"zone\":\"null\"},\"debugContext\":{\"debugData\":{\"authnRequestId\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"requestId\":\"XkccyyMli2Uay2I93ZgRzQAAB0c\",\"requestUri\":\"/login/signout\",\"threatSuspected\":\"false\",\"url\":\"/login/signout?message=login_page_messages.session_has_expired\"}},\"displayMessage\":\"User logout from Okta\",\"eventType\":\"user.session.end\",\"legacyEventType\":\"core.user_auth.logout_success\",\"outcome\":{\"reason\":null,\"result\":\"SUCCESS\"},\"published\":\"2020-02-14T22:18:51.843Z\",\"request\":{\"ipChain\":[{\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"ip\":\"175.16.199.1\",\"source\":null,\"version\":\"V4\"}]},\"securityContext\":{\"asNumber\":null,\"asOrg\":null,\"domain\":null,\"isProxy\":null,\"isp\":null},\"severity\":\"INFO\",\"target\":null,\"transaction\":{\"detail\":{},\"id\":\"XkccyyMli2Uay2I93ZgRzQAAB0c\",\"type\":\"WEB\"},\"uuid\":\"faf7398a-4f77-11ea-97fb-5925e98228bd\",\"version\":\"0\"}",
+ "outcome": "success",
+ "type": [
+ "end",
+ "info"
+ ]
+ },
+ "host": {
+ "name": "trial-xxxxxxx-admin.okta.com"
+ },
+ "okta": {
+ "actor": {
+ "alternate_id": "username@elastic.co",
+ "display_name": "xxxxxx",
+ "id": "00u1abvz4pYqdM8ms4x6",
+ "type": "User"
+ },
+ "authentication_context": {
+ "authentication_step": 0,
+ "external_session_id": "102nZHzd6OHSfGG51vsoc22gw"
+ },
+ "client": {
+ "device": "Computer",
+ "ip": "175.16.199.1",
+ "user_agent": {
+ "browser": "FIREFOX",
+ "os": "Mac OS X",
+ "raw_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0"
+ },
+ "zone": "null"
+ },
+ "debug_context": {
+ "debug_data": {
+ "flattened": {
+ "authnRequestId": "XkcAsWb8WjwDP76xh@1v8wAABp0",
+ "requestId": "XkccyyMli2Uay2I93ZgRzQAAB0c",
+ "requestUri": "/login/signout",
+ "threatSuspected": "false",
+ "url": "/login/signout?message=login_page_messages.session_has_expired"
+ },
+ "request_id": "XkccyyMli2Uay2I93ZgRzQAAB0c",
+ "request_uri": "/login/signout",
+ "threat_suspected": "false",
+ "url": "/login/signout?message=login_page_messages.session_has_expired"
+ }
+ },
+ "display_message": "User logout from Okta",
+ "event_type": "user.session.end",
+ "outcome": {
+ "result": "SUCCESS"
+ },
+ "request": {
+ "ip_chain": [
+ {
+ "geographical_context": {
+ "city": "Dublin",
+ "country": "United States",
+ "geolocation": {
+ "lat": 37.7201,
+ "lon": -121.919
+ },
+ "postal_code": "94568",
+ "state": "California"
+ },
+ "ip": "175.16.199.1",
+ "version": "V4"
+ }
+ ]
+ },
+ "transaction": {
+ "id": "XkccyyMli2Uay2I93ZgRzQAAB0c",
+ "type": "WEB"
+ },
+ "uuid": "faf7398a-4f77-11ea-97fb-5925e98228bd"
+ },
+ "related": {
+ "ip": [
+ "175.16.199.1"
+ ],
+ "user": [
+ "xxxxxx",
+ "username@elastic.co"
+ ]
+ },
+ "source": {
+ "geo": {
+ "city_name": "Changchun",
+ "continent_name": "Asia",
+ "country_iso_code": "CN",
+ "country_name": "China",
+ "location": {
+ "lat": 43.88,
+ "lon": 125.3228
+ },
+ "region_iso_code": "CN-22",
+ "region_name": "Jilin Sheng"
+ },
+ "ip": "175.16.199.1",
+ "user": {
+ "full_name": "xxxxxx",
+ "id": "00u1abvz4pYqdM8ms4x6",
+ "name": "username@elastic.co"
+ }
+ },
+ "user": {
+ "full_name": "xxxxxx",
+ "name": "username@elastic.co"
+ },
+ "user_agent": {
+ "device": {
+ "name": "Mac"
+ },
+ "name": "Firefox",
+ "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0",
+ "os": {
+ "full": "Mac OS X 10.15",
+ "name": "Mac OS X",
+ "version": "10.15"
+ },
+ "version": "72.0"
+ }
+ }
+ ]
+}

packages/okta/data_stream/system/agent/stream/httpjson.yml.hbs

@@ -109,7 +109,11 @@ fields_under_root: true
 fields:
  _conf:
  remove_flattened_debug: {{remove_flattened_debug}}
-{{#if processors}}
 processors:
+{{#if processors}}
 {{processors}}
 {{/if}}
+- add_fields:
+ target: ''
+ fields:
+ okta_url: {{url}}

packages/okta/data_stream/system/elasticsearch/ingest_pipeline/default.yml

@@ -625,6 +625,26 @@ processors:
  field: _conf
  ignore_missing: true
  ignore_failure: true
+ - uri_parts:
+ field: okta_url
+ target_field: okta_url
+ keep_original: false
+ ignore_missing: true
+ - rename:
+ field: okta_url.domain
+ target_field: host.name
+ ignore_missing: true
+ on_failure:
+ - rename:
+ field: okta_url.domain
+ target_field: okta.okta_domain
+ on_failure:
+ - rename:
+ field: okta_url.domain
+ target_field: okta_domain
+ - remove:
+ field: okta_url
+ ignore_missing: true
 on_failure:
  - set:
  field: event.kind

packages/okta/data_stream/system/sample_event.json

@@ -1,11 +1,11 @@
 {
  "@timestamp": "2020-02-14T20:18:57.718Z",
  "agent": {
- "ephemeral_id": "2314526d-0d90-4c35-ba8f-5a48507e7dac",
- "id": "4bff512b-da80-4d11-9e9c-477b723e11d4",
- "name": "elastic-agent-20826",
+ "ephemeral_id": "eeed25bf-dc6f-48ab-b30f-a9c7fa45d6c4",
+ "id": "f466dfde-d7b7-4c3a-a3e1-9c0175d94132",
+ "name": "elastic-agent-20611",
  "type": "filebeat",
- "version": "8.17.3"
+ "version": "8.18.0"
  },
  "client": {
  "geo": {
@@ -26,16 +26,16 @@
  },
  "data_stream": {
  "dataset": "okta.system",
- "namespace": "54167",
+ "namespace": "92902",
  "type": "logs"
  },
  "ecs": {
  "version": "8.11.0"
  },
  "elastic_agent": {
- "id": "4bff512b-da80-4d11-9e9c-477b723e11d4",
+ "id": "f466dfde-d7b7-4c3a-a3e1-9c0175d94132",
  "snapshot": false,
- "version": "8.17.3"
+ "version": "8.18.0"
  },
  "event": {
  "action": "user.session.start",
@@ -44,10 +44,10 @@
  "authentication",
  "session"
  ],
- "created": "2025-04-03T02:49:51.730Z",
+ "created": "2025-04-30T01:38:47.331Z",
  "dataset": "okta.system",
  "id": "3aeede38-4f67-11ea-abd3-1f5d113f2546",
- "ingested": "2025-04-03T02:49:52Z",
+ "ingested": "2025-04-30T01:38:48Z",
  "kind": "event",
  "original": "{\"actor\":{\"alternateId\":\"xxxxxx@elastic.co\",\"detailEntry\":null,\"displayName\":\"xxxxxx\",\"id\":\"00u1abvz4pYqdM8ms4x6\",\"type\":\"User\"},\"authenticationContext\":{\"authenticationProvider\":null,\"authenticationStep\":0,\"credentialProvider\":null,\"credentialType\":null,\"externalSessionId\":\"102bZDNFfWaQSyEZQuDgWt-uQ\",\"interface\":null,\"issuer\":null},\"client\":{\"device\":\"Computer\",\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"id\":null,\"ipAddress\":\"108.255.197.247\",\"userAgent\":{\"browser\":\"FIREFOX\",\"os\":\"Mac OS X\",\"rawUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0\"},\"zone\":\"null\"},\"debugContext\":{\"debugData\":{\"deviceFingerprint\":\"541daf91d15bef64a7e08c946fd9a9d0\",\"requestId\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"requestUri\":\"/api/v1/authn\",\"threatSuspected\":\"false\",\"url\":\"/api/v1/authn?\"}},\"displayMessage\":\"User login to Okta\",\"eventType\":\"user.session.start\",\"legacyEventType\":\"core.user_auth.login_success\",\"outcome\":{\"reason\":null,\"result\":\"SUCCESS\"},\"published\":\"2020-02-14T20:18:57.718Z\",\"request\":{\"ipChain\":[{\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"ip\":\"108.255.197.247\",\"source\":null,\"version\":\"V4\"}]},\"securityContext\":{\"asNumber\":null,\"asOrg\":null,\"domain\":null,\"isProxy\":null,\"isp\":null},\"severity\":\"INFO\",\"target\":null,\"transaction\":{\"detail\":{},\"id\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"type\":\"WEB\"},\"uuid\":\"3aeede38-4f67-11ea-abd3-1f5d113f2546\",\"version\":\"0\"}",
  "outcome": "success",
@@ -56,6 +56,9 @@
  "info"
  ]
  },
+ "host": {
+ "name": "svc-okta-oauth2"
+ },
  "input": {
  "type": "httpjson"
  },

packages/okta/docs/README.md

@@ -58,11 +58,11 @@ An example event for `system` looks as following:
 {
  "@timestamp": "2020-02-14T20:18:57.718Z",
  "agent": {
- "ephemeral_id": "2314526d-0d90-4c35-ba8f-5a48507e7dac",
- "id": "4bff512b-da80-4d11-9e9c-477b723e11d4",
- "name": "elastic-agent-20826",
+ "ephemeral_id": "eeed25bf-dc6f-48ab-b30f-a9c7fa45d6c4",
+ "id": "f466dfde-d7b7-4c3a-a3e1-9c0175d94132",
+ "name": "elastic-agent-20611",
  "type": "filebeat",
- "version": "8.17.3"
+ "version": "8.18.0"
  },
  "client": {
  "geo": {
@@ -83,16 +83,16 @@ An example event for `system` looks as following:
  },
  "data_stream": {
  "dataset": "okta.system",
- "namespace": "54167",
+ "namespace": "92902",
  "type": "logs"
  },
  "ecs": {
  "version": "8.11.0"
  },
  "elastic_agent": {
- "id": "4bff512b-da80-4d11-9e9c-477b723e11d4",
+ "id": "f466dfde-d7b7-4c3a-a3e1-9c0175d94132",
  "snapshot": false,
- "version": "8.17.3"
+ "version": "8.18.0"
  },
  "event": {
  "action": "user.session.start",
@@ -101,10 +101,10 @@ An example event for `system` looks as following:
  "authentication",
  "session"
  ],
- "created": "2025-04-03T02:49:51.730Z",
+ "created": "2025-04-30T01:38:47.331Z",
  "dataset": "okta.system",
  "id": "3aeede38-4f67-11ea-abd3-1f5d113f2546",
- "ingested": "2025-04-03T02:49:52Z",
+ "ingested": "2025-04-30T01:38:48Z",
  "kind": "event",
  "original": "{\"actor\":{\"alternateId\":\"xxxxxx@elastic.co\",\"detailEntry\":null,\"displayName\":\"xxxxxx\",\"id\":\"00u1abvz4pYqdM8ms4x6\",\"type\":\"User\"},\"authenticationContext\":{\"authenticationProvider\":null,\"authenticationStep\":0,\"credentialProvider\":null,\"credentialType\":null,\"externalSessionId\":\"102bZDNFfWaQSyEZQuDgWt-uQ\",\"interface\":null,\"issuer\":null},\"client\":{\"device\":\"Computer\",\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"id\":null,\"ipAddress\":\"108.255.197.247\",\"userAgent\":{\"browser\":\"FIREFOX\",\"os\":\"Mac OS X\",\"rawUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0\"},\"zone\":\"null\"},\"debugContext\":{\"debugData\":{\"deviceFingerprint\":\"541daf91d15bef64a7e08c946fd9a9d0\",\"requestId\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"requestUri\":\"/api/v1/authn\",\"threatSuspected\":\"false\",\"url\":\"/api/v1/authn?\"}},\"displayMessage\":\"User login to Okta\",\"eventType\":\"user.session.start\",\"legacyEventType\":\"core.user_auth.login_success\",\"outcome\":{\"reason\":null,\"result\":\"SUCCESS\"},\"published\":\"2020-02-14T20:18:57.718Z\",\"request\":{\"ipChain\":[{\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"ip\":\"108.255.197.247\",\"source\":null,\"version\":\"V4\"}]},\"securityContext\":{\"asNumber\":null,\"asOrg\":null,\"domain\":null,\"isProxy\":null,\"isp\":null},\"severity\":\"INFO\",\"target\":null,\"transaction\":{\"detail\":{},\"id\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"type\":\"WEB\"},\"uuid\":\"3aeede38-4f67-11ea-abd3-1f5d113f2546\",\"version\":\"0\"}",
  "outcome": "success",
@@ -113,6 +113,9 @@ An example event for `system` looks as following:
  "info"
  ]
  },
+ "host": {
+ "name": "svc-okta-oauth2"
+ },
  "input": {
  "type": "httpjson"
  },

packages/okta/manifest.yml

@@ -1,6 +1,6 @@
 name: okta
 title: Okta
-version: "3.6.0"
+version: "3.7.0"
 description: Collect and parse event logs from Okta API with Elastic Agent.
 type: integration
 format_version: "3.2.3"