google_workspace: fix JSON error and login challenge_status handling (#13070)

Also improve error reporting for cases where JSON parsing cannot be successfully carried out. Previously JSON parsing failures would be silently ignored leaving no indication that that was the cause of a document not being processed correctly. With the changes here, failing to parse the JSON results in pipeline termination with an informative message since no furthe work can be done without the parsed JSON.

Author: efd6

packages/google_workspace/changelog.yml

@@ -1,4 +1,15 @@
 # newer versions go on top
+- version: "2.34.0"
+ changes:
+ - description: Improve error handling in ingest pipeline.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/13070
+ - description: Improve handling of JSON parsing failures.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/13070
+ - description: Fix handing of `google_workspace.login.challenge_status`.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/13070
 - version: "2.33.0"
  changes:
  - description: Enable request trace log removal.

packages/google_workspace/data_stream/admin/elasticsearch/ingest_pipeline/default.yml

@@ -20,7 +20,6 @@ processors:
  - json:
  field: event.original
  target_field: json
- ignore_failure: true
  - set:
  field: event.kind
  value: event
@@ -834,4 +833,7 @@ on_failure:
  allow_duplicates: false
  - append:
  field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'

packages/google_workspace/data_stream/alert/elasticsearch/ingest_pipeline/default.yml

@@ -17,7 +17,6 @@ processors:
  - json:
  field: event.original
  target_field: json
- ignore_failure: true
  - append:
  field: event.type
  value: info
@@ -1067,4 +1066,7 @@ on_failure:
  allow_duplicates: false
  - append:
  field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'

packages/google_workspace/data_stream/drive/elasticsearch/ingest_pipeline/default.yml

@@ -23,7 +23,6 @@ processors:
  - json:
  field: event.original
  target_field: json
- ignore_failure: true
  - date:
  field: json.id.time
  if: ctx.json?.id?.time != null && ctx.json.id.time != ''
@@ -287,4 +286,7 @@ on_failure:
  allow_duplicates: false
  - append:
  field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'

packages/google_workspace/data_stream/groups/elasticsearch/ingest_pipeline/default.yml

@@ -26,7 +26,6 @@ processors:
  - json:
  field: event.original
  target_field: json
- ignore_failure: true
  - date:
  field: json.id.time
  if: ctx.json?.id?.time != null && ctx.json.id.time != ''
@@ -318,4 +317,7 @@ on_failure:
  allow_duplicates: false
  - append:
  field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'

packages/google_workspace/data_stream/login/_dev/test/pipeline/test-login.log

@@ -13,4 +13,4 @@
 {"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"67.43.156.13","events":{"type":"login","name":"logout","parameters":[{"name":"login_type","value":"exchange"}]}}
 {"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"67.43.156.13","events":{"type":"login","name":"login_success","parameters":[{"name":"login_challenge_method","value":"backup_code"},{"name":"is_suspicious","boolValue":false},{"name":"login_type","value":"exchange"}]}}
 {"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"login","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"67.43.156.13","events":{"type":"login","name":"login_success","parameters":[{"name":"login_challenge_method","value":"password"},{"name":"is_suspicious","boolValue":true},{"name":"login_type","value":"google_password"}]}}
-
+{"actor":{"email":"tl.zeous.daclitan@company.com","profileId":"111111111"},"etag":"Q2W123123123123","events":{"name":"login_verification","parameters":[{"name":"login_type","value":"google_password"},{"multiValue":["security_key"],"name":"login_challenge_method"},{"name":"login_challenge_status","value":"passed"},{"boolValue":true,"name":"is_second_factor"}],"type":"login"},"id":{"applicationName":"login","customerId":"123","time":"2025-02-27T05:59:58.481Z","uniqueQualifier":"123"},"ipAddress":"81.2.69.144","kind":"admin#reports#activity"}

packages/google_workspace/data_stream/login/_dev/test/pipeline/test-login.log-expected.json

@@ -817,7 +817,7 @@
  "id": "1",
  "kind": "event",
  "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"login\",\"name\":\"login_challenge\",\"parameters\":[{\"name\":\"login_challenge_method\",\"value\":\"backup_code\"},{\"name\":\"login_challenge_status\",\"value\":\"Challenge Passed.\"},{\"name\":\"login_type\",\"value\":\"exchange\"}]}}",
- "outcome": "failure",
+ "outcome": "success",
  "provider": "login",
  "type": [
  "info"
@@ -895,7 +895,7 @@
  "id": "1",
  "kind": "event",
  "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"login\",\"name\":\"login_verification\",\"parameters\":[{\"name\":\"is_second_factor\",\"boolValue\":false},{\"name\":\"login_challenge_method\",\"value\":\"backup_code\"},{\"name\":\"login_challenge_status\",\"value\":\"Challenge Passed.\"},{\"name\":\"login_type\",\"value\":\"exchange\"}]}}",
- "outcome": "failure",
+ "outcome": "success",
  "provider": "login",
  "type": [
  "info"
@@ -1196,16 +1196,79 @@
  }
  },
  {
+ "@timestamp": "2025-02-27T05:59:58.481Z",
  "ecs": {
  "version": "8.16.0"
  },
  "event": {
+ "action": "login_verification",
+ "category": [
+ "authentication"
+ ],
+ "id": "123",
  "kind": "event",
- "original": ""
+ "original": "{\"actor\":{\"email\":\"tl.zeous.daclitan@company.com\",\"profileId\":\"111111111\"},\"etag\":\"Q2W123123123123\",\"events\":{\"name\":\"login_verification\",\"parameters\":[{\"name\":\"login_type\",\"value\":\"google_password\"},{\"multiValue\":[\"security_key\"],\"name\":\"login_challenge_method\"},{\"name\":\"login_challenge_status\",\"value\":\"passed\"},{\"boolValue\":true,\"name\":\"is_second_factor\"}],\"type\":\"login\"},\"id\":{\"applicationName\":\"login\",\"customerId\":\"123\",\"time\":\"2025-02-27T05:59:58.481Z\",\"uniqueQualifier\":\"123\"},\"ipAddress\":\"81.2.69.144\",\"kind\":\"admin#reports#activity\"}",
+ "outcome": "success",
+ "provider": "login",
+ "type": [
+ "info"
+ ]
+ },
+ "google_workspace": {
+ "event": {
+ "type": "login"
+ },
+ "kind": "admin#reports#activity",
+ "login": {
+ "challenge_method": [
+ "security_key"
+ ],
+ "challenge_status": "passed",
+ "is_second_factor": true,
+ "type": "google_password"
+ }
+ },
+ "organization": {
+ "id": "123"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ],
+ "user": [
+ "tl.zeous.daclitan"
+ ]
+ },
+ "source": {
+ "geo": {
+ "city_name": "London",
+ "continent_name": "Europe",
+ "country_iso_code": "GB",
+ "country_name": "United Kingdom",
+ "location": {
+ "lat": 51.5142,
+ "lon": -0.0931
+ },
+ "region_iso_code": "GB-ENG",
+ "region_name": "England"
+ },
+ "ip": "81.2.69.144",
+ "user": {
+ "domain": "company.com",
+ "email": "tl.zeous.daclitan@company.com",
+ "id": "111111111",
+ "name": "tl.zeous.daclitan"
+ }
  },
  "tags": [
  "preserve_original_event"
- ]
+ ],
+ "user": {
+ "domain": "company.com",
+ "email": "tl.zeous.daclitan@company.com",
+ "id": "111111111",
+ "name": "tl.zeous.daclitan"
+ }
  }
  ]
 }

packages/google_workspace/data_stream/login/elasticsearch/ingest_pipeline/default.yml

@@ -17,7 +17,6 @@ processors:
  - json:
  field: event.original
  target_field: json
- ignore_failure: true
  - set:
  field: event.kind
  value: event
@@ -216,11 +215,11 @@ processors:
  - set: 
  field: event.outcome
  value: success
- if: 'ctx?.google_workspace?.login?.challenge_status != null && ctx?.event?.outcome == null && ctx?.google_workspace?.login?.challenge_status == "Challenge Passed"'
+ if: ctx.event?.outcome == null && ctx.google_workspace?.login?.challenge_status?.toLowerCase()?.contains('passed') == true
  - set: 
  field: event.outcome
  value: failure
- if: 'ctx?.google_workspace?.login?.challenge_status != null && ctx?.event?.outcome == null'
+ if: 'ctx.google_workspace?.login?.challenge_status != null && ctx.event?.outcome == null'
  - script:
  lang: painless
  if: 'ctx?.google_workspace?.login?.affected_email_address != null && ctx?.google_workspace?.login?.affected_email_address.contains("@")'
@@ -279,4 +278,7 @@ on_failure:
  allow_duplicates: false
  - append:
  field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'

packages/google_workspace/data_stream/rules/elasticsearch/ingest_pipeline/default.yml

@@ -17,7 +17,6 @@ processors:
  - json:
  field: event.original
  target_field: json
- ignore_failure: true
  - date:
  field: json.id.time
  if: ctx.json?.id?.time != null && ctx.json.id.time != ''
@@ -538,4 +537,7 @@ on_failure:
  allow_duplicates: false
  - append:
  field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'

packages/google_workspace/data_stream/saml/elasticsearch/ingest_pipeline/default.yml

@@ -29,7 +29,6 @@ processors:
  - json:
  field: event.original
  target_field: json
- ignore_failure: true
  - date:
  field: json.id.time
  if: ctx.json?.id?.time != null && ctx.json.id.time != ''
@@ -199,4 +198,7 @@ on_failure:
  allow_duplicates: false
  - append:
  field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'