Fix error handling for machine data stream and ensure parameter is correctly formatted in vulnerability data stream

Author: brijesh-elastic

packages/microsoft_defender_endpoint/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "3.1.1"
+ changes:
+ - description: Ensure large `$skip` API parameter values are correctly formatted in `vulnerability` data stream.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/1
+ - description: 'Enhanced error handling in the CEL program for API calls to prevent "no such key: page_size" errors in the machine data stream.'
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/1
 - version: "3.1.0"
  changes:
  - description: Add `vulnerability_workflow` sub category label.

packages/microsoft_defender_endpoint/data_stream/machine/agent/stream/cel.yml.hbs

@@ -49,6 +49,7 @@ program: |
  ),
  },
  },
+ "page_size": state.page_size,
  "want_more": false,
  }
  )

packages/microsoft_defender_endpoint/data_stream/vulnerability/agent/stream/cel.yml.hbs

@@ -60,7 +60,7 @@ program: |-
  "GET",
  state.url.trim_right("/") + "/api/vulnerabilities/machinesVulnerabilities?" + {
  "$top": [string(state.config.product_batch_size)],
- "$skip": [string(state.product_skip)],
+ "$skip": [string(int(state.product_skip))],
  }.format_query()
  ).do_request().as(productResp, (productResp.StatusCode == 200) ?
  productResp.Body.decode_json().as(productBody,
@@ -119,7 +119,7 @@ program: |-
  "GET",
  state.url.trim_right("/") + "/api/machines?" + {
  "$top": [string(state.config.machine_batch_size)],
- "$skip": [string(res.machine_skip)],
+ "$skip": [string(int(res.machine_skip))],
  }.format_query()
  ).do_request().as(machineResp, (machineResp.StatusCode == 200) ?
  machineResp.Body.decode_json().as(machineBody,
@@ -182,7 +182,7 @@ program: |-
  "GET",
  state.url.trim_right("/") + "/api/vulnerabilities?" + {
  "$top": [string(state.config.vulnerabilities_batch_size)],
- "$skip": [string(res.vulnerability_skip)],
+ "$skip": [string(int(res.vulnerability_skip))],
  }.format_query()
  ).do_request().as(vulnerabilityResp, (vulnerabilityResp.StatusCode == 200) ?
  vulnerabilityResp.Body.decode_json().as(vulnerabilityBody,

packages/microsoft_defender_endpoint/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.4.0"
 name: microsoft_defender_endpoint
 title: Microsoft Defender for Endpoint
-version: "3.1.0"
+version: "3.1.1"
 description: Collect logs from Microsoft Defender for Endpoint with Elastic Agent.
 categories:
  - security