[System] Add dimensions to system package metrics data_streams only, except core data_streams

packages/system/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.30.0"
+ changes:
+ - description: Add dimension fields to metrics all data_streams, except core, network and process to support TSDS migration
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/6118
 - version: "1.29.0"
  changes:
  - description: support ip or domain in sshd messages

packages/system/data_stream/cpu/fields/agent.yml

@@ -13,12 +13,14 @@
 
  Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
  example: 666777888999
+ dimension: true
  - name: availability_zone
  level: extended
  type: keyword
  ignore_above: 1024
  description: Availability zone in which this host is running.
  example: us-east-1c
+ dimension: true
  - name: instance.id
  level: extended
  type: keyword
@@ -50,10 +52,10 @@
  ignore_above: 1024
  description: Region in which this host is running.
  example: us-east-1
+ dimension: true
  - name: project.id
  type: keyword
  description: Name of the project in Google Cloud.
- dimension: true
  - name: image.id
  type: keyword
  description: Image ID for the cloud instance.
@@ -138,10 +140,10 @@
  level: core
  type: keyword
  ignore_above: 1024
- dimension: true
  description: 'Name of the host.
 
  It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'
+ dimension: true
  - name: os.family
  level: extended
  type: keyword
@@ -201,11 +203,3 @@
  description: >
  OS codename, if any.
 
-- name: agent
- title: Agent
- type: group
- fields:
- - name: id
- type: keyword
- ignore_above: 1024
- dimension: true

packages/system/data_stream/cpu/fields/ecs.yml

@@ -22,3 +22,6 @@
  name: host.os.version
 - external: ecs
  name: host.type
+- external: ecs
+ name: agent.id
+ dimension: true

packages/system/data_stream/diskio/fields/agent.yml

@@ -8,6 +8,7 @@
  - name: account.id
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
@@ -16,12 +17,14 @@
  - name: availability_zone
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Availability zone in which this host is running.
  example: us-east-1c
  - name: instance.id
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Instance ID of the host machine.
  example: i-1234567890abcdef0
@@ -39,12 +42,14 @@
  - name: provider
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
  example: aws
  - name: region
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Region in which this host is running.
  example: us-east-1
@@ -67,6 +72,7 @@
  type: keyword
  ignore_above: 1024
  description: Unique container id.
+ dimension: true
  - name: image.name
  level: extended
  type: keyword
@@ -133,6 +139,7 @@
  - name: name
  level: core
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: 'Name of the host.
 
@@ -149,17 +156,6 @@
  ignore_above: 1024
  description: Operating system kernel version as a raw string.
  example: 4.4.0-112-generic
- - name: os.name
- level: extended
- type: keyword
- ignore_above: 1024
- multi_fields:
- - name: text
- type: text
- norms: false
- default_field: false
- description: Operating system name, without the version.
- example: Mac OS X
  - name: os.platform
  level: extended
  type: keyword

packages/system/data_stream/diskio/fields/ecs.yml

@@ -24,3 +24,6 @@
  name: host.os.version
 - external: ecs
  name: host.type
+- external: ecs
+ name: agent.id
+ dimension: true

packages/system/data_stream/diskio/fields/fields.yml

@@ -3,6 +3,7 @@
  fields:
  - name: name
  type: keyword
+ dimension: true
  description: |
  The disk name.
  - name: serial_number

packages/system/data_stream/filesystem/fields/agent.yml

@@ -9,19 +9,22 @@
  level: extended
  type: keyword
  ignore_above: 1024
+ dimension: true
  description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
  Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
  example: 666777888999
  - name: availability_zone
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Availability zone in which this host is running.
  example: us-east-1c
  - name: instance.id
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Instance ID of the host machine.
  example: i-1234567890abcdef0
@@ -39,12 +42,14 @@
  - name: provider
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
  example: aws
  - name: region
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Region in which this host is running.
  example: us-east-1
@@ -66,6 +71,7 @@
  level: core
  type: keyword
  ignore_above: 1024
+ dimension: true
  description: Unique container id.
  - name: image.name
  level: extended
@@ -134,6 +140,7 @@
  level: core
  type: keyword
  ignore_above: 1024
+ dimension: true
  description: 'Name of the host.
 
  It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'

packages/system/data_stream/filesystem/fields/ecs.yml

@@ -0,0 +1,3 @@
+- external: ecs
+ name: agent.id
+ dimension: true

packages/system/data_stream/filesystem/fields/fields.yml

@@ -10,6 +10,7 @@
  The disk space available to an unprivileged user in bytes.
  - name: device_name
  type: keyword
+ dimension: true
  description: |
  The disk name. For example: `/dev/disk1`
  - name: type
@@ -18,6 +19,7 @@
  The disk type. For example: `ext4`
  - name: mount_point
  type: keyword
+ dimension: true
  description: |
  The mounting point. For example: `/`
  - name: files

packages/system/data_stream/fsstat/fields/agent.yml

@@ -8,6 +8,7 @@
  - name: account.id
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
@@ -16,12 +17,14 @@
  - name: availability_zone
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Availability zone in which this host is running.
  example: us-east-1c
  - name: instance.id
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Instance ID of the host machine.
  example: i-1234567890abcdef0
@@ -39,12 +42,14 @@
  - name: provider
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
  example: aws
  - name: region
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Region in which this host is running.
  example: us-east-1
@@ -65,6 +70,7 @@
  - name: id
  level: core
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Unique container id.
  - name: image.name
@@ -133,6 +139,7 @@
  - name: name
  level: core
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: 'Name of the host.
 

packages/system/data_stream/fsstat/fields/ecs.yml

@@ -22,3 +22,6 @@
  name: host.os.version
 - external: ecs
  name: host.type
+- external: ecs
+ name: agent.id
+ dimension: true

packages/system/data_stream/load/fields/agent.yml

@@ -8,6 +8,7 @@
  - name: account.id
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
@@ -16,6 +17,7 @@
  - name: availability_zone
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Availability zone in which this host is running.
  example: us-east-1c
@@ -47,13 +49,13 @@
  - name: region
  level: extended
  type: keyword
+ dimension: true
  ignore_above: 1024
  description: Region in which this host is running.
  example: us-east-1
  - name: project.id
  type: keyword
  description: Name of the project in Google Cloud.
- dimension: true
  - name: image.id
  type: keyword
  description: Image ID for the cloud instance.
@@ -154,17 +156,6 @@
  ignore_above: 1024
  description: Operating system kernel version as a raw string.
  example: 4.4.0-112-generic
- - name: os.name
- level: extended
- type: keyword
- ignore_above: 1024
- multi_fields:
- - name: text
- type: text
- norms: false
- default_field: false
- description: Operating system name, without the version.
- example: Mac OS X
  - name: os.platform
  level: extended
  type: keyword
@@ -201,11 +192,3 @@
  description: >
  OS codename, if any.
 
-- name: agent
- title: Agent
- type: group
- fields:
- - name: id
- type: keyword
- ignore_above: 1024
- dimension: true

packages/system/data_stream/load/fields/ecs.yml

@@ -22,3 +22,6 @@
  name: host.os.version
 - external: ecs
  name: host.type
+- external: ecs
+ name: agent.id
+ dimension: true

packages/system/data_stream/memory/fields/agent.yml

@@ -13,12 +13,14 @@
 
  Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
  example: 666777888999
+ dimension: true
  - name: availability_zone
  level: extended
  type: keyword
  ignore_above: 1024
  description: Availability zone in which this host is running.
  example: us-east-1c
+ dimension: true
  - name: instance.id
  level: extended
  type: keyword
@@ -50,10 +52,10 @@
  ignore_above: 1024
  description: Region in which this host is running.
  example: us-east-1
+ dimension: true
  - name: project.id
  type: keyword
  description: Name of the project in Google Cloud.
- dimension: true
  - name: image.id
  type: keyword
  description: Image ID for the cloud instance.
@@ -201,11 +203,3 @@
  description: >
  OS codename, if any.
 
-- name: agent
- title: Agent
- type: group
- fields:
- - name: id
- type: keyword
- ignore_above: 1024
- dimension: true