Update dimension fields for transaction log datastream to enable TSDB.

packages/microsoft_sqlserver/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.16.0"
+ changes:
+ - description: Update dimension fields for transaction log datastream to enable TSDB.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/5540
 - version: "1.15.0"
  changes:
  - description: Update metrictype for fields of performance data stream.

packages/microsoft_sqlserver/data_stream/transaction_log/elasticsearch/ingest_pipeline/default.yml

@@ -5,10 +5,6 @@ processors:
  field: sql.driver
  ignore_failure: true
  ignore_missing: true
-- remove:
- field: sql.query
- ignore_failure: true
- ignore_missing: true
 - rename:
  field: sql
  target_field: mssql
@@ -39,6 +35,15 @@ processors:
  target_field: mssql.metrics.log_since_last_log_backup
  ignore_missing: true
  ignore_failure: true
+- fingerprint:
+ fields: ["mssql.query"]
+ target_field: mssql.metrics.query_id
+ ignore_failure: true
+ ignore_missing: true
+- remove:
+ field: mssql.query
+ ignore_missing: true
+ ignore_failure: true
 - script:
  lang: painless
  source: ctx.mssql.metrics.log_since_last_checkpoint = Math.round(ctx.mssql.metrics.log_since_last_checkpoint * params.scale)

packages/microsoft_sqlserver/data_stream/transaction_log/fields/ecs.yml

@@ -2,5 +2,26 @@
  name: ecs.version
 - external: ecs
  name: service.address
+ dimension: true
 - external: ecs
  name: service.type
+- external: ecs
+ name: service.type
+- external: ecs
+ name: host.name
+ dimension: true
+- external: ecs
+ name: agent.id
+ dimension: true
+- external: ecs
+ name: cloud.project.id
+ dimension: true
+- external: ecs
+ name: cloud.instance.id
+ dimension: true
+- external: ecs
+ name: cloud.provider
+ dimension: true
+- external: ecs
+ name: container.id
+ dimension: true

packages/microsoft_sqlserver/data_stream/transaction_log/fields/fields.yml

@@ -7,17 +7,27 @@
  fields:
  - name: server_name
  type: keyword
+ # Reason to add as a dimension field: there can be more than one servers.
+ dimension: true
  description: Name of the mssql server.
  - name: instance_name
  type: keyword
+ # Reason to add as a dimension field: there can be multiple instances of sqlserver.
+ dimension: true
  description: Name of the mssql connected instance.
  - name: database_id
  type: long
- dimension: true
  description: Unique ID of the database inside MSSQL.
  - name: database_name
  type: keyword
+ # Reason to add as a dimension field: represents the database instance name of the report server database.
+ dimension: true
  description: Name of the database.
+ - name: query_id
+ type: keyword
+ # Reason to add as a dimension field: shows results based on queries.
+ dimension: true
+ description: Autogenerated ID representing the mssql query that is executed to fetch the results.
  - name: log_since_last_checkpoint
  type: long
  unit: byte

packages/microsoft_sqlserver/docs/README.md

@@ -584,10 +584,16 @@ An example event for `transaction_log` looks as following:
 | Field | Description | Type | Unit | Metric Type |
 |---|---|---|---|---|
 | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | |
+| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | | |
+| cloud.instance.id | Instance ID of the host machine. | keyword | | |
+| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | |
+| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | |
+| container.id | Unique container id. | keyword | | |
 | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | |
 | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | |
 | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | |
 | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | |
+| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | |
 | mssql.metrics.active_log_size | Total active transaction log size in bytes. | long | byte | counter |
 | mssql.metrics.database_id | Unique ID of the database inside MSSQL. | long | | |
 | mssql.metrics.database_name | Name of the database. | keyword | | |
@@ -597,6 +603,7 @@ An example event for `transaction_log` looks as following:
 | mssql.metrics.log_since_last_checkpoint | Log size in bytes since last checkpoint log sequence number (LSN). | long | byte | gauge |
 | mssql.metrics.log_since_last_log_backup | Log file size since last backup in bytes. | long | byte | gauge |
 | mssql.metrics.log_space_in_bytes_since_last_backup | The amount of space used since the last log backup in bytes. | long | byte | gauge |
+| mssql.metrics.query_id | Autogenerated ID representing the mssql query that is executed to fetch the results. | keyword | | |
 | mssql.metrics.server_name | Name of the mssql server. | keyword | | |
 | mssql.metrics.total_log_size | Total log size. | long | byte | counter |
 | mssql.metrics.total_log_size_bytes | Total transaction log size in bytes. | long | byte | counter |

packages/microsoft_sqlserver/kibana/dashboard/microsoft_sqlserver-18d66970-1fb4-11e9-8a4d-eb34d2834f6b.json

@@ -114,7 +114,7 @@
  {
  "field": "mssql.metrics.total_log_size_bytes",
  "id": "db45fa88-df7c-46ed-8a45-ef9e8f11a2c2",
- "type": "avg"
+ "type": "max"
  }
  ],
  "offset_time": "",
@@ -672,7 +672,7 @@
  {
  "field": "mssql.metrics.active_log_size",
  "id": "db45fa88-df7c-46ed-8a45-ef9e8f11a2c2",
- "type": "avg"
+ "type": "max"
  }
  ],
  "offset_time": "",

packages/microsoft_sqlserver/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: microsoft_sqlserver
 title: "Microsoft SQL Server"
-version: "1.15.0"
+version: "1.16.0"
 license: basic
 description: Collect events from Microsoft SQL Server with Elastic Agent
 type: integration