[AWS] Add lambda logs dataset

.github/CODEOWNERS

@@ -52,6 +52,7 @@
 /packages/aws/data_stream/guardduty @elastic/security-service-integrations
 /packages/aws/data_stream/kinesis @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/lambda @elastic/obs-infraobs-integrations
+/packages/aws/data_stream/lambda_logs @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/natgateway @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/rds @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/redshift @elastic/obs-infraobs-integrations

packages/aws/_dev/build/docs/lambda.md

@@ -10,7 +10,10 @@ For example, you could use this integration to track throttled lambda functions,
 
 ## Data streams
 
-The AWS Lambda integration collects one type of data: metrics.
+The AWS Lambda integration collects metrics and logs.
+
+**Logs** provide detailed information about the execution of AWS Lambda functions.
+They include invocation events, function output, error messages, stack traces, initialization logs, and AWS-generated reports. These logs help you troubleshoot issues, analyze performance, and monitor the behavior of your Lambda functions during runtime.
 
 **Metrics** give you insight into the state of AWS Lambda.
 Metrics collected by the AWS Lambda integration include the number of times your function code is executed, the amount of time that your function code spends processing an event, the number of invocations that result in a function error, and more.
@@ -54,6 +57,21 @@ When you configure the AWS integration, you can collect data from as many AWS se
 For step-by-step instructions on how to set up an integration, see the
 {{ url "getting-started-observability" "Getting started" }} guide.
 
+To enable AWS Lambda logs, ensure that your function's execution role includes the necessary permissions to write to Amazon CloudWatch Logs. Specifically, the role should have the following permissions: 
+- `logs:CreateLogGroup`
+- `logs:CreateLogStream`
+- `logs:PutLogEvents` 
+
+You can grant these permissions by attaching the AWS managed policy `AWSLambdaBasicExecutionRole` to your function's execution role.
+
+By default, AWS Lambda automatically streams logs to CloudWatch Logs. You can view these logs by navigating to the AWS Lambda console, selecting your function, and choosing the "Monitor" tab. From there, click on "View CloudWatch logs" to access the logs in the CloudWatch console.
+
+For more detailed information, refer to the AWS documentation on [Sending Lambda function logs to CloudWatch Logs](https://docs.aws.amazon.com/lambda/latest/dg/monitoring-cloudwatchlogs.html).
+
+## Logs reference
+
+{{event "lambda_logs"}}
+
 ## Metrics reference
 
 {{event "lambda"}}
@@ -62,4 +80,6 @@ For step-by-step instructions on how to set up an integration, see the
 
 Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
 
-{{fields "lambda"}}
+{{fields "lambda"}}
+
+{{fields "lambda_logs"}}

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.6.0"
+ changes:
+ - description: Add AWS lambda logs dataset.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/14070
 - version: "3.5.2"
  changes:
  - description: Add Redshift `InstanceType` dimension.

packages/aws/data_stream/lambda_logs/_dev/test/pipeline/test-awslambda-dotnet-json.log

@@ -0,0 +1,30 @@
+{
+ "timestamp": "2023-09-07T01:30:06.977Z",
+ "level": "Information",
+ "requestId": "8f711428-7e55-46f9-ae88-2a65d4f85fc5",
+ "traceId": "1-6408af34-50f56f5b5677a7d763973804",
+ "message": "{@users} have joined the group",
+ "users": 
+ [
+ {
+ "FirstName": "Alejandro",
+ "LastName": "Rosalez"
+ },
+ {
+ "FirstName": "John",
+ "LastName": "Stiles"
+ } 
+ ] 
+}
+{
+ "timestamp": "2023-09-07T01:30:06.977Z",
+ "level": "Information",
+ "requestId": "8f711428-7e55-46f9-ae88-2a65d4f85fc5",
+ "traceId": "1-6408af34-50f56f5b5677a7d763973804",
+ "message": "{users} have joined the group",
+ "users": 
+ [
+ "Rosalez, Alejandro",
+ "Stiles, John" 
+ ] 
+}

packages/aws/data_stream/lambda_logs/_dev/test/pipeline/test-awslambda-dotnet-json.log-expected.json

@@ -0,0 +1,74 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2025-03-20T11:07:58.223Z",
+ "aws": {
+ "lambda": {
+ "message": "{@users} have joined the group",
+ "request_id": "8f711428-7e55-46f9-ae88-2a65d4f85fc5",
+ "trace_id": "1-6408af34-50f56f5b5677a7d763973804",
+ "users": [
+ {
+ "FirstName": "Alejandro",
+ "LastName": "Rosalez"
+ },
+ {
+ "FirstName": "John",
+ "LastName": "Stiles"
+ }
+ ]
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "{\n \"timestamp\": \"2023-09-07T01:30:06.977Z\",\n \"level\": \"Information\",\n \"requestId\": \"8f711428-7e55-46f9-ae88-2a65d4f85fc5\",\n \"traceId\": \"1-6408af34-50f56f5b5677a7d763973804\",\n \"message\": \"{@users} have joined the group\",\n \"users\": \n [\n {\n \"FirstName\": \"Alejandro\",\n \"LastName\": \"Rosalez\"\n },\n {\n \"FirstName\": \"John\",\n \"LastName\": \"Stiles\"\n } \n ] \n}"
+ },
+ "log": {
+ "level": "Information"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2025-03-20T11:07:58.223Z",
+ "aws": {
+ "lambda": {
+ "message": "{users} have joined the group",
+ "request_id": "8f711428-7e55-46f9-ae88-2a65d4f85fc5",
+ "trace_id": "1-6408af34-50f56f5b5677a7d763973804",
+ "users": [
+ "Rosalez, Alejandro",
+ "Stiles, John"
+ ]
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "{\n \"timestamp\": \"2023-09-07T01:30:06.977Z\",\n \"level\": \"Information\",\n \"requestId\": \"8f711428-7e55-46f9-ae88-2a65d4f85fc5\",\n \"traceId\": \"1-6408af34-50f56f5b5677a7d763973804\",\n \"message\": \"{users} have joined the group\",\n \"users\": \n [\n \"Rosalez, Alejandro\",\n \"Stiles, John\" \n ] \n}"
+ },
+ "log": {
+ "level": "Information"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ }
+ ]
+}

packages/aws/data_stream/lambda_logs/_dev/test/pipeline/test-awslambda-dotnet-plain.log

@@ -0,0 +1,8 @@
+2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:27.988000 INIT_START Runtime Version: dotnet:6.v13 Runtime Version ARN: arn:aws:lambda:ap-southeast-2::runtime:699f346a05dae24c58c45790bc4089f252bf17dae3997e79b17d939a288aa1ec
+2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:28.229000 START RequestId: bed25b38-d012-42e7-ba28-f272535fb80e Version: $LATEST
+2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:29.259000 2023-02-20T14:15:29.201Z bed25b38-d012-42e7-ba28-f272535fb80e info {"_aws":{"Timestamp":1676902528962,"CloudWatchMetrics":[{"Namespace":"sam-app-logging","Metrics":[{"Name":"ColdStart","Unit":"Count"}],"Dimensions":[["FunctionName"],["Service"]]}]},"FunctionName":"sam-app-HelloWorldFunction-haKIoVeose2p","Service":"PowertoolsHelloWorld","ColdStart":1}
+2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.479000 2023-02-20T14:15:30.479Z bed25b38-d012-42e7-ba28-f272535fb80e info {"ColdStart":true,"XrayTraceId":"1-63f3807f-5dbcb9910c96f50742707542","CorrelationId":"d3d4de7f-4ccc-411a-a549-4d67b2fdc015","FunctionName":"sam-app-HelloWorldFunction-haKIoVeose2p","FunctionVersion":"$LATEST","FunctionMemorySize":256,"FunctionArn":"arn:aws:lambda:ap-southeast-2:123456789012:function:sam-app-HelloWorldFunction-haKIoVeose2p","FunctionRequestId":"bed25b38-d012-42e7-ba28-f272535fb80e","Timestamp":"2023-02-20T14:15:30.4602970Z","Level":"Information","Service":"PowertoolsHelloWorld","Name":"AWS.Lambda.Powertools.Logging.Logger","Message":"Hello world API - HTTP 200"}
+2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.599000 2023-02-20T14:15:30.599Z bed25b38-d012-42e7-ba28-f272535fb80e info {"_aws":{"Timestamp":1676902528922,"CloudWatchMetrics":[{"Namespace":"sam-app-logging","Metrics":[{"Name":"ApiRequestCount","Unit":"Count"}],"Dimensions":[["Service"]]}]},"Service":"PowertoolsHelloWorld","ApiRequestCount":1}
+2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.680000 END RequestId: bed25b38-d012-42e7-ba28-f272535fb80e
+2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.680000 REPORT RequestId: bed25b38-d012-42e7-ba28-f272535fb80e Duration: 2450.99 ms Billed Duration: 2451 ms Memory Size: 256 MB Max Memory Used: 74 MB Init Duration: 240.05 ms
+XRAY TraceId: 1-63f3807f-5dbcb9910c96f50742707542 SegmentId: 16b362cd5f52cba0

packages/aws/data_stream/lambda_logs/_dev/test/pipeline/test-awslambda-dotnet-plain.log-expected.json

@@ -0,0 +1,211 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2023-02-20T14:15:27.988Z",
+ "aws": {
+ "lambda": {
+ "event_type": "INIT_START",
+ "log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8",
+ "runtime_version": "dotnet:6.v13",
+ "runtime_version_arn": "arn:aws:lambda:ap-southeast-2::runtime:699f346a05dae24c58c45790bc4089f252bf17dae3997e79b17d939a288aa1ec"
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:27.988000 INIT_START Runtime Version: dotnet:6.v13 Runtime Version ARN: arn:aws:lambda:ap-southeast-2::runtime:699f346a05dae24c58c45790bc4089f252bf17dae3997e79b17d939a288aa1ec"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2023-02-20T14:15:28.229Z",
+ "aws": {
+ "lambda": {
+ "event_type": "START",
+ "log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8",
+ "request_id": "bed25b38-d012-42e7-ba28-f272535fb80e"
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:28.229000 START RequestId: bed25b38-d012-42e7-ba28-f272535fb80e Version: $LATEST"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2023-02-20T14:15:29.259Z",
+ "aws": {
+ "lambda": {
+ "log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8"
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:29.259000 2023-02-20T14:15:29.201Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"_aws\":{\"Timestamp\":1676902528962,\"CloudWatchMetrics\":[{\"Namespace\":\"sam-app-logging\",\"Metrics\":[{\"Name\":\"ColdStart\",\"Unit\":\"Count\"}],\"Dimensions\":[[\"FunctionName\"],[\"Service\"]]}]},\"FunctionName\":\"sam-app-HelloWorldFunction-haKIoVeose2p\",\"Service\":\"PowertoolsHelloWorld\",\"ColdStart\":1}"
+ },
+ "message": "2023-02-20T14:15:29.201Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"_aws\":{\"Timestamp\":1676902528962,\"CloudWatchMetrics\":[{\"Namespace\":\"sam-app-logging\",\"Metrics\":[{\"Name\":\"ColdStart\",\"Unit\":\"Count\"}],\"Dimensions\":[[\"FunctionName\"],[\"Service\"]]}]},\"FunctionName\":\"sam-app-HelloWorldFunction-haKIoVeose2p\",\"Service\":\"PowertoolsHelloWorld\",\"ColdStart\":1}",
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2023-02-20T14:15:30.479Z",
+ "aws": {
+ "lambda": {
+ "log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8"
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.479000 2023-02-20T14:15:30.479Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"ColdStart\":true,\"XrayTraceId\":\"1-63f3807f-5dbcb9910c96f50742707542\",\"CorrelationId\":\"d3d4de7f-4ccc-411a-a549-4d67b2fdc015\",\"FunctionName\":\"sam-app-HelloWorldFunction-haKIoVeose2p\",\"FunctionVersion\":\"$LATEST\",\"FunctionMemorySize\":256,\"FunctionArn\":\"arn:aws:lambda:ap-southeast-2:123456789012:function:sam-app-HelloWorldFunction-haKIoVeose2p\",\"FunctionRequestId\":\"bed25b38-d012-42e7-ba28-f272535fb80e\",\"Timestamp\":\"2023-02-20T14:15:30.4602970Z\",\"Level\":\"Information\",\"Service\":\"PowertoolsHelloWorld\",\"Name\":\"AWS.Lambda.Powertools.Logging.Logger\",\"Message\":\"Hello world API - HTTP 200\"}"
+ },
+ "message": "2023-02-20T14:15:30.479Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"ColdStart\":true,\"XrayTraceId\":\"1-63f3807f-5dbcb9910c96f50742707542\",\"CorrelationId\":\"d3d4de7f-4ccc-411a-a549-4d67b2fdc015\",\"FunctionName\":\"sam-app-HelloWorldFunction-haKIoVeose2p\",\"FunctionVersion\":\"$LATEST\",\"FunctionMemorySize\":256,\"FunctionArn\":\"arn:aws:lambda:ap-southeast-2:123456789012:function:sam-app-HelloWorldFunction-haKIoVeose2p\",\"FunctionRequestId\":\"bed25b38-d012-42e7-ba28-f272535fb80e\",\"Timestamp\":\"2023-02-20T14:15:30.4602970Z\",\"Level\":\"Information\",\"Service\":\"PowertoolsHelloWorld\",\"Name\":\"AWS.Lambda.Powertools.Logging.Logger\",\"Message\":\"Hello world API - HTTP 200\"}",
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2023-02-20T14:15:30.599Z",
+ "aws": {
+ "lambda": {
+ "log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8"
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.599000 2023-02-20T14:15:30.599Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"_aws\":{\"Timestamp\":1676902528922,\"CloudWatchMetrics\":[{\"Namespace\":\"sam-app-logging\",\"Metrics\":[{\"Name\":\"ApiRequestCount\",\"Unit\":\"Count\"}],\"Dimensions\":[[\"Service\"]]}]},\"Service\":\"PowertoolsHelloWorld\",\"ApiRequestCount\":1}"
+ },
+ "message": "2023-02-20T14:15:30.599Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"_aws\":{\"Timestamp\":1676902528922,\"CloudWatchMetrics\":[{\"Namespace\":\"sam-app-logging\",\"Metrics\":[{\"Name\":\"ApiRequestCount\",\"Unit\":\"Count\"}],\"Dimensions\":[[\"Service\"]]}]},\"Service\":\"PowertoolsHelloWorld\",\"ApiRequestCount\":1}",
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2023-02-20T14:15:30.680Z",
+ "aws": {
+ "lambda": {
+ "event_type": "END",
+ "log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8",
+ "request_id": "bed25b38-d012-42e7-ba28-f272535fb80e"
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.680000 END RequestId: bed25b38-d012-42e7-ba28-f272535fb80e"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2023-02-20T14:15:30.680Z",
+ "aws": {
+ "lambda": {
+ "event_type": "REPORT",
+ "log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8",
+ "metrics": {
+ "billed_duration_ms": 2451.0,
+ "duration_ms": 2450.99,
+ "init_duration_ms": 240.05,
+ "max_memory_used_mb": 74.0,
+ "memory_size_mb": 256.0
+ },
+ "request_id": "bed25b38-d012-42e7-ba28-f272535fb80e"
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.680000 REPORT RequestId: bed25b38-d012-42e7-ba28-f272535fb80e Duration: 2450.99 ms Billed Duration: 2451 ms Memory Size: 256 MB Max Memory Used: 74 MB Init Duration: 240.05 ms"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2025-03-20T11:07:58.223Z",
+ "aws": {
+ "lambda": {
+ "event_type": "XRAY",
+ "tracing": {
+ "segment_id": "16b362cd5f52cba0",
+ "xray_trace_id": "1-63f3807f-5dbcb9910c96f50742707542"
+ }
+ }
+ },
+ "cloud": {
+ "provider": "aws",
+ "service": {
+ "name": "aws_lambda"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "original": "XRAY TraceId: 1-63f3807f-5dbcb9910c96f50742707542 SegmentId: 16b362cd5f52cba0"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ }
+ ]
+}