Skip to content

Commit 31e8b30

Browse files
agithomasagithomas
authored
[AWS] Add lambda logs dataset (#14070)
* Add AWS Lambda logs dataset
1 parent 412ecef commit 31e8b30

39 files changed

+4714
-5
lines changed

‎.github/CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
/packages/aws/data_stream/guardduty @elastic/security-service-integrations
5353
/packages/aws/data_stream/kinesis @elastic/obs-infraobs-integrations
5454
/packages/aws/data_stream/lambda @elastic/obs-infraobs-integrations
55+
/packages/aws/data_stream/lambda_logs @elastic/obs-infraobs-integrations
5556
/packages/aws/data_stream/natgateway @elastic/obs-infraobs-integrations
5657
/packages/aws/data_stream/rds @elastic/obs-infraobs-integrations
5758
/packages/aws/data_stream/redshift @elastic/obs-infraobs-integrations

‎packages/aws/_dev/build/docs/lambda.md‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@ For example, you could use this integration to track throttled lambda functions,
1010

1111
## Data streams
1212

13-
The AWS Lambda integration collects one type of data: metrics.
13+
The AWS Lambda integration collects metrics and logs.
14+
15+
**Logs** provide detailed information about the execution of AWS Lambda functions.
16+
They include invocation events, function output, error messages, stack traces, initialization logs, and AWS-generated reports. These logs help you troubleshoot issues, analyze performance, and monitor the behavior of your Lambda functions during runtime.
1417

1518
**Metrics** give you insight into the state of AWS Lambda.
1619
Metrics collected by the AWS Lambda integration include the number of times your function code is executed, the amount of time that your function code spends processing an event, the number of invocations that result in a function error, and more.
@@ -54,6 +57,21 @@ When you configure the AWS integration, you can collect data from as many AWS se
5457
For step-by-step instructions on how to set up an integration, see the
5558
{{ url "getting-started-observability" "Getting started" }} guide.
5659

60+
To enable AWS Lambda logs, ensure that your function's execution role includes the necessary permissions to write to Amazon CloudWatch Logs. Specifically, the role should have the following permissions:
61+
- `logs:CreateLogGroup`
62+
- `logs:CreateLogStream`
63+
- `logs:PutLogEvents`
64+
65+
You can grant these permissions by attaching the AWS managed policy `AWSLambdaBasicExecutionRole` to your function's execution role.
66+
67+
By default, AWS Lambda automatically streams logs to CloudWatch Logs. You can view these logs by navigating to the AWS Lambda console, selecting your function, and choosing the "Monitor" tab. From there, click on "View CloudWatch logs" to access the logs in the CloudWatch console.
68+
69+
For more detailed information, refer to the AWS documentation on [Sending Lambda function logs to CloudWatch Logs](https://docs.aws.amazon.com/lambda/latest/dg/monitoring-cloudwatchlogs.html).
70+
71+
## Logs reference
72+
73+
{{event "lambda_logs"}}
74+
5775
## Metrics reference
5876

5977
{{event "lambda"}}
@@ -62,4 +80,6 @@ For step-by-step instructions on how to set up an integration, see the
6280

6381
Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
6482

65-
{{fields "lambda"}}
83+
{{fields "lambda"}}
84+
85+
{{fields "lambda_logs"}}

‎packages/aws/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "3.6.0"
3+
changes:
4+
- description: Add AWS lambda logs dataset.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/14070
27
- version: "3.5.2"
38
changes:
49
- description: Add Redshift `InstanceType` dimension.

‎packages/aws/data_stream/lambda_logs/_dev/test/pipeline/test-awslambda-dotnet-json.log‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"timestamp": "2023-09-07T01:30:06.977Z",
3+
"level": "Information",
4+
"requestId": "8f711428-7e55-46f9-ae88-2a65d4f85fc5",
5+
"traceId": "1-6408af34-50f56f5b5677a7d763973804",
6+
"message": "{@users} have joined the group",
7+
"users":
8+
[
9+
{
10+
"FirstName": "Alejandro",
11+
"LastName": "Rosalez"
12+
},
13+
{
14+
"FirstName": "John",
15+
"LastName": "Stiles"
16+
}
17+
]
18+
}
19+
{
20+
"timestamp": "2023-09-07T01:30:06.977Z",
21+
"level": "Information",
22+
"requestId": "8f711428-7e55-46f9-ae88-2a65d4f85fc5",
23+
"traceId": "1-6408af34-50f56f5b5677a7d763973804",
24+
"message": "{users} have joined the group",
25+
"users":
26+
[
27+
"Rosalez, Alejandro",
28+
"Stiles, John"
29+
]
30+
}

‎packages/aws/data_stream/lambda_logs/_dev/test/pipeline/test-awslambda-dotnet-json.log-expected.json‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
{
2+
"expected": [
3+
{
4+
"@timestamp": "2025-03-20T11:07:58.223Z",
5+
"aws": {
6+
"lambda": {
7+
"message": "{@users} have joined the group",
8+
"request_id": "8f711428-7e55-46f9-ae88-2a65d4f85fc5",
9+
"trace_id": "1-6408af34-50f56f5b5677a7d763973804",
10+
"users": [
11+
{
12+
"FirstName": "Alejandro",
13+
"LastName": "Rosalez"
14+
},
15+
{
16+
"FirstName": "John",
17+
"LastName": "Stiles"
18+
}
19+
]
20+
}
21+
},
22+
"cloud": {
23+
"provider": "aws",
24+
"service": {
25+
"name": "aws_lambda"
26+
}
27+
},
28+
"ecs": {
29+
"version": "8.11.0"
30+
},
31+
"event": {
32+
"original": "{\n \"timestamp\": \"2023-09-07T01:30:06.977Z\",\n \"level\": \"Information\",\n \"requestId\": \"8f711428-7e55-46f9-ae88-2a65d4f85fc5\",\n \"traceId\": \"1-6408af34-50f56f5b5677a7d763973804\",\n \"message\": \"{@users} have joined the group\",\n \"users\": \n [\n {\n \"FirstName\": \"Alejandro\",\n \"LastName\": \"Rosalez\"\n },\n {\n \"FirstName\": \"John\",\n \"LastName\": \"Stiles\"\n } \n ] \n}"
33+
},
34+
"log": {
35+
"level": "Information"
36+
},
37+
"tags": [
38+
"preserve_original_event"
39+
]
40+
},
41+
{
42+
"@timestamp": "2025-03-20T11:07:58.223Z",
43+
"aws": {
44+
"lambda": {
45+
"message": "{users} have joined the group",
46+
"request_id": "8f711428-7e55-46f9-ae88-2a65d4f85fc5",
47+
"trace_id": "1-6408af34-50f56f5b5677a7d763973804",
48+
"users": [
49+
"Rosalez, Alejandro",
50+
"Stiles, John"
51+
]
52+
}
53+
},
54+
"cloud": {
55+
"provider": "aws",
56+
"service": {
57+
"name": "aws_lambda"
58+
}
59+
},
60+
"ecs": {
61+
"version": "8.11.0"
62+
},
63+
"event": {
64+
"original": "{\n \"timestamp\": \"2023-09-07T01:30:06.977Z\",\n \"level\": \"Information\",\n \"requestId\": \"8f711428-7e55-46f9-ae88-2a65d4f85fc5\",\n \"traceId\": \"1-6408af34-50f56f5b5677a7d763973804\",\n \"message\": \"{users} have joined the group\",\n \"users\": \n [\n \"Rosalez, Alejandro\",\n \"Stiles, John\" \n ] \n}"
65+
},
66+
"log": {
67+
"level": "Information"
68+
},
69+
"tags": [
70+
"preserve_original_event"
71+
]
72+
}
73+
]
74+
}

‎packages/aws/data_stream/lambda_logs/_dev/test/pipeline/test-awslambda-dotnet-plain.log‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:27.988000 INIT_START Runtime Version: dotnet:6.v13 Runtime Version ARN: arn:aws:lambda:ap-southeast-2::runtime:699f346a05dae24c58c45790bc4089f252bf17dae3997e79b17d939a288aa1ec
2+
2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:28.229000 START RequestId: bed25b38-d012-42e7-ba28-f272535fb80e Version: $LATEST
3+
2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:29.259000 2023-02-20T14:15:29.201Z bed25b38-d012-42e7-ba28-f272535fb80e info {"_aws":{"Timestamp":1676902528962,"CloudWatchMetrics":[{"Namespace":"sam-app-logging","Metrics":[{"Name":"ColdStart","Unit":"Count"}],"Dimensions":[["FunctionName"],["Service"]]}]},"FunctionName":"sam-app-HelloWorldFunction-haKIoVeose2p","Service":"PowertoolsHelloWorld","ColdStart":1}
4+
2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.479000 2023-02-20T14:15:30.479Z bed25b38-d012-42e7-ba28-f272535fb80e info {"ColdStart":true,"XrayTraceId":"1-63f3807f-5dbcb9910c96f50742707542","CorrelationId":"d3d4de7f-4ccc-411a-a549-4d67b2fdc015","FunctionName":"sam-app-HelloWorldFunction-haKIoVeose2p","FunctionVersion":"$LATEST","FunctionMemorySize":256,"FunctionArn":"arn:aws:lambda:ap-southeast-2:123456789012:function:sam-app-HelloWorldFunction-haKIoVeose2p","FunctionRequestId":"bed25b38-d012-42e7-ba28-f272535fb80e","Timestamp":"2023-02-20T14:15:30.4602970Z","Level":"Information","Service":"PowertoolsHelloWorld","Name":"AWS.Lambda.Powertools.Logging.Logger","Message":"Hello world API - HTTP 200"}
5+
2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.599000 2023-02-20T14:15:30.599Z bed25b38-d012-42e7-ba28-f272535fb80e info {"_aws":{"Timestamp":1676902528922,"CloudWatchMetrics":[{"Namespace":"sam-app-logging","Metrics":[{"Name":"ApiRequestCount","Unit":"Count"}],"Dimensions":[["Service"]]}]},"Service":"PowertoolsHelloWorld","ApiRequestCount":1}
6+
2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.680000 END RequestId: bed25b38-d012-42e7-ba28-f272535fb80e
7+
2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.680000 REPORT RequestId: bed25b38-d012-42e7-ba28-f272535fb80e Duration: 2450.99 ms Billed Duration: 2451 ms Memory Size: 256 MB Max Memory Used: 74 MB Init Duration: 240.05 ms
8+
XRAY TraceId: 1-63f3807f-5dbcb9910c96f50742707542 SegmentId: 16b362cd5f52cba0

‎packages/aws/data_stream/lambda_logs/_dev/test/pipeline/test-awslambda-dotnet-plain.log-expected.json‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
{
2+
"expected": [
3+
{
4+
"@timestamp": "2023-02-20T14:15:27.988Z",
5+
"aws": {
6+
"lambda": {
7+
"event_type": "INIT_START",
8+
"log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8",
9+
"runtime_version": "dotnet:6.v13",
10+
"runtime_version_arn": "arn:aws:lambda:ap-southeast-2::runtime:699f346a05dae24c58c45790bc4089f252bf17dae3997e79b17d939a288aa1ec"
11+
}
12+
},
13+
"cloud": {
14+
"provider": "aws",
15+
"service": {
16+
"name": "aws_lambda"
17+
}
18+
},
19+
"ecs": {
20+
"version": "8.11.0"
21+
},
22+
"event": {
23+
"original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:27.988000 INIT_START Runtime Version: dotnet:6.v13 Runtime Version ARN: arn:aws:lambda:ap-southeast-2::runtime:699f346a05dae24c58c45790bc4089f252bf17dae3997e79b17d939a288aa1ec"
24+
},
25+
"tags": [
26+
"preserve_original_event"
27+
]
28+
},
29+
{
30+
"@timestamp": "2023-02-20T14:15:28.229Z",
31+
"aws": {
32+
"lambda": {
33+
"event_type": "START",
34+
"log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8",
35+
"request_id": "bed25b38-d012-42e7-ba28-f272535fb80e"
36+
}
37+
},
38+
"cloud": {
39+
"provider": "aws",
40+
"service": {
41+
"name": "aws_lambda"
42+
}
43+
},
44+
"ecs": {
45+
"version": "8.11.0"
46+
},
47+
"event": {
48+
"original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:28.229000 START RequestId: bed25b38-d012-42e7-ba28-f272535fb80e Version: $LATEST"
49+
},
50+
"tags": [
51+
"preserve_original_event"
52+
]
53+
},
54+
{
55+
"@timestamp": "2023-02-20T14:15:29.259Z",
56+
"aws": {
57+
"lambda": {
58+
"log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8"
59+
}
60+
},
61+
"cloud": {
62+
"provider": "aws",
63+
"service": {
64+
"name": "aws_lambda"
65+
}
66+
},
67+
"ecs": {
68+
"version": "8.11.0"
69+
},
70+
"event": {
71+
"original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:29.259000 2023-02-20T14:15:29.201Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"_aws\":{\"Timestamp\":1676902528962,\"CloudWatchMetrics\":[{\"Namespace\":\"sam-app-logging\",\"Metrics\":[{\"Name\":\"ColdStart\",\"Unit\":\"Count\"}],\"Dimensions\":[[\"FunctionName\"],[\"Service\"]]}]},\"FunctionName\":\"sam-app-HelloWorldFunction-haKIoVeose2p\",\"Service\":\"PowertoolsHelloWorld\",\"ColdStart\":1}"
72+
},
73+
"message": "2023-02-20T14:15:29.201Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"_aws\":{\"Timestamp\":1676902528962,\"CloudWatchMetrics\":[{\"Namespace\":\"sam-app-logging\",\"Metrics\":[{\"Name\":\"ColdStart\",\"Unit\":\"Count\"}],\"Dimensions\":[[\"FunctionName\"],[\"Service\"]]}]},\"FunctionName\":\"sam-app-HelloWorldFunction-haKIoVeose2p\",\"Service\":\"PowertoolsHelloWorld\",\"ColdStart\":1}",
74+
"tags": [
75+
"preserve_original_event"
76+
]
77+
},
78+
{
79+
"@timestamp": "2023-02-20T14:15:30.479Z",
80+
"aws": {
81+
"lambda": {
82+
"log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8"
83+
}
84+
},
85+
"cloud": {
86+
"provider": "aws",
87+
"service": {
88+
"name": "aws_lambda"
89+
}
90+
},
91+
"ecs": {
92+
"version": "8.11.0"
93+
},
94+
"event": {
95+
"original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.479000 2023-02-20T14:15:30.479Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"ColdStart\":true,\"XrayTraceId\":\"1-63f3807f-5dbcb9910c96f50742707542\",\"CorrelationId\":\"d3d4de7f-4ccc-411a-a549-4d67b2fdc015\",\"FunctionName\":\"sam-app-HelloWorldFunction-haKIoVeose2p\",\"FunctionVersion\":\"$LATEST\",\"FunctionMemorySize\":256,\"FunctionArn\":\"arn:aws:lambda:ap-southeast-2:123456789012:function:sam-app-HelloWorldFunction-haKIoVeose2p\",\"FunctionRequestId\":\"bed25b38-d012-42e7-ba28-f272535fb80e\",\"Timestamp\":\"2023-02-20T14:15:30.4602970Z\",\"Level\":\"Information\",\"Service\":\"PowertoolsHelloWorld\",\"Name\":\"AWS.Lambda.Powertools.Logging.Logger\",\"Message\":\"Hello world API - HTTP 200\"}"
96+
},
97+
"message": "2023-02-20T14:15:30.479Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"ColdStart\":true,\"XrayTraceId\":\"1-63f3807f-5dbcb9910c96f50742707542\",\"CorrelationId\":\"d3d4de7f-4ccc-411a-a549-4d67b2fdc015\",\"FunctionName\":\"sam-app-HelloWorldFunction-haKIoVeose2p\",\"FunctionVersion\":\"$LATEST\",\"FunctionMemorySize\":256,\"FunctionArn\":\"arn:aws:lambda:ap-southeast-2:123456789012:function:sam-app-HelloWorldFunction-haKIoVeose2p\",\"FunctionRequestId\":\"bed25b38-d012-42e7-ba28-f272535fb80e\",\"Timestamp\":\"2023-02-20T14:15:30.4602970Z\",\"Level\":\"Information\",\"Service\":\"PowertoolsHelloWorld\",\"Name\":\"AWS.Lambda.Powertools.Logging.Logger\",\"Message\":\"Hello world API - HTTP 200\"}",
98+
"tags": [
99+
"preserve_original_event"
100+
]
101+
},
102+
{
103+
"@timestamp": "2023-02-20T14:15:30.599Z",
104+
"aws": {
105+
"lambda": {
106+
"log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8"
107+
}
108+
},
109+
"cloud": {
110+
"provider": "aws",
111+
"service": {
112+
"name": "aws_lambda"
113+
}
114+
},
115+
"ecs": {
116+
"version": "8.11.0"
117+
},
118+
"event": {
119+
"original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.599000 2023-02-20T14:15:30.599Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"_aws\":{\"Timestamp\":1676902528922,\"CloudWatchMetrics\":[{\"Namespace\":\"sam-app-logging\",\"Metrics\":[{\"Name\":\"ApiRequestCount\",\"Unit\":\"Count\"}],\"Dimensions\":[[\"Service\"]]}]},\"Service\":\"PowertoolsHelloWorld\",\"ApiRequestCount\":1}"
120+
},
121+
"message": "2023-02-20T14:15:30.599Z bed25b38-d012-42e7-ba28-f272535fb80e info {\"_aws\":{\"Timestamp\":1676902528922,\"CloudWatchMetrics\":[{\"Namespace\":\"sam-app-logging\",\"Metrics\":[{\"Name\":\"ApiRequestCount\",\"Unit\":\"Count\"}],\"Dimensions\":[[\"Service\"]]}]},\"Service\":\"PowertoolsHelloWorld\",\"ApiRequestCount\":1}",
122+
"tags": [
123+
"preserve_original_event"
124+
]
125+
},
126+
{
127+
"@timestamp": "2023-02-20T14:15:30.680Z",
128+
"aws": {
129+
"lambda": {
130+
"event_type": "END",
131+
"log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8",
132+
"request_id": "bed25b38-d012-42e7-ba28-f272535fb80e"
133+
}
134+
},
135+
"cloud": {
136+
"provider": "aws",
137+
"service": {
138+
"name": "aws_lambda"
139+
}
140+
},
141+
"ecs": {
142+
"version": "8.11.0"
143+
},
144+
"event": {
145+
"original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.680000 END RequestId: bed25b38-d012-42e7-ba28-f272535fb80e"
146+
},
147+
"tags": [
148+
"preserve_original_event"
149+
]
150+
},
151+
{
152+
"@timestamp": "2023-02-20T14:15:30.680Z",
153+
"aws": {
154+
"lambda": {
155+
"event_type": "REPORT",
156+
"log_stream_id": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8",
157+
"metrics": {
158+
"billed_duration_ms": 2451.0,
159+
"duration_ms": 2450.99,
160+
"init_duration_ms": 240.05,
161+
"max_memory_used_mb": 74.0,
162+
"memory_size_mb": 256.0
163+
},
164+
"request_id": "bed25b38-d012-42e7-ba28-f272535fb80e"
165+
}
166+
},
167+
"cloud": {
168+
"provider": "aws",
169+
"service": {
170+
"name": "aws_lambda"
171+
}
172+
},
173+
"ecs": {
174+
"version": "8.11.0"
175+
},
176+
"event": {
177+
"original": "2023/02/20/[$LATEST]4eaf8445ba7a4a93b999cb17fbfbecd8 2023-02-20T14:15:30.680000 REPORT RequestId: bed25b38-d012-42e7-ba28-f272535fb80e Duration: 2450.99 ms Billed Duration: 2451 ms Memory Size: 256 MB Max Memory Used: 74 MB Init Duration: 240.05 ms"
178+
},
179+
"tags": [
180+
"preserve_original_event"
181+
]
182+
},
183+
{
184+
"@timestamp": "2025-03-20T11:07:58.223Z",
185+
"aws": {
186+
"lambda": {
187+
"event_type": "XRAY",
188+
"tracing": {
189+
"segment_id": "16b362cd5f52cba0",
190+
"xray_trace_id": "1-63f3807f-5dbcb9910c96f50742707542"
191+
}
192+
}
193+
},
194+
"cloud": {
195+
"provider": "aws",
196+
"service": {
197+
"name": "aws_lambda"
198+
}
199+
},
200+
"ecs": {
201+
"version": "8.11.0"
202+
},
203+
"event": {
204+
"original": "XRAY TraceId: 1-63f3807f-5dbcb9910c96f50742707542 SegmentId: 16b362cd5f52cba0"
205+
},
206+
"tags": [
207+
"preserve_original_event"
208+
]
209+
}
210+
]
211+
}

0 commit comments

Comments
 (0)