Skip to content

Commit 0aa6b3e

Browse files
CzechhCzechh
authored
Add more integration tests (#74)
* fix flawkyness * add errors too * add cond decorator * add conditional decorator and update snapshots * update conditions * add serverless plugin config and update integration tests * add latest snapshots * attempt to fix formatting * fix format * add note for external dependency
1 parent a1139b4 commit 0aa6b3e

File tree

54 files changed

+1044
-186
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1044
-186
lines changed

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,4 +38,7 @@ nosetests.xml
3838
.idea/
3939

4040

41-
**/.serverless/
41+
**/.serverless/
42+
43+
node_modules
44+
yarn-lock.json

‎scripts/run_integration_tests.sh‎

Lines changed: 111 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@ set -e
1010

1111
# These values need to be in sync with serverless.yml, where there needs to be a function
1212
# defined for every handler_runtime combination
13-
LAMBDA_HANDLERS=("async-metrics" "sync-metrics" "http-requests")
13+
LAMBDA_HANDLERS=("async-metrics" "sync-metrics" "http-requests" "http-error")
1414
RUNTIMES=("python27" "python36" "python37" "python38")
15+
CONFIGS=("with-plugin" "without-plugin")
1516

1617
LOGS_WAIT_SECONDS=20
1718

@@ -45,115 +46,139 @@ input_event_files=$(ls ./input_events)
4546
# Sort event files by name so that snapshots stay consistent
4647
input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_name; done | sort))
4748

48-
echo "Deploying functions"
49+
echo "Deploying functions with plugin"
50+
serverless deploy -c "./serverless-plugin.yml"
51+
echo "Deploying functions without plugin"
4952
serverless deploy
5053

5154
echo "Invoking functions"
5255
set +e # Don't exit this script if an invocation fails or there's a diff
53-
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
54-
for runtime in "${RUNTIMES[@]}"; do
55-
function_name="${handler_name}_${runtime}"
56-
# Invoke function once for each input event
57-
for input_event_file in "${input_event_files[@]}"; do
58-
# Get event name without trailing ".json" so we can build the snapshot file name
59-
input_event_name=$(echo "$input_event_file" | sed "s/.json//")
60-
# Return value snapshot file format is snapshots/return_values/{handler}_{runtime}_{input-event}
61-
snapshot_path="./snapshots/return_values/${function_name}_${input_event_name}.json"
62-
63-
return_value=$(serverless invoke -f $function_name --path "./input_events/$input_event_file")
64-
65-
if [ ! -f $snapshot_path ]; then
66-
# If the snapshot file doesn't exist yet, we create it
67-
echo "Writing return value to $snapshot_path because no snapshot exists yet"
68-
echo "$return_value" >$snapshot_path
69-
elif [ -n "$UPDATE_SNAPSHOTS" ]; then
70-
# If $UPDATE_SNAPSHOTS is set to true, write the new logs over the current snapshot
71-
echo "Overwriting return value snapshot for $snapshot_path"
72-
echo "$return_value" >$snapshot_path
56+
for _sls_type in "${CONFIGS[@]}"; do
57+
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
58+
for runtime in "${RUNTIMES[@]}"; do
59+
if [ "$_sls_type" = "with-plugin" ]; then
60+
function_name="${handler_name}_${runtime}_with_plugin"
7361
else
74-
# Compare new return value to snapshot
75-
diff_output=$(echo "$return_value" | diff - $snapshot_path)
76-
if [ $? -eq 1 ]; then
77-
echo "Failed: Return value for $function_name does not match snapshot:"
78-
echo "$diff_output"
79-
mismatch_found=true
62+
function_name="${handler_name}_${runtime}"
63+
fi
64+
65+
# Invoke function once for each input event
66+
for input_event_file in "${input_event_files[@]}"; do
67+
# Get event name without trailing ".json" so we can build the snapshot file name
68+
input_event_name=$(echo "$input_event_file" | sed "s/.json//")
69+
# Return value snapshot file format is snapshots/return_values/{handler}_{runtime}_{input-event}
70+
snapshot_path="./snapshots/return_values/${handler_name}_${runtime}_${input_event_name}.json"
71+
72+
if [ "$_sls_type" = "with-plugin" ]; then
73+
return_value=$(serverless invoke -f $function_name --path "./input_events/$input_event_file" -c "serverless-plugin.yml")
8074
else
81-
echo "Ok: Return value for $function_name with $input_event_name event matches snapshot"
75+
return_value=$(serverless invoke -f $function_name --path "./input_events/$input_event_file")
8276
fi
83-
fi
84-
done
8577

78+
if [ ! -f $snapshot_path ]; then
79+
# If the snapshot file doesn't exist yet, we create it
80+
echo "Writing return value to $snapshot_path because no snapshot exists yet"
81+
echo "$return_value" >$snapshot_path
82+
elif [ -n "$UPDATE_SNAPSHOTS" ]; then
83+
# If $UPDATE_SNAPSHOTS is set to true, write the new logs over the current snapshot
84+
echo "Overwriting return value snapshot for $snapshot_path"
85+
echo "$return_value" >$snapshot_path
86+
else
87+
# Compare new return value to snapshot
88+
diff_output=$(echo "$return_value" | diff - $snapshot_path)
89+
if [ $? -eq 1 ]; then
90+
echo "Failed: Return value for $function_name does not match snapshot:"
91+
echo "$diff_output"
92+
mismatch_found=true
93+
else
94+
echo "Ok: Return value for $function_name with $input_event_name event matches snapshot"
95+
fi
96+
fi
97+
done
98+
done
8699
done
87-
88100
done
89101
set -e
90102

91103
echo "Sleeping $LOGS_WAIT_SECONDS seconds to wait for logs to appear in CloudWatch..."
92104
sleep $LOGS_WAIT_SECONDS
93105

94106
echo "Fetching logs for invocations and comparing to snapshots"
95-
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
96-
for runtime in "${RUNTIMES[@]}"; do
97-
function_name="${handler_name}_${runtime}"
98-
function_snapshot_path="./snapshots/logs/$function_name.log"
99-
100-
# Fetch logs with serverless cli
101-
raw_logs=$(serverless logs -f $function_name --startTime $script_start_time)
102-
103-
# Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions
104-
logs=$(
105-
echo "$raw_logs" |
106-
# Filter serverless cli errors
107-
sed '/Serverless: Recoverable error occurred/d' |
108-
# Remove blank lines
109-
sed '/^$/d' |
110-
# Normalize Lambda runtime report logs
111-
sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"): [a-z0-9\.\-]+/\1: XXXX/g' |
112-
# Normalize DD APM headers and AWS account ID
113-
sed -E "s/(x-datadog-parent-id:|x-datadog-trace-id:|account_id:)[0-9]+/\1XXXX/g" |
114-
# Normalize timestamps in datapoints POSTed to DD
115-
sed -E 's/"points": \[\[[0-9\.]+,/"points": \[\[XXXX,/g' |
116-
# Strip API key from logged requests
117-
sed -E "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" |
118-
# Normalize minor package version so that these snapshots aren't broken on version bumps
119-
sed -E "s/(dd_lambda_layer:datadog-python[0-9]+_2\.)[0-9]+\.0/\1XX\.0/g" |
120-
# Strip out trace/span/parent/timestamps
121-
sed -E "s/(\"trace_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
122-
sed -E "s/(\"span_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
123-
sed -E "s/(\"parent_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
124-
sed -E "s/(\"request_id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
125-
sed -E "s/(\"duration\"\: )[0-9\.\-]+/\1XXXX/g" |
126-
sed -E "s/(\"start\"\: )[0-9\.\-]+/\1XXXX/g" |
127-
sed -E "s/(\"system\.pid\"\: )[0-9\.\-]+/\1XXXX/g"
128-
)
129-
130-
if [ ! -f $function_snapshot_path ]; then
131-
# If no snapshot file exists yet, we create one
132-
echo "Writing logs to $function_snapshot_path because no snapshot exists yet"
133-
echo "$logs" >$function_snapshot_path
134-
elif [ -n "$UPDATE_SNAPSHOTS" ]; then
135-
# If $UPDATE_SNAPSHOTS is set to true write the new logs over the current snapshot
136-
echo "Overwriting log snapshot for $function_snapshot_path"
137-
echo "$logs" >$function_snapshot_path
138-
else
139-
# Compare new logs to snapshots
140-
set +e # Don't exit this script if there is a diff
141-
diff_output=$(echo "$logs" | diff - $function_snapshot_path)
142-
if [ $? -eq 1 ]; then
143-
echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):"
144-
echo "$diff_output"
145-
mismatch_found=true
107+
for _sls_type in "${CONFIGS[@]}"; do
108+
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
109+
for runtime in "${RUNTIMES[@]}"; do
110+
if [ "$_sls_type" = "with-plugin" ]; then
111+
function_name="${handler_name}_${runtime}_with_plugin"
146112
else
147-
echo "Ok: New logs for $function_name match snapshot"
113+
function_name="${handler_name}_${runtime}"
148114
fi
149-
set -e
150-
fi
115+
116+
function_snapshot_path="./snapshots/logs/$function_name.log"
117+
118+
# Fetch logs with serverless cli
119+
if [ "$_sls_type" = "with-plugin" ]; then
120+
raw_logs=$(serverless logs -f $function_name --startTime $script_start_time -c "serverless-plugin.yml")
121+
else
122+
raw_logs=$(serverless logs -f $function_name --startTime $script_start_time)
123+
fi
124+
125+
# Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions
126+
logs=$(
127+
echo "$raw_logs" |
128+
# Filter serverless cli errors
129+
sed '/Serverless: Recoverable error occurred/d' |
130+
# Remove blank lines
131+
sed '/^$/d' |
132+
# Normalize Lambda runtime report logs
133+
sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"): [a-z0-9\.\-]+/\1: XXXX/g' |
134+
# Normalize DD APM headers and AWS account ID
135+
sed -E "s/(x-datadog-parent-id:|x-datadog-trace-id:|account_id:)[0-9]+/\1XXXX/g" |
136+
# Normalize timestamps in datapoints POSTed to DD
137+
sed -E 's/"points": \[\[[0-9\.]+,/"points": \[\[XXXX,/g' |
138+
# Strip API key from logged requests
139+
sed -E "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" |
140+
# Normalize minor package version so that these snapshots aren't broken on version bumps
141+
sed -E "s/(dd_lambda_layer:datadog-python[0-9]+_2\.)[0-9]+\.0/\1XX\.0/g" |
142+
# Strip out trace/span/parent/timestamps
143+
sed -E "s/(\"trace_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
144+
sed -E "s/(\"span_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
145+
sed -E "s/(\"parent_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
146+
sed -E "s/(\"request_id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
147+
sed -E "s/(\"duration\"\: )[0-9\.\-]+/\1XXXX/g" |
148+
sed -E "s/(\"start\"\: )[0-9\.\-]+/\1XXXX/g" |
149+
sed -E "s/(\"system\.pid\"\: )[0-9\.\-]+/\1XXXX/g" |
150+
sed -E "s/(\"runtime-id\"\: \")[a-z0-9\.\-]+/\1XXXX/g"
151+
)
152+
153+
if [ ! -f $function_snapshot_path ]; then
154+
# If no snapshot file exists yet, we create one
155+
echo "Writing logs to $function_snapshot_path because no snapshot exists yet"
156+
echo "$logs" >$function_snapshot_path
157+
elif [ -n "$UPDATE_SNAPSHOTS" ]; then
158+
# If $UPDATE_SNAPSHOTS is set to true write the new logs over the current snapshot
159+
echo "Overwriting log snapshot for $function_snapshot_path"
160+
echo "$logs" >$function_snapshot_path
161+
else
162+
# Compare new logs to snapshots
163+
set +e # Don't exit this script if there is a diff
164+
diff_output=$(echo "$logs" | diff - $function_snapshot_path)
165+
if [ $? -eq 1 ]; then
166+
echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):"
167+
echo "$diff_output"
168+
mismatch_found=true
169+
else
170+
echo "Ok: New logs for $function_name match snapshot"
171+
fi
172+
set -e
173+
fi
174+
done
151175
done
152176
done
153177

154178
if [ "$mismatch_found" = true ]; then
155179
echo "FAILURE: A mismatch between new data and a snapshot was found and printed above."
156180
echo "If the change is expected, generate new snapshots by running 'UPDATE_SNAPSHOTS=true DD_API_KEY=XXXX ./scripts/run_integration_tests.sh'"
181+
echo "Make sure https://httpstat.us/400/ is UP for `http_error` test case"
157182
exit 1
158183
fi
159184

‎tests/integration/decorator.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
def conditional_decorator(dec, condition):
2+
def decorator(func):
3+
if condition:
4+
return func
5+
return dec(func)
6+
7+
return decorator

‎tests/integration/handle.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
1+
import os
2+
3+
from decorator import conditional_decorator
14
from datadog_lambda.metric import lambda_metric
25
from datadog_lambda.wrapper import datadog_lambda_wrapper
36

7+
with_plugin = os.getenv("WITH_PLUGIN", False)
8+
49

5-
@datadog_lambda_wrapper
10+
@conditional_decorator(datadog_lambda_wrapper, with_plugin)
611
def handle(event, context):
712
# Parse request ID and record ids out of the event to include in the response
813
request_id = event.get("requestContext", {}).get("requestId")

‎tests/integration/http_error.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import os
2+
import requests
3+
4+
from decorator import conditional_decorator
5+
from datadog_lambda.metric import lambda_metric
6+
from datadog_lambda.wrapper import datadog_lambda_wrapper
7+
from ddtrace import tracer
8+
from ddtrace.internal.writer import LogWriter
9+
10+
tracer.writer = LogWriter()
11+
with_plugin = os.getenv("WITH_PLUGIN", False)
12+
13+
14+
@conditional_decorator(datadog_lambda_wrapper, with_plugin)
15+
def handle(event, context):
16+
lambda_metric("hello.dog", 1, tags=["team:serverless", "role:hello"])
17+
lambda_metric(
18+
"tests.integration.count", 21, tags=["test:integration", "role:hello"]
19+
)
20+
21+
requests.get("https://httpstat.us/400/")
22+
23+
return {"statusCode": 200, "body": {"message": "hello, dog!"}}

‎tests/integration/http_requests.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
1+
import os
12
import requests
23

4+
from decorator import conditional_decorator
35
from datadog_lambda.metric import lambda_metric
46
from datadog_lambda.wrapper import datadog_lambda_wrapper
57
from ddtrace import tracer
68
from ddtrace.internal.writer import LogWriter
79

810
tracer.writer = LogWriter()
11+
with_plugin = os.getenv("WITH_PLUGIN", False)
912

1013

11-
@datadog_lambda_wrapper
14+
@conditional_decorator(datadog_lambda_wrapper, with_plugin)
1215
def handle(event, context):
1316
lambda_metric("hello.dog", 1, tags=["team:serverless", "role:hello"])
1417
lambda_metric(

‎tests/integration/package.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"devDependencies": {
3+
"serverless-plugin-datadog": "^2.2.1"
4+
}
5+
}

0 commit comments

Comments
 (0)