Merge branch 'master' into darcy.rayner/add-integration-tests

Author: DarcyRaynerDD

aws/logs_monitoring/README.md

@@ -35,10 +35,13 @@ Note: you can find the installed Forwarder under the stack's "Resources" tab.
 Since version 3.0.0, the forwarder Lambda function is managed by CloudFormation. To upgrade an older forwarder installation to 3.0.0 and above, follow the steps below.
 
 1. Install a new forwarder following the [installation](#installation) steps.
-1. Manually migrate a few triggers on the old forwarder to the new one.
-1. Ensure the new forwarder is working as expected.
+1. Manually migrate a few triggers (CloudWatch log group subscription filter and S3 bucket event notification) on the old forwarder to the new one.
+1. Ensure the new forwarder is working as expected, i.e., being invoked regularly without errors.
+1. Ensure the logs from the migrated triggers (sources) are showing up in Datadog log explorer and look right to you.
 1. Update all the triggers on the old forwarder to the new one, following these [steps](https://docs.datadoghq.com/integrations/amazon_web_services/?tab=allpermissions#send-aws-service-logs-to-datadog).
+1. Ensure the old forwarder Lambda function's invocations count drops to zero.
 1. Delete the old forwarder Lambda function when you feel comfortable.
+1. If you have old forwarder Lambda functions installed in multiple AWS accounts and regions, repeat the steps above in every account and region combination.
 
 ### Adjusting forwarder settings
 

aws/logs_monitoring/lambda_function.py

@@ -321,7 +321,7 @@ def compileRegex(rule, pattern):
 DD_CUSTOM_TAGS = "ddtags"
 DD_SERVICE = "service"
 DD_HOST = "host"
-DD_FORWARDER_VERSION = "3.2.0"
+DD_FORWARDER_VERSION = "3.3.0"
 
 
 class RetriableException(Exception):

aws/logs_monitoring/release.sh

@@ -12,10 +12,6 @@ else
  BUCKET=$1
 fi
 
-# Get the latest code
-git checkout master
-git pull origin master
-
 # Read the current version
 CURRENT_VERSION=$(grep -o 'Version: \d\+\.\d\+\.\d\+' template.yaml | cut -d' ' -f2)
 
@@ -26,60 +22,89 @@ if [ -z "$2" ]; then
 elif [[ ! $2 =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then
  echo "Must use a semantic version, e.g., 3.1.4"
  exit 1
-elif [[ ! "$2" > "$CURRENT_VERSION" ]]; then
- echo "Must use a version greater than the current ($CURRENT_VERSION)"
- exit 1
 else
  VERSION=$2
 fi
 
-# Make the template private (default is public) - useful for developers
-if [[ $# -eq 3 ]] && [[ $3 = "--private" ]]; then
- PRIVATE_TEMPLATE=true
+# Do a production release (default is staging) - useful for developers
+if [[ $# -eq 3 ]] && [[ $3 = "--prod" ]]; then
+ PROD_RELEASE=true
 else
- PRIVATE_TEMPLATE=false
+ PROD_RELEASE=false
 fi
 
+# Validate identity
+aws sts get-caller-identity
+
 # Validate the template
 echo "Validating template.yaml"
 aws cloudformation validate-template --template-body file://template.yaml
 
-# Confirm to proceed
-read -p "About to bump the version from ${CURRENT_VERSION} to ${VERSION}, create a release aws-dd-forwarder-${VERSION} on Github and upload the template.yaml to s3://${BUCKET}/aws/forwarder/${VERSION}.yaml. Continue (y/n)?" CONT
-if [ "$CONT" != "y" ]; then
- echo "Exiting"
- exit 1
-fi
+if [ "$PROD_RELEASE" = true ] ; then
 
-# Bump version number
-echo "Bumping the current version number to the desired"
-perl -pi -e "s/DD_FORWARDER_VERSION = \"${CURRENT_VERSION}/DD_FORWARDER_VERSION = \"${VERSION}/g" lambda_function.py
-perl -pi -e "s/Version: ${CURRENT_VERSION}/Version: ${VERSION}/g" template.yaml
+ if [[ ! $(./tools/semver.sh "$VERSION" "$CURRENT_VERSION") > 0 ]]; then
+ echo "Must use a version greater than the current ($CURRENT_VERSION)"
+ exit 1
+ fi
+ # Make sure we are on master
+ BRANCH=$(git rev-parse --abbrev-ref HEAD)
+ if [ $BRANCH != "master" ]; then
+ echo "Not on master, aborting"
+ exit 1
+ fi
 
-# Commit version number changes to git
-git add lambda_function.py template.yaml README.md
-git commit -m "Bump version from ${CURRENT_VERSION} to ${VERSION}"
-git push origin master
+ # Confirm to proceed
+ read -p "About to bump the version from ${CURRENT_VERSION} to ${VERSION}, create a release aws-dd-forwarder-${VERSION} on Github and upload the template.yaml to s3://${BUCKET}/aws/forwarder/${VERSION}.yaml. Continue (y/n)?" CONT
+ if [ "$CONT" != "y" ]; then
+ echo "Exiting"
+ exit 1
+ fi
 
-# Create a github release
-echo "Release aws-dd-forwarder-${VERSION} to github"
-go get github.com/github/hub
-rm -f aws-dd-forwarder-*.zip
-zip -r aws-dd-forwarder-${VERSION}.zip .
-hub release create -a aws-dd-forwarder-${VERSION}.zip -m "aws-dd-forwarder-${VERSION}" aws-dd-forwarder-${VERSION}
+ # Get the latest code
+ git pull origin master
+
+ # Bump version number
+ echo "Bumping the current version number to the desired"
+ perl -pi -e "s/DD_FORWARDER_VERSION = \"${CURRENT_VERSION}/DD_FORWARDER_VERSION = \"${VERSION}/g" lambda_function.py
+ perl -pi -e "s/Version: ${CURRENT_VERSION}/Version: ${VERSION}/g" template.yaml
+
+ # Commit version number changes to git
+ git add lambda_function.py template.yaml README.md
+ git commit -m "Bump version from ${CURRENT_VERSION} to ${VERSION}"
+ git push origin master
+
+ # Create a github release
+ echo "Release aws-dd-forwarder-${VERSION} to github"
+ go get github.com/github/hub
+ rm -f aws-dd-forwarder-*.zip
+ zip -r aws-dd-forwarder-${VERSION}.zip .
+ hub release create -a aws-dd-forwarder-${VERSION}.zip -m "aws-dd-forwarder-${VERSION}" aws-dd-forwarder-${VERSION}
+ TEMPLATE_URL="https://${BUCKET}.s3.amazonaws.com/aws/forwarder/latest.yaml"
+ FORWARDER_SOURCE_URL="https://github.com/DataDog/datadog-serverless-functions/releases/download/aws-dd-forwarder-${VERSION}/aws-dd-forwarder-${VERSION}.zip'"
+else
+ echo "About to release non-public staging version of forwarder, upload aws-dd-forwarder-${VERSION} to s3, and upload the template.yaml to s3://${BUCKET}/aws/forwarder-staging/${VERSION}.yaml"
+ # Upload to s3 instead of github
+ rm -f aws-dd-forwarder-*.zip
+ zip -r aws-dd-forwarder-${VERSION}.zip .
+ aws s3 cp aws-dd-forwarder-${VERSION}.zip s3://${BUCKET}/aws/forwarder-staging-zip/aws-dd-forwarder-${VERSION}.zip
+ TEMPLATE_URL="https://${BUCKET}.s3.amazonaws.com/aws/forwarder-staging/latest.yaml"
+ FORWARDER_SOURCE_URL="s3://${BUCKET}/aws/forwarder-staging-zip/aws-dd-forwarder-${VERSION}.zip"
+fi
 
 # Upload the template to the S3 bucket
 echo "Uploading template.yaml to s3://${BUCKET}/aws/forwarder/${VERSION}.yaml"
-if [ "$PRIVATE_TEMPLATE" = true ] ; then
- aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder/${VERSION}.yaml
- aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder/latest.yaml
-else
+
+if [ "$PROD_RELEASE" = true ] ; then
  aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder/${VERSION}.yaml \
  --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
  aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder/latest.yaml \
  --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
+else
+ aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder-staging/${VERSION}.yaml
+ aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder-staging/latest.yaml
 fi
+
 echo "Done uploading the template, and here is the CloudFormation quick launch URL"
-echo "https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=datadog-serverless&templateURL=https://${BUCKET}.s3.amazonaws.com/aws/forwarder/latest.yaml"
+echo "https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=datadog-serverless&templateURL=${TEMPLATE_URL}"
 
 echo "Done!"