New CD workflow

.github/workflows/release.yml

@@ -0,0 +1,56 @@
+name: Release
+
+env:
+ GH_TOKEN: ${{ github.token }}
+
+on:
+ workflow_dispatch:
+ inputs:
+ release_type:
+ type: choice
+ description: "Release type:"
+ required: true
+ options:
+ - bug fix (PATCH)
+ - new feature (MINOR)
+ release_title:
+ description: "The title of the release"
+ required: true
+
+jobs:
+ update-version:
+ runs-on: ubuntu-20.04 # latest
+ permissions:
+ contents: write # allow push
+
+ steps:
+ - name: Checkout Sources
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0
+
+ - name: Make new release
+ env:
+ Title: ${{ github.event.inputs.release_title }}
+ run: |
+ Title=$(echo ${Title//[\"]\\\"})
+ Title=$(echo ${Title//[\']\\\'})
+ Title=$(echo ${Title//[\$]})
+
+ ./utils/publish-release.sh "${{ github.event.inputs.release_type }}" "$Title"
+
+ - name: configure AWS credentials (Release)
+ uses: aws-actions/configure-aws-credentials@v1
+ with:
+ role-to-assume: ${{ secrets.AWS_CI_RELEASE_ROLE }}
+ aws-region: us-east-1
+
+ - name: "Create VERSION file and trigger release"
+ run: |
+ version=$(git describe --tags --abbrev=0)
+ version_without_v=$(echo ${version} | cut -f2 -dv)
+ echo "${version_without_v}" > VERSION
+
+ zip VERSION.zip VERSION
+ export S3_URL=$(aws secretsmanager get-secret-value --secret-id ci/python_v2_version --query "SecretString" | cut -f2 -d\")
+ aws s3 cp VERSION.zip $S3_URL

continuous-delivery/publish_to_prod_pypi.yml → codebuild/cd/publish-to-prod-pypi.yml

@@ -1,5 +1,9 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes a release deployment to production PyPi repo.
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
 version: 0.2
-# this image assumes Ubuntu base image
 phases:
  install:
  commands:
@@ -8,11 +12,13 @@ phases:
  - export PATH=$PATH:$HOME/.local/bin
  - python3 -m pip install --user --upgrade pip
  - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
+ - echo "\nBuild version data:"
+ - echo "\nPython Version:"; python3 --version
  pre_build:
  commands:
- - cd aws-iot-device-sdk-python-v2
- - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-sdk-python-v2/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
- - export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
+ - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+ - pypirc=$(aws secretsmanager get-secret-value --secret-id "cd/aws-sdk-python-v2-prod/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
+ - export PKG_VERSION=$(cat $CODEBUILD_SRC_DIR/VERSION)
  - echo "Updating package version to ${PKG_VERSION}"
  - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" awsiot/__init__.py
  build:

continuous-delivery/publish_to_test_pypi.yml → codebuild/cd/publish-to-test-pypi.yml

@@ -1,5 +1,9 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes a release deployment to test PyPi repo.
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
 version: 0.2
-# this image assumes Ubuntu base image
 phases:
  install:
  commands:
@@ -8,11 +12,13 @@ phases:
  - export PATH=$PATH:$HOME/.local/bin
  - python3 -m pip install --user --upgrade pip
  - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
+ - echo "\nBuild version data:"
+ - echo "\nPython Version:"; python3 --version
  pre_build:
  commands:
- - pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
- - cd aws-iot-device-sdk-python-v2
- - export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
+ - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+ - pypirc=$(aws secretsmanager get-secret-value --secret-id "cd/aws-sdk-python-v2-test/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
+ - export PKG_VERSION=$(cat $CODEBUILD_SRC_DIR/VERSION)
  - echo "Updating package version to ${PKG_VERSION}"
  - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" awsiot/__init__.py
  build:

codebuild/cd/test-prod-pypi.yml

@@ -0,0 +1,36 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes sure the release to test PyPi repository worked
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
+version: 0.2
+phases:
+ install:
+ commands:
+ - sudo apt-get update -y
+ - sudo apt-get install python3 python3-pip -y
+ - python3 -m pip install --upgrade pip
+ - python3 -m pip install --upgrade setuptools
+ - echo "\nBuild version data:"
+ - echo "\nPython Version:"; python3 --version
+
+ pre_build:
+ commands:
+ # Material for PubSub sample
+ - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
+ - cert=$(aws secretsmanager get-secret-value --secret-id "ci/PubSub/cert" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
+ - key=$(aws secretsmanager get-secret-value --secret-id "ci/PubSub/key" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
+ - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+ build:
+ commands:
+ - echo Build started on `date`
+ - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+ - CURRENT_TAG_VERSION=$(cat $CODEBUILD_SRC_DIR/VERSION)
+ - python3 codebuild/cd/pip-install-with-retry.py --no-cache-dir --user awsiotsdk==$CURRENT_TAG_VERSION
+ # Run PubSub sample
+ - python3 samples/pubsub.py --endpoint ${ENDPOINT} --cert /tmp/certificate.pem --key /tmp/privatekey.pem --ca_file /tmp/AmazonRootCA1.pem --verbosity Trace
+
+ post_build:
+ commands:
+ - echo Build completed on `date`
+

codebuild/cd/test-test-pypi.yml

@@ -0,0 +1,36 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes sure the release to test PyPi repository worked
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
+version: 0.2
+phases:
+ install:
+ commands:
+ - sudo apt-get update -y
+ - sudo apt-get install python3 python3-pip -y
+ - python3 -m pip install --upgrade pip
+ - python3 -m pip install --upgrade setuptools
+ - echo "\nBuild version data:"
+ - echo "\nPython Version:"; python3 --version
+
+ pre_build:
+ commands:
+ - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
+ - cert=$(aws secretsmanager get-secret-value --secret-id "ci/PubSub/cert" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
+ - key=$(aws secretsmanager get-secret-value --secret-id "ci/PubSub/key" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
+ - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+ build:
+ commands:
+ - echo Build started on `date`
+ - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+ - CURRENT_TAG_VERSION=$(cat $CODEBUILD_SRC_DIR/VERSION)
+ # this is here because typing isn't in test-pypi, so pull it from prod instead
+ - python3 -m pip install typing
+ - python3 codebuild/cd/pip-install-with-retry.py -i https://testpypi.python.org/simple --user awsiotsdk==$CURRENT_TAG_VERSION
+ # Run PubSub sample
+ - python3 samples/pubsub.py --endpoint ${ENDPOINT} --cert /tmp/certificate.pem --key /tmp/privatekey.pem --ca_file /tmp/AmazonRootCA1.pem --verbosity Trace
+
+ post_build:
+ commands:
+ - echo Build completed on `date`

codebuild/cd/test-version-exists.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+VERSION_FILE_PATH=$1
+if [ ! readlink -e "$VERSION_FILE_PATH" ]; then
+ echo "No VERSION file found! Cannot make release!"
+ exit 1
+else
+ echo "VERSION file found..."
+fi
+VERSION=$(cat $VERSION_FILE_PATH)
+
+# Make sure the version variable is populated
+if [ -z "${VERSION}" ]; then
+ echo "VERSION file is empty!"
+ exit 1
+else
+ echo "VERSION file contains: ${VERSION}"
+fi
+
+# Make sure the version follows the correct format: major.minor.patch
+LENGTH_CHECK="${VERSION//[^.]}"
+if [ ${#LENGTH_CHECK} != 2 ]; then
+ echo "VERSION file contains invalid version (not in format major.minor.patch)"
+ exit 1
+fi
+# Use RegX to ensure it only contains numbers and periods
+REGX_CHECK='^([0-9]+\.){0,2}(\*|[0-9]+)$'
+if [[ $VERSION =~ $REGX_CHECK ]]; then
+ echo "VERSION file contains valid version"
+else
+ echo "VERSION file contains invalid version (RegX validator failed)"
+ exit 1
+fi
+
+# Does PyPi have the version? If so, do not allow it!
+if python3 -m pip install --no-cache-dir -vvv awsiotsdk==$VERSION; then
+ echo "$VERSION is already in pypi, cut a new tag if you want to upload another version."
+ exit 1
+fi
+
+echo "$VERSION currently does not exist in pypi, allowing pipeline to continue."
+exit 0

codebuild/cd/test-version-exists.yml

@@ -0,0 +1,19 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes sure the version in CD has not already been released.
+# Will fail the build and stop the pipeline if the version has already been released.
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
+version: 0.2
+phases:
+ install:
+ commands:
+ - sudo apt-get update -y
+ - sudo apt-get install python3 python3-pip -y
+ - pip3 install --upgrade setuptools
+ - echo "\nBuild version data:"
+ - echo "\nPython Version:"; python3 --version
+ build:
+ commands:
+ - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+ - bash ./codebuild/cd/test-version-exists.sh $CODEBUILD_SRC_DIR/VERSION