chore(ci): update layer ARN docs and create PR during release (#2240)

Author: heitorlessa

.github/actions/create-pr/create_pr_for_staged_changes.sh

@@ -3,7 +3,7 @@ set -uo pipefail # prevent accessing unset env vars, prevent masking pipeline er
 
 #docs
 #title :create_pr_for_staged_changes.sh
-#description :This script will create a PR for staged changes, detect and close duplicate PRs.
+#description :This script will create a PR for staged changes, detect and close duplicate PRs. All PRs will be omitted from Release Notes and Changelogs
 #author :@heitorlessa
 #date :May 8th 2023
 #version :0.1
@@ -61,6 +61,8 @@ function set_environment_variables() {
  export readonly PR_BODY="This is an automated PR created from the following workflow"
  export readonly FILENAME=".github/scripts/$(basename "$0")"
  export readonly NO_DUPLICATES_MESSAGE="No duplicated PRs found"
+ export readonly SKIP_LABEL="skip-changelog"
+
  end_span
 }
 
@@ -86,7 +88,8 @@ function create_temporary_branch_with_changes() {
 
 function create_pr() {
  start_span "Creating PR against ${TEMP_BRANCH} branch"
- NEW_PR_URL=$(gh pr create --title "${PR_TITLE}" --body "${PR_BODY}: ${WORKFLOW_URL}" --base "${BASE_BRANCH}" || error "Failed to create PR") # e.g, https://github.com/awslabs/aws-lambda-powertools/pull/13
+ # TODO: create label
+ NEW_PR_URL=$(gh pr create --title "${PR_TITLE}" --body "${PR_BODY}: ${WORKFLOW_URL}" --base "${BASE_BRANCH}" --label "${SKIP_LABEL}" || error "Failed to create PR") # e.g, https://github.com/awslabs/aws-lambda-powertools/pull/13
 
  # greedy remove any string until the last URL path, including the last '/'. https://opensource.com/article/17/6/bash-parameter-expansion
  debug "Extracing PR Number from PR URL: "${NEW_PR_URL}""

.github/workflows/publish_v2_layer.yml

@@ -1,10 +1,5 @@
 name: Deploy v2 layer to all regions
 
-permissions:
- id-token: write
- contents: write
- pages: write
-
 on:
  workflow_dispatch:
  inputs:
@@ -31,7 +26,11 @@ on:
 jobs:
  build-layer:
  permissions:
+ # lower privilege propagated from parent workflow (release.yml)
  contents: read
+ id-token: none
+ pages: none
+ pull-requests: none
  runs-on: aws-lambda-powertools_ubuntu-latest_8-core
  defaults:
  run:
@@ -87,6 +86,12 @@ jobs:
 
  beta:
  needs: build-layer
+ # lower privilege propagated from parent workflow (release.yml)
+ permissions:
+ id-token: write
+ contents: read
+ pages: write # docs will be updated with latest Layer ARNs
+ pull-requests: write # creation-action will create a PR with Layer ARN updates
  uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml
  secrets: inherit
  with:
@@ -97,6 +102,12 @@ jobs:
 
  prod:
  needs: beta
+ # lower privilege propagated from parent workflow (release.yml)
+ permissions:
+ id-token: write
+ contents: read
+ pages: write # docs will be updated with latest Layer ARNs
+ pull-requests: write # creation-action will create a PR with Layer ARN updates
  uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml
  secrets: inherit
  with:
@@ -107,6 +118,12 @@ jobs:
 
  sar-beta:
  needs: build-layer
+ permissions:
+ # lower privilege propagated from parent workflow (release.yml)
+ id-token: write
+ contents: read
+ pull-requests: none
+ pages: none
  uses: ./.github/workflows/reusable_deploy_v2_sar.yml
  secrets: inherit
  with:
@@ -117,6 +134,12 @@ jobs:
 
  sar-prod:
  needs: [build-layer, sar-beta]
+ permissions:
+ # lower privilege propagated from parent workflow (release.yml)
+ id-token: write
+ contents: read
+ pull-requests: none
+ pages: none
  uses: ./.github/workflows/reusable_deploy_v2_sar.yml
  secrets: inherit
  with:
@@ -125,10 +148,62 @@ jobs:
  environment: "layer-prod"
  package-version: ${{ inputs.latest_published_version }}
 
+ # Updating the documentation with the latest Layer ARNs is a two-phase process
+ #
+ # 1. Update layer ARNs with latest deployed locally and create a PR with these changes
+ # 2. Pull from temporary branch with these changes and update the docs we're releasing
+ #
+ # This keeps our permissions tight and we don't run into a conflict,
+ # where a new release creates a new doc (2.16.0) while layers are still pointing to 2.15
+ # because the PR has to be merged while release process is running
+
+ update_v2_layer_arn_docs:
+ needs: prod
+ outputs:
+ temp_branch: ${{ steps.create-pr.outputs.temp_branch }}
+ runs-on: ubuntu-latest
+ permissions:
+ # lower privilege propagated from parent workflow (release.yml)
+ contents: write
+ pull-requests: write
+ id-token: none
+ pages: none
+ steps:
+ - name: Checkout repository # reusable workflows start clean, so we need to checkout again
+ uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+ with:
+ fetch-depth: 0
+ - name: Download CDK layer artifact
+ uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+ with:
+ name: cdk-layer-stack
+ path: cdk-layer-stack/
+ - name: Replace layer versions in documentation
+ run: |
+ ls -la cdk-layer-stack/
+ ./layer/scripts/update_layer_arn.sh cdk-layer-stack
+ # NOTE: It felt unnecessary creating yet another PR to update changelog w/ latest tag
+ # since this is the only step in the release where we update docs from a temp branch
+ - name: Update changelog with latest tag
+ run: make changelog
+ - name: Create PR
+ id: create-pr
+ uses: ./.github/actions/create-pr
+ with:
+ files: "docs/index.md examples CHANGELOG.md"
+ temp_branch_prefix: "ci-layer-docs"
+ pull_request_title: "chore(ci): layer docs update"
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+
+
  prepare_docs_alias:
  runs-on: ubuntu-latest
  permissions:
+ # lower privilege propagated from parent workflow (release.yml)
  contents: read
+ pages: none
+ id-token: none
+ pull-requests: none
  outputs:
  DOCS_ALIAS: ${{ steps.set-alias.outputs.DOCS_ALIAS }}
  steps:
@@ -141,13 +216,16 @@ jobs:
  fi
  echo DOCS_ALIAS="$DOCS_ALIAS" >> "$GITHUB_OUTPUT"
 
- release-docs:
- needs: [prod, prepare_docs_alias]
+ release_docs:
+ needs: [update_v2_layer_arn_docs, prepare_docs_alias]
  permissions:
+ # lower privilege propagated from parent workflow (release.yml)
  contents: write
  pages: write
+ pull-requests: none
+ id-token: none
  uses: ./.github/workflows/reusable_publish_docs.yml
  with:
  version: ${{ inputs.latest_published_version }}
  alias: ${{ needs.prepare_docs_alias.outputs.DOCS_ALIAS }}
- detached_mode: true
+ git_ref: ${{ needs.update_v2_layer_arn_docs.outputs.temp_branch }}

.github/workflows/release.yml

@@ -5,12 +5,14 @@ name: Release
 # === Automated activities ===
 #
 # 1. Run tests, linting, security and complexity base line
-# 2. Bump package version, build release artifact, and generate latest Changelog
+# 2. Bump package version and build release artifact
 # 3. Publish package to PyPi prod repository using cached artifact
-# 4. Kick off Layers pipeline to compile and publish latest version
-# 5. Updates documentation to use the latest Layer ARN for all commercial regions
-# 6. Builds a new user guide and API docs with release version; update /latest pointing to newly released version
-# 7. Close all issues labeled "pending-release" and notify customers about the release
+# 4. Compile Layer and kick off pipeline for beta, prod, and canary releases
+# 5. Update docs with latest Layer ARNs and Changelog
+# 6. Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged
+# 7. Builds a new user guide and API docs with release version; update /latest pointing to newly released version
+# 8. Create PR to update package version on trunk
+# 9. Close all issues labeled "pending-release" and notify customers about the release
 #
 # === Manual activities ===
 #
@@ -126,15 +128,36 @@ jobs:
  # with:
  # repository-url: https://test.pypi.org/legacy/
 
+ create_tag:
+ needs: [build, release]
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ env:
+ RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }}
+ steps:
+ - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+ - id: setup-git
+ name: Git client setup and refresh tip
+ run: |
+ git config user.name "Powertools bot"
+ git config user.email "aws-lambda-powertools-feedback@amazon.com"
+ git config remote.origin.url >&-
+ - name: Create Git Tag
+ run: |
+ git tag -a v"${RELEASE_VERSION}" -m "release_version: v${RELEASE_VERSION}"
+ git push origin v"${RELEASE_VERSION}"
+
  # NOTE: Watch out for the depth limit of 4 nested workflow_calls.
- # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack -> reusable_update_v2_layer_arn_docs
+ # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack
  publish_layer:
- needs: [build, release]
+ needs: [build, release, create_tag]
  secrets: inherit
  permissions:
  id-token: write
  contents: write
  pages: write
+ pull-requests: write
  uses: ./.github/workflows/publish_v2_layer.yml
  with:
  latest_published_version: ${{ needs.build.outputs.RELEASE_VERSION }}

.github/workflows/reusable_deploy_v2_layer_stack.yml

@@ -1,9 +1,5 @@
 name: Deploy CDK Layer v2 stack
 
-permissions:
- id-token: write
- contents: write
-
 on:
  workflow_call:
  inputs:
@@ -28,6 +24,12 @@ jobs:
  deploy-cdk-stack:
  runs-on: ubuntu-latest
  environment: ${{ inputs.environment }}
+ # lower privilege propagated from parent workflow (publish_v2_layer.yml)
+ permissions:
+ id-token: write
+ pull-requests: none
+ contents: read
+ pages: none
  defaults:
  run:
  working-directory: ./layer
@@ -149,10 +151,3 @@ jobs:
  retention-days: 1
  - name: CDK Deploy Canary
  run: npx cdk deploy --app cdk.out --context region=${{ matrix.region }} --parameters DeployStage="${{ inputs.stage }}" --parameters HasARM64Support=${{ matrix.has_arm64_support }} 'CanaryV2Stack' --require-approval never --verbose
-
- update_v2_layer_arn_docs:
- needs: deploy-cdk-stack
- if: ${{ inputs.stage == 'PROD' }}
- uses: ./.github/workflows/reusable_update_v2_layer_arn_docs.yml
- with:
- latest_published_version: ${{ inputs.latest_published_version }}

.github/workflows/reusable_publish_docs.yml

@@ -1,7 +1,6 @@
 name: Reusable publish documentation
 
 env:
- BRANCH: develop
  ORIGIN: awslabs/aws-lambda-powertools-python
 
 on:
@@ -20,6 +19,11 @@ on:
  required: false
  default: false
  type: boolean
+ git_ref:
+ description: "Branch or commit ID to checkout from"
+ required: false
+ type: string
+ default: develop
 
 permissions:
  contents: write
@@ -36,6 +40,7 @@ jobs:
  - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
  with:
  fetch-depth: 0
+ ref: ${{ inputs.git_ref }}
  - name: Install poetry
  run: pipx install poetry
  - name: Set up Python
@@ -56,6 +61,8 @@ jobs:
  git config pull.rebase true
  git config remote.origin.url >&- || git remote add origin https://github.com/"$ORIGIN"
  git pull origin "$BRANCH"
+ env:
+ BRANCH: ${{ inputs.git_ref }}
  - name: Build docs website and API reference
  env:
  VERSION: ${{ inputs.version }}

.github/workflows/reusable_update_v2_layer_arn_docs.yml

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# This script is run during the reusable_update_v2_layer_arn_docs CI job,
+# This script is run during the publish_v2_layer.yml CI job,
 # and it is responsible for replacing the layer ARN in our documentation,
 # based on the output files generated by CDK when deploying to each pseudo_region.
 #