Restore repository indexing (#532)

* Slim down GH action to use the same code path for dry and wet runs * Let Ansible handle repository filtering This fixes repository indexing (fixes #411), and makes config/repositories.yaml authoritative (again) on the operator configuration. * Try to fix GH workflow syntax, maybe? * Document sharding for the local test setup * .yml is not .yaml * Satisfy pre-commit

Author: nightkr

.github/workflows/generate_prs.yml

@@ -27,83 +27,26 @@ jobs:
  runs-on: ubuntu-latest
  strategy:
  fail-fast: ${{ inputs.fail-fast }}
- # If you add a new repository here, also add it to config/repositories.yaml
  matrix:
+ # This should be the list of repository names from config/repositories.yaml:
+ # yq '.repositories | map(.name)' config/repositories.yaml
  repository:
- - name: airflow-operator
- pretty_string: Apache Airflow
- product_string: airflow
- url: stackabletech/airflow-operator.git
- - name: commons-operator
- include_productconfig: false
- has_product: false
- pretty_string: Stackable Commons
- product_string: commons
- url: stackabletech/commons-operator.git
- - name: druid-operator
- pretty_string: Apache Druid
- product_string: druid
- url: stackabletech/druid-operator.git
- - name: hbase-operator
- pretty_string: Apache HBase
- product_string: hbase
- url: stackabletech/hbase-operator.git
- - name: hdfs-operator
- pretty_string: Apache HDFS
- product_string: hdfs
- url: stackabletech/hdfs-operator.git
- - name: hive-operator
- pretty_string: Apache Hive
- product_string: hive
- url: stackabletech/hive-operator.git
- - name: kafka-operator
- pretty_string: Apache Kafka
- product_string: kafka
- url: stackabletech/kafka-operator.git
- - name: nifi-operator
- pretty_string: Apache NiFi
- product_string: nifi
- url: stackabletech/nifi-operator.git
- - name: listener-operator
- include_productconfig: false
- has_product: false
- pretty_string: Stackable Listener Operator
- product_string: listener-operator
- run_as: custom
- url: stackabletech/listener-operator.git
- - name: opa-operator
- extra_crates:
- - stackable-opa-bundle-builder
- pretty_string: OpenPolicyAgent
- product_string: opa
- url: stackabletech/opa-operator.git
- - name: opensearch-operator
- pretty_string: OpenSearch
- product_string: opensearch
- url: stackabletech/opensearch-operator.git
- - name: secret-operator
- include_productconfig: false
- has_product: false
- pretty_string: Stackable Secret Operator
- product_string: secret-operator
- run_as: custom
- url: stackabletech/secret-operator.git
- - name: spark-k8s-operator
- pretty_string: Apache Spark-on-Kubernetes
- product_string: spark-k8s
- url: stackabletech/spark-k8s-operator.git
- - name: superset-operator
- pretty_string: Apache Superset
- product_string: superset
- url: stackabletech/superset-operator.git
- - name: trino-operator
- pretty_string: Trino
- product_string: trino
- url: stackabletech/trino-operator.git
- - name: zookeeper-operator
- pretty_string: Apache ZooKeeper
- product_string: zookeeper
- url: stackabletech/zookeeper-operator.git
+ - airflow-operator
+ - commons-operator
+ - druid-operator
+ - hbase-operator
+ - hdfs-operator
+ - hive-operator
+ - kafka-operator
+ - nifi-operator
+ - listener-operator
+ - opa-operator
+ - opensearch-operator
+ - secret-operator
+ - spark-k8s-operator
+ - superset-operator
+ - trino-operator
+ - zookeeper-operator
  steps:
  - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
  with:
@@ -113,10 +56,10 @@ jobs:
  env:
  DEBIAN_FRONTEND: noninteractive
  run: |
-  sudo apt update && \
-  sudo apt install -y software-properties-common && \
-  sudo apt-add-repository ppa:ansible/ansible -y && \
-  sudo apt install -y ansible
+ sudo apt update && \
+ sudo apt install -y software-properties-common && \
+ sudo apt-add-repository ppa:ansible/ansible -y && \
+ sudo apt install -y ansible
 
  # NOTE (@NickLarsenNZ): This could be removed in favor of nix-shell and rrbutani/use-nix-shell-action
  - name: Install deps for operators
@@ -148,36 +91,18 @@ jobs:
  echo "AUTHOR=stackabletech/developers"
  echo "REASON=Daily run triggered" >> "$GITHUB_ENV"
 
- - name: Overwrite repositories.yaml for ${{ matrix.repository.name }}
- run: |
- cat >config/repositories.yaml <<CONFIG
- ---
- repositories:
- - ${{ toJSON(matrix.repository) }}
- CONFIG
-
- # For debugging
- cat config/repositories.yaml
-
- # Generate PRs
  - name: Run playbook
- if: ${{ !inputs.dry-run }}
  env:
  CUSTOM_PR_TITLE: ${{ inputs.custom-pr-title }}
  GH_ACCESS_TOKEN: ${{ secrets.gh_access_token }}
+ REPOSITORY: ${{ matrix.repository }}
+ # tags local excludes all actions that actually generate PRs
+ DRY_RUN_FLAGS: ${{ inputs.dry-run && '--tags=local' || '' }}
+ # shellsheck disable=SC2086
  run: |
- # Funnel via JSON to ensure that values are escaped properly
- # Check if custom PR title is set. If so, insert it into the extra vars
- if [ -n "$CUSTOM_PR_TITLE" ]; then
- echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, custom_pr_title: $ENV.CUSTOM_PR_TITLE, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN}' --arg pwd "$(pwd)" > vars.json
- else
- echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN}' --arg pwd "$(pwd)" > vars.json
- fi
- ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json"
-
- # Do Not Generate PRs
- - name: Run playbook (dry-run)
- if: ${{ inputs.dry-run }}
- run: ./test.sh
- env:
- GH_ACCESS_TOKEN: ""
+ echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, custom_pr_title: $ENV.CUSTOM_PR_TITLE, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN, shard_repositories: [$ENV.REPOSITORY]} | with_entries(select(.value != null and .value != ""))' --arg pwd "$(pwd)" > vars.json
+ # $DRY_RUN_FLAGS is intentionally not quoted, since we need to avoid
+ # passing an argument if we're not in dry mode.
+ # This is safe, since it always has one of two hard-coded values.
+ # shellcheck disable=SC2086
+ ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json" $DRY_RUN_FLAGS

README.adoc

@@ -85,7 +85,7 @@ Consult the section above to learn more about the structure of the template.
 
 === Test changes locally
 
-1. Run the `test.sh` script.
+1. Run the `test.sh` script. To limit the run to a single operator, add `--extra-vars "shard_repositories=['nifi-operator']"` (or your choice of operator).
  It will automatically delete and recreate a `work` directory.
 2. The changes can be examined with `git status`.
  When the pull request is later merged into the `main` branch then pull requests with these changes will be created automatically.

config/repositories.yaml

@@ -1,7 +1,5 @@
-# This file exists just for the purpose of running test.sh locally.
-# The GH workflow generate_prs.yml will overwrite it with the contents if the
-# matrix run.
-# If you add a new repository here, you also need to add it to that workflow file.
+# If you add a new repository here, you also need to add it to the list of
+# repositories in ../.github/workflows/generate_prs.yml.
 ---
 repositories:
  - name: airflow-operator

playbook/playbook.yaml

@@ -24,6 +24,7 @@
 
  - name: Update repositories from templates
  include_tasks: "update_repo.yaml"
+ when: shard_repositories is undefined or operator.name in shard_repositories
  with_items: "{{ repositories }}"
  loop_control:
  loop_var: operator

test.sh

@@ -2,4 +2,4 @@
 
 rm -fr work || true
 mkdir -p work
-ansible-playbook playbook/playbook.yaml --tags "local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'"
+ansible-playbook playbook/playbook.yaml --tags="local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'" "$@"