Publish connector command now auto bumps version

.github/workflows/publish-command.yml

@@ -20,6 +20,10 @@ on:
  comment-id:
  description: "The comment-id of the slash command. Used to update the comment with the status."
  required: false
+ auto-bump-version:
+ description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec"
+ required: true
+ default: "true"
 
 jobs:
  find_valid_pat:
@@ -95,17 +99,25 @@ jobs:
  with:
  repository: ${{ github.event.inputs.repo }}
  ref: ${{ github.event.inputs.gitref }}
+ token: ${{ secrets.OCTAVIA_PAT }}
  - name: Install Java
  uses: actions/setup-java@v1
  with:
- java-version: '17'
+ java-version: "17"
  - name: Install Pyenv and Tox
  # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/
  run: |
  python3 -m pip install --quiet virtualenv==16.7.9 --user
  python3 -m virtualenv venv
  source venv/bin/activate
  pip install --quiet tox==3.24.4
+ - name: Install yq
+ if: github.event.inputs.auto-bump-version == 'true' && success()
+ run: |
+ sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+ sudo add-apt-repository ppa:rmescandon/yq
+ sudo apt update
+ sudo apt install yq -y
  - name: Test and install CI scripts
  # all CI python packages have the prefix "ci_"
  run: |
@@ -118,14 +130,17 @@ jobs:
  ci_credentials ${{ github.event.inputs.connector }}
  env:
  GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
- - name: Prepare Sentry Release
+ - name: Set Name and Version Environment Vars
  if: startsWith(github.event.inputs.connector, 'connectors')
  run: |
- curl -sL https://sentry.io/get-cli/ | bash
  source tools/lib/lib.sh
  DOCKERFILE=airbyte-integrations/${{ github.event.inputs.connector }}/Dockerfile
  echo "IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV
  echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV
+ - name: Prepare Sentry
+ if: startsWith(github.event.inputs.connector, 'connectors')
+ run: |
+ curl -sL https://sentry.io/get-cli/ | bash
  - name: Create Sentry Release
  if: startsWith(github.event.inputs.connector, 'connectors')
  run: |
@@ -153,20 +168,20 @@ jobs:
  SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
  SENTRY_ORG: airbyte-5j
  SENTRY_PROJECT: airbyte-connectors
- - name: Add Success Comment
+ - name: Add Published Success Comment
  if: github.event.inputs.comment-id && success()
  uses: peter-evans/create-or-update-comment@v1
  with:
  comment-id: ${{ github.event.inputs.comment-id }}
  body: |
- > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
- - name: Add Failure Comment
+ > :rocket: Successfully published ${{github.event.inputs.connector}}
+ - name: Add Published Failure Comment
  if: github.event.inputs.comment-id && !success()
  uses: peter-evans/create-or-update-comment@v1
  with:
  comment-id: ${{ github.event.inputs.comment-id }}
  body: |
- > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+ > :x: Failed to publish ${{github.event.inputs.connector}}
  - name: Slack Notification - Failure
  if: failure()
  uses: rtCamp/action-slack-notify@master
@@ -177,6 +192,63 @@ jobs:
  SLACK_COLOR: DC143C
  SLACK_TITLE: "Failed to publish connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}"
  SLACK_FOOTER: ""
+ - name: Check if connector in definitions yaml
+ if: github.event.inputs.auto-bump-version == 'true' && success()
+ run: |
+ connector="airbyte/${{ env.IMAGE_NAME }}"
+ definitionpath=./airbyte-config/init/src/main/resources/seed/
+ sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml)
+ destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml)
+ if [[ (-z "$sourcecheck" && -z "$destcheck") ]]
+ then exit 1
+ fi
+ - name: Bump version in definitions yaml
+ if: github.event.inputs.auto-bump-version == 'true' && success()
+ run: |
+ connector="airbyte/${{ env.IMAGE_NAME }}"
+ definitionpath=./airbyte-config/init/src/main/resources/seed/
+ sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml)
+ destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml)
+ if [ -z "$sourcename" ]
+ then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml
+ else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml
+ fi
+ - name: Run gradle process changes
+ if: github.event.inputs.auto-bump-version == 'true' && success()
+ run: |
+ ./gradlew :airbyte-config:init:processResources
+ - name: git config
+ if: github.event.inputs.auto-bump-version == 'true' && success()
+ run: |
+ git config user.name 'Octavia Squidington III'
+ git config user.email 'octavia-squidington-iii@users.noreply.github.com'
+ - name: git commit and push
+ if: github.event.inputs.auto-bump-version == 'true' && success()
+ run: |
+ git add -u
+ git commit -m "auto-bump connector version"
+ git push origin ${{ github.ref }}
+ - name: Add Version Bump Success Comment
+ if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success()
+ uses: peter-evans/create-or-update-comment@v1
+ with:
+ comment-id: ${{ github.event.inputs.comment-id }}
+ body: |
+ > :rocket: Auto-bumped version for ${{github.event.inputs.connector}}
+ - name: Add Version Bump Failure Comment
+ if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success()
+ uses: peter-evans/create-or-update-comment@v1
+ with:
+ comment-id: ${{ github.event.inputs.comment-id }}
+ body: |
+ > :x: Couldn't auto-bump version for ${{github.event.inputs.connector}}
+ - name: Add Final Success Comment
+ if: github.event.inputs.comment-id && success()
+ uses: peter-evans/create-or-update-comment@v1
+ with:
+ comment-id: ${{ github.event.inputs.comment-id }}
+ body: |
+ > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
  # In case of self-hosted EC2 errors, remove this block.
  stop-publish-image-runner:
  name: Stop Build EC2 Runner

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -42,7 +42,7 @@
 - name: Apify Dataset
  sourceDefinitionId: 47f17145-fe20-4ef5-a548-e29b048adf84
  dockerRepository: airbyte/source-apify-dataset
- dockerImageTag: 0.1.3
+ dockerImageTag: 0.1.9
  documentationUrl: https://docs.airbyte.io/integrations/sources/apify-dataset
  icon: apify.svg
  sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -480,7 +480,7 @@
  supportsNormalization: false
  supportsDBT: false
  supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-apify-dataset:0.1.3"
+- dockerImage: "airbyte/source-apify-dataset:0.1.9"
  spec:
  documentationUrl: "https://docs.airbyte.io/integrations/sources/apify-dataset"
  connectionSpecification:

airbyte-integrations/connectors/destination-dev-null/Dockerfile

@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.2.3
+LABEL io.airbyte.version=0.2.6
 LABEL io.airbyte.name=airbyte/destination-dev-null

airbyte-integrations/connectors/source-apify-dataset/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.3
+LABEL io.airbyte.version=0.1.9
 LABEL io.airbyte.name=airbyte/source-apify-dataset

docs/connector-development/README.md

@@ -119,15 +119,17 @@ Once you've finished iterating on the changes to a connector as specified in its
  # Example: /test connector=connectors/source-hubspot
  /test connector=(connectors|bases)/<connector_name> 
 
- # to run integration tests and publish the connector
+ # to run integration tests, publish the connector and bump the connector version
  # Example: /publish connector=connectors/source-hubspot
  /publish connector=(connectors|bases)/<connector_name>
  ```
-4. Update the connector definition in the Airbyte connector index to use the new version:
- * `airbyte-config/init/src/main/resources/seed/source_definitions.yaml` if it is a source
- * `airbyte-config/init/src/main/resources/seed/destination_definitions.yaml` if it is a destination.
 
- Then run the commend `./gradlew :airbyte-config:init:processResources` to generate the seed spec yaml files, and commit the changes to the PR. See [this readme](https://github.com/airbytehq/airbyte/tree/a534bb2a8f29b20e3cc7c52fef1bc3c34783695d/airbyte-config/specs) for more information.
+4. OPTIONAL: Only necessary if the automated connector version bump fails
+ * Update the connector definition in the Airbyte connector index to use the new version:
+ * `airbyte-config/init/src/main/resources/seed/source_definitions.yaml` if it is a source
+ * `airbyte-config/init/src/main/resources/seed/destination_definitions.yaml` if it is a destination.
+
+ * Then run the command `./gradlew :airbyte-config:init:processResources` to generate the seed spec yaml files, and commit the changes to the PR. See [this readme](https://github.com/airbytehq/airbyte/tree/a534bb2a8f29b20e3cc7c52fef1bc3c34783695d/airbyte-config/specs) for more information.
 
 5. The new version of the connector is now available for everyone who uses it. Thank you!
 

docs/integrations/sources/apify-dataset.md

@@ -43,6 +43,7 @@ The Apify dataset connector uses [Apify Python Client](https://docs.apify.com/ap
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.9 | 2022-04-05 | [PR\#11712](https://github.com/airbytehq/airbyte/pull/11712) | No changes from 0.1.4. Used connector to test publish workflow changes. |
 | 0.1.4 | 2021-12-23 | [PR\#8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications |
 | 0.1.2 | 2021-11-08 | [PR\#7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies |
 | 0.1.0 | 2021-07-29 | [PR\#5069](https://github.com/airbytehq/airbyte/pull/5069) | Initial version of the connector |