fix(docker): Prevent all possible "silent errors" during docker build (antonbabenko#644)

--------- Co-authored-by: George L. Yermulnik <yz@yz.kiev.ua>

Author: MaxymVlasov

.dockerignore

@@ -2,3 +2,4 @@
 !.dockerignore
 !Dockerfile
 !tools/entrypoint.sh
+!tools/install/*.sh

.github/.container-structure-test-config.yaml

@@ -13,7 +13,7 @@ commandTests:
  - name: "terraform"
  command: "terraform"
  args: ["-version"]
- expectedOutput: ["^Terraform v([0-9]+\\.){2}[0-9]+\\non linux_amd64\\n$"]
+ expectedOutput: ["^Terraform v([0-9]+\\.){2}[0-9]+\\n"]
 
  - name: "gcc"
  command: "gcc"
@@ -28,12 +28,12 @@ commandTests:
  - name: "infracost"
  command: "infracost"
  args: ["--version"]
- expectedOutput: ["^Infracost v([0-9]+\\.){2}[0-9]+\\n$"]
+ expectedOutput: ["^Infracost v([0-9]+\\.){2}[0-9]+"]
 
  - name: "terraform-docs"
  command: "terraform-docs"
  args: ["--version"]
- expectedOutput: ["^terraform-docs version v([0-9]+\\.){2}[0-9]+ [a-z0-9]+ linux/amd64\\n$"]
+ expectedOutput: ["^terraform-docs version v([0-9]+\\.){2}[0-9]+ [a-z0-9]+"]
 
  - name: "terragrunt"
  command: "terragrunt"

.github/CONTRIBUTING.md

@@ -113,6 +113,9 @@ You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/
 
 ### Add code
 
+> [!TIP]
+> Here is a screencast of [how to add new dependency in `tools/install/`](https://github.com/antonbabenko/pre-commit-terraform/assets/11096782/8fc461e9-f163-4592-9497-4a18fa89c0e8) - used in Dockerfile
+
 1. Based on prev. block, add hook dependencies installation to [Dockerfile](../Dockerfile). 
  Check that works:
  * `docker build -t pre-commit --build-arg INSTALL_ALL=true .`

.github/workflows/build-image-test.yaml

@@ -27,6 +27,13 @@ jobs:
  .dockerignore
  tools/entrypoint.sh
  .github/workflows/build-image-test.yaml
+ tools/*.sh
+
+ - name: Set up QEMU
+ if: matrix.os != 'ubuntu-latest' || matrix.arch != 'amd64'
+ uses: docker/setup-qemu-action@v2
+ with:
+ platforms: 'arm64'
 
  - name: Set up Docker Buildx
  uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
@@ -38,7 +45,7 @@ jobs:
  context: .
  build-args: |
  INSTALL_ALL=true
- platforms: linux/amd64 # Only one allowed here, see https://github.com/docker/buildx/issues/59#issuecomment-1433097926
+ platforms: linux/${{ matrix.arch }} # Only one allowed here, see https://github.com/docker/buildx/issues/59#issuecomment-1433097926
  push: false
  load: true
  tags: |

.pre-commit-config.yaml

@@ -42,13 +42,14 @@ repos:
  hooks:
  - id: hadolint
  args: [
- '--ignore', 'DL3027', # Do not use apt
  '--ignore', 'DL3007', # Using latest
+ '--ignore', 'DL3013', # Pin versions in pip
+ '--ignore', 'DL3027', # Do not use apt
+ '--ignore', 'DL3059', # Docker `RUN`s shouldn't be consolidated here
  '--ignore', 'DL4006', # Not related to alpine
  '--ignore', 'SC1091', # Useless check
  '--ignore', 'SC2015', # Useless check
  '--ignore', 'SC3037', # Not related to alpine
- '--ignore', 'DL3013', # Pin versions in pip
  ]
 
 # JSON5 Linter

Dockerfile

@@ -7,209 +7,89 @@ WORKDIR /bin_dir
 
 RUN apk add --no-cache \
  # Builder deps
+ bash=~5 \
  curl=~8 && \
  # Upgrade packages for be able get latest Checkov
  python3 -m pip install --no-cache-dir --upgrade \
  pip \
  setuptools
 
+COPY tools/install/ /install/
+
+#
+# Install required tools
+#
 ARG PRE_COMMIT_VERSION=${PRE_COMMIT_VERSION:-latest}
 ARG TERRAFORM_VERSION=${TERRAFORM_VERSION:-latest}
 
-# Install pre-commit
-RUN if [ ${PRE_COMMIT_VERSION} = "latest" ]; \
- then pip3 install --no-cache-dir pre-commit; \
- else pip3 install --no-cache-dir pre-commit==${PRE_COMMIT_VERSION}; \
+RUN touch /.env && \
+ if [ "$PRE_COMMIT_VERSION" = "false" ] || [ "$TERRAFORM_VERSION" = "false" ]; then \
+ echo "Vital software can't be skipped" && exit 1; \
  fi
 
-# Install terraform because pre-commit needs it
-RUN if [ "${TERRAFORM_VERSION}" = "latest" ]; then \
- TERRAFORM_VERSION="$(curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | grep -o -E -m 1 "[0-9.]+")" \
- ; fi && \
- curl -L "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip" > terraform.zip && \
- unzip terraform.zip terraform && rm terraform.zip
+
+RUN /install/pre-commit.sh
+RUN /install/terraform.sh
 
 #
 # Install tools
 #
 ARG CHECKOV_VERSION=${CHECKOV_VERSION:-false}
+ARG HCLEDIT_VERSION=${HCLEDIT_VERSION:-false}
 ARG INFRACOST_VERSION=${INFRACOST_VERSION:-false}
 ARG TERRAFORM_DOCS_VERSION=${TERRAFORM_DOCS_VERSION:-false}
 ARG TERRAGRUNT_VERSION=${TERRAGRUNT_VERSION:-false}
 ARG TERRASCAN_VERSION=${TERRASCAN_VERSION:-false}
 ARG TFLINT_VERSION=${TFLINT_VERSION:-false}
 ARG TFSEC_VERSION=${TFSEC_VERSION:-false}
-ARG TRIVY_VERSION=${TRIVY_VERSION:-false}
 ARG TFUPDATE_VERSION=${TFUPDATE_VERSION:-false}
-ARG HCLEDIT_VERSION=${HCLEDIT_VERSION:-false}
+ARG TRIVY_VERSION=${TRIVY_VERSION:-false}
 
 
 # Tricky thing to install all tools by set only one arg.
 # In RUN command below used `. /.env` <- this is sourcing vars that
 # specified in step below
 ARG INSTALL_ALL=${INSTALL_ALL:-false}
 RUN if [ "$INSTALL_ALL" != "false" ]; then \
- echo "export CHECKOV_VERSION=latest" >> /.env && \
- echo "export INFRACOST_VERSION=latest" >> /.env && \
- echo "export TERRAFORM_DOCS_VERSION=latest" >> /.env && \
- echo "export TERRAGRUNT_VERSION=latest" >> /.env && \
- echo "export TERRASCAN_VERSION=latest" >> /.env && \
- echo "export TFLINT_VERSION=latest" >> /.env && \
- echo "export TFSEC_VERSION=latest" >> /.env && \
- echo "export TRIVY_VERSION=latest" >> /.env && \
- echo "export TFUPDATE_VERSION=latest" >> /.env && \
- echo "export HCLEDIT_VERSION=latest" >> /.env \
- ; else \
- touch /.env \
+ echo "CHECKOV_VERSION=latest" >> /.env && \
+ echo "HCLEDIT_VERSION=latest" >> /.env && \
+ echo "INFRACOST_VERSION=latest" >> /.env && \
+ echo "TERRAFORM_DOCS_VERSION=latest" >> /.env && \
+ echo "TERRAGRUNT_VERSION=latest" >> /.env && \
+ echo "TERRASCAN_VERSION=latest" >> /.env && \
+ echo "TFLINT_VERSION=latest" >> /.env && \
+ echo "TFSEC_VERSION=latest" >> /.env && \
+ echo "TFUPDATE_VERSION=latest" >> /.env && \
+ echo "TRIVY_VERSION=latest" >> /.env \
  ; fi
 
-
-# Checkov
-RUN . /.env && \
- if [ "$CHECKOV_VERSION" != "false" ]; then \
- ( \
- # cargo, gcc, git, musl-dev, rust and CARGO envvar required for compilation of rustworkx@0.13.2, no longer required once checkov version depends on rustworkx >0.14.0
- # https://github.com/bridgecrewio/checkov/pull/6045
- # gcc libffi-dev musl-dev required for compilation of cffi, until it contains musl aarch64
- export CARGO_NET_GIT_FETCH_WITH_CLI=true && \
- apk add --no-cache cargo=~1 gcc=~12 git=~2 libffi-dev=~3 libgcc=~12 musl-dev=~1 rust=~1 ; \
- if [ "$CHECKOV_VERSION" = "latest" ]; \
- then pip3 install --no-cache-dir checkov || exit 1; \
- else pip3 install --no-cache-dir checkov==${CHECKOV_VERSION} || exit 1; \
- fi; \
- apk del cargo gcc git libffi-dev musl-dev rust \
- ) \
- ; fi
-
-# infracost
-RUN . /.env && \
- if [ "$INFRACOST_VERSION" != "false" ]; then \
- ( \
- INFRACOST_RELEASES="https://api.github.com/repos/infracost/infracost/releases" && \
- if [ "$INFRACOST_VERSION" = "latest" ]; \
- then curl -L "$(curl -s ${INFRACOST_RELEASES}/latest | grep -o -E -m 1 "https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz")" > infracost.tgz; \
- else curl -L "$(curl -s ${INFRACOST_RELEASES} | grep -o -E "https://.+?v${INFRACOST_VERSION}/infracost-${TARGETOS}-${TARGETARCH}.tar.gz")" > infracost.tgz; \
- fi; \
- ) && tar -xzf infracost.tgz && rm infracost.tgz && mv infracost-${TARGETOS}-${TARGETARCH} infracost \
- ; fi
-
-# Terraform docs
-RUN . /.env && \
- if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then \
- ( \
- TERRAFORM_DOCS_RELEASES="https://api.github.com/repos/terraform-docs/terraform-docs/releases" && \
- if [ "$TERRAFORM_DOCS_VERSION" = "latest" ]; \
- then curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES}/latest | grep -o -E -m 1 "https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz")" > terraform-docs.tgz; \
- else curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES} | grep -o -E "https://.+?v${TERRAFORM_DOCS_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz")" > terraform-docs.tgz; \
- fi; \
- ) && tar -xzf terraform-docs.tgz terraform-docs && rm terraform-docs.tgz && chmod +x terraform-docs \
- ; fi
-
-# Terragrunt
-RUN . /.env \
- && if [ "$TERRAGRUNT_VERSION" != "false" ]; then \
- ( \
- TERRAGRUNT_RELEASES="https://api.github.com/repos/gruntwork-io/terragrunt/releases" && \
- if [ "$TERRAGRUNT_VERSION" = "latest" ]; \
- then curl -L "$(curl -s ${TERRAGRUNT_RELEASES}/latest | grep -o -E -m 1 "https://.+?/terragrunt_${TARGETOS}_${TARGETARCH}")" > terragrunt; \
- else curl -L "$(curl -s ${TERRAGRUNT_RELEASES} | grep -o -E -m 1 "https://.+?v${TERRAGRUNT_VERSION}/terragrunt_${TARGETOS}_${TARGETARCH}")" > terragrunt; \
- fi; \
- ) && chmod +x terragrunt \
- ; fi
+RUN /install/checkov.sh
+RUN /install/hcledit.sh
+RUN /install/infracost.sh
+RUN /install/terraform-docs.sh
+RUN /install/terragrunt.sh
+RUN /install/terrascan.sh
+RUN /install/tflint.sh
+RUN /install/tfsec.sh
+RUN /install/tfupdate.sh
+RUN /install/trivy.sh
 
 
-# Terrascan
-RUN . /.env && \
- if [ "$TERRASCAN_VERSION" != "false" ]; then \
- if [ "$TARGETARCH" != "amd64" ]; then ARCH="$TARGETARCH"; else ARCH="x86_64"; fi; \
- # Convert the first letter to Uppercase
- OS="$(echo ${TARGETOS} | cut -c1 | tr '[:lower:]' '[:upper:]' | xargs echo -n; echo ${TARGETOS} | cut -c2-)"; \
- ( \
- TERRASCAN_RELEASES="https://api.github.com/repos/tenable/terrascan/releases" && \
- if [ "$TERRASCAN_VERSION" = "latest" ]; \
- then curl -L "$(curl -s ${TERRASCAN_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${OS}_${ARCH}.tar.gz")" > terrascan.tar.gz; \
- else curl -L "$(curl -s ${TERRASCAN_RELEASES} | grep -o -E "https://.+?${TERRASCAN_VERSION}_${OS}_${ARCH}.tar.gz")" > terrascan.tar.gz; \
- fi; \
- ) && tar -xzf terrascan.tar.gz terrascan && rm terrascan.tar.gz && \
- ./terrascan init \
- ; fi
-
-# TFLint
-RUN . /.env && \
- if [ "$TFLINT_VERSION" != "false" ]; then \
- ( \
- TFLINT_RELEASES="https://api.github.com/repos/terraform-linters/tflint/releases" && \
- if [ "$TFLINT_VERSION" = "latest" ]; \
- then curl -L "$(curl -s ${TFLINT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.zip")" > tflint.zip; \
- else curl -L "$(curl -s ${TFLINT_RELEASES} | grep -o -E "https://.+?/v${TFLINT_VERSION}/tflint_${TARGETOS}_${TARGETARCH}.zip")" > tflint.zip; \
- fi; \
- ) && unzip tflint.zip && rm tflint.zip \
- ; fi
-
-# TFSec
-RUN . /.env && \
- if [ "$TFSEC_VERSION" != "false" ]; then \
- ( \
- TFSEC_RELEASES="https://api.github.com/repos/aquasecurity/tfsec/releases" && \
- if [ "$TFSEC_VERSION" = "latest" ]; then \
- curl -L "$(curl -s ${TFSEC_RELEASES}/latest | grep -o -E -m 1 "https://.+?/tfsec-${TARGETOS}-${TARGETARCH}")" > tfsec; \
- else curl -L "$(curl -s ${TFSEC_RELEASES} | grep -o -E -m 1 "https://.+?v${TFSEC_VERSION}/tfsec-${TARGETOS}-${TARGETARCH}")" > tfsec; \
- fi; \
- ) && chmod +x tfsec \
- ; fi
-
-# Trivy
-RUN . /.env && \
- if [ "$TRIVY_VERSION" != "false" ]; then \
- if [ "$TARGETARCH" != "amd64" ]; then ARCH="$TARGETARCH"; else ARCH="64bit"; fi; \
- ( \
- TRIVY_RELEASES="https://api.github.com/repos/aquasecurity/trivy/releases" && \
- if [ "$TRIVY_VERSION" = "latest" ]; \
- then curl -L "$(curl -s ${TRIVY_RELEASES}/latest | grep -o -E -i -m 1 "https://.+?/trivy_.+?_${TARGETOS}-${ARCH}.tar.gz")" > trivy.tar.gz; \
- else curl -L "$(curl -s ${TRIVY_RELEASES} | grep -o -E -i -m 1 "https://.+?/v${TRIVY_VERSION}/trivy_.+?_${TARGETOS}-${ARCH}.tar.gz")" > trivy.tar.gz; \
- fi; \
- ) && tar -xzf trivy.tar.gz trivy && rm trivy.tar.gz \
- ; fi
-
-# TFUpdate
-RUN . /.env && \
- if [ "$TFUPDATE_VERSION" != "false" ]; then \
- ( \
- TFUPDATE_RELEASES="https://api.github.com/repos/minamijoyo/tfupdate/releases" && \
- if [ "$TFUPDATE_VERSION" = "latest" ]; \
- then curl -L "$(curl -s ${TFUPDATE_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz")" > tfupdate.tgz; \
- else curl -L "$(curl -s ${TFUPDATE_RELEASES} | grep -o -E -m 1 "https://.+?${TFUPDATE_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz")" > tfupdate.tgz; \
- fi; \
- ) && tar -xzf tfupdate.tgz tfupdate && rm tfupdate.tgz \
- ; fi
-
-# hcledit
-RUN . /.env && \
- if [ "$HCLEDIT_VERSION" != "false" ]; then \
- ( \
- HCLEDIT_RELEASES="https://api.github.com/repos/minamijoyo/hcledit/releases" && \
- if [ "$HCLEDIT_VERSION" = "latest" ]; \
- then curl -L "$(curl -s ${HCLEDIT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz")" > hcledit.tgz; \
- else curl -L "$(curl -s ${HCLEDIT_RELEASES} | grep -o -E -m 1 "https://.+?${HCLEDIT_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz")" > hcledit.tgz; \
- fi; \
- ) && tar -xzf hcledit.tgz hcledit && rm hcledit.tgz \
- ; fi
-
 # Checking binaries versions and write it to debug file
 RUN . /.env && \
  F=tools_versions_info && \
  pre-commit --version >> $F && \
  ./terraform --version | head -n 1 >> $F && \
  (if [ "$CHECKOV_VERSION" != "false" ]; then echo "checkov $(checkov --version)" >> $F; else echo "checkov SKIPPED" >> $F ; fi) && \
+ (if [ "$HCLEDIT_VERSION" != "false" ]; then echo "hcledit $(./hcledit version)" >> $F; else echo "hcledit SKIPPED" >> $F ; fi) && \
  (if [ "$INFRACOST_VERSION" != "false" ]; then echo "$(./infracost --version)" >> $F; else echo "infracost SKIPPED" >> $F ; fi) && \
  (if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then ./terraform-docs --version >> $F; else echo "terraform-docs SKIPPED" >> $F ; fi) && \
  (if [ "$TERRAGRUNT_VERSION" != "false" ]; then ./terragrunt --version >> $F; else echo "terragrunt SKIPPED" >> $F ; fi) && \
  (if [ "$TERRASCAN_VERSION" != "false" ]; then echo "terrascan $(./terrascan version)" >> $F; else echo "terrascan SKIPPED" >> $F ; fi) && \
  (if [ "$TFLINT_VERSION" != "false" ]; then ./tflint --version >> $F; else echo "tflint SKIPPED" >> $F ; fi) && \
  (if [ "$TFSEC_VERSION" != "false" ]; then echo "tfsec $(./tfsec --version)" >> $F; else echo "tfsec SKIPPED" >> $F ; fi) && \
- (if [ "$TRIVY_VERSION" != "false" ]; then echo "trivy $(./trivy --version)" >> $F; else echo "trivy SKIPPED" >> $F ; fi) && \
  (if [ "$TFUPDATE_VERSION" != "false" ]; then echo "tfupdate $(./tfupdate --version)" >> $F; else echo "tfupdate SKIPPED" >> $F ; fi) && \
- (if [ "$HCLEDIT_VERSION" != "false" ]; then echo "hcledit $(./hcledit version)" >> $F; else echo "hcledit SKIPPED" >> $F ; fi) && \
+ (if [ "$TRIVY_VERSION"  != "false" ]; then echo "trivy $(./trivy --version)" >> $F;  else echo "trivy SKIPPED" >> $F  ; fi) && \
  echo -e "\n\n" && cat $F && echo -e "\n\n"
 
 

hooks/infracost_breakdown.sh

@@ -2,8 +2,8 @@
 set -eo pipefail
 
 # globals variables
-# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines
-readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+readonly SCRIPT_DIR
 # shellcheck source=_common.sh
 . "$SCRIPT_DIR/_common.sh"
 

hooks/terraform_checkov.sh

@@ -2,8 +2,8 @@
 set -eo pipefail
 
 # globals variables
-# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines
-readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+readonly SCRIPT_DIR
 # shellcheck source=_common.sh
 . "$SCRIPT_DIR/_common.sh"
 

hooks/terraform_docs.sh

@@ -2,8 +2,8 @@
 set -eo pipefail
 
 # globals variables
-# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines
-readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+readonly SCRIPT_DIR
 # shellcheck source=_common.sh
 . "$SCRIPT_DIR/_common.sh"
 

hooks/terraform_fmt.sh

@@ -2,8 +2,8 @@
 set -eo pipefail
 
 # globals variables
-# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines
-readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+readonly SCRIPT_DIR
 # shellcheck source=_common.sh
 . "$SCRIPT_DIR/_common.sh"