ci: Update ACN Pipeline for Compliance

.gitignore

@@ -10,6 +10,7 @@ ipam-*.xml
 
 # Environment
 .vscode/*
+**/*.sw?
 
 # Coverage 
 *.out

.pipelines/build/binaries.jobs.yaml

@@ -0,0 +1,45 @@
+parameters:
+- name: binaries
+ type: jobList
+
+
+jobs:
+- ${{ each job_data in parameters.binaries }}:
+ - ${{ if eq(job_data.templateContext.action, 'build') }}:
+ - job: binaries_${{ job_data.job }}
+ displayName: "Build Binary - ${{ job_data.displayName }} -"
+ strategy: ${{ job_data.strategy }}
+ pool:
+ type: linux
+ ${{ if eq(job_data.job, 'linux_arm64') }}:
+ hostArchitecture: arm64
+
+ variables:
+ ob_outputDirectory: $(Build.ArtifactStagingDirectory)/out
+ ob_artifactSuffix: _$(artifact)
+ ob_git_checkout: false
+ REPO_ROOT: $(Build.SourcesDirectory)/${{ job_data.templateContext.repositoryArtifact }}
+ ${{ if eq(job_data.job, 'linux_amd64') }}:
+ LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest'
+ ARCH: amd64
+ OS: linux
+ ${{ elseif eq(job_data.job, 'windows_amd64') }}:
+ LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest'
+ ARCH: amd64
+ OS: windows
+ ${{ elseif eq(job_data.job, 'linux_arm64') }}:
+ ob_enable_qemu: true
+ ARCH: arm64
+ OS: linux
+
+ steps:
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ targetPath: $(Build.SourcesDirectory)/${{ job_data.templateContext.repositoryArtifact }}
+ artifact: '${{ job_data.templateContext.repositoryArtifact }}'
+
+ - template: binary.steps.yaml
+ parameters:
+ target: $(name)
+ os: $(OS)
+ arch: $(ARCH)

.pipelines/build/binary.steps.yaml

@@ -0,0 +1,72 @@
+parameters:
+- name: target
+ type: string
+
+- name: os
+ type: string
+
+- name: arch
+ type: string
+
+
+steps:
+- task: GoTool@0
+ inputs:
+ version: '$(GOVERSION)'
+
+- bash: |
+ # Ubuntu
+ if [[ -f /etc/debian_version ]];then
+ sudo apt-get update -y
+ if [[ $GOARCH =~ amd64 ]]; then
+ sudo apt-get install -y llvm clang linux-libc-dev linux-headers-generic libbpf-dev libc6-dev nftables iproute2 gcc-multilib tree
+ for dir in /usr/include/x86_64-linux-gnu/*; do 
+ sudo ln -sfn "$dir" /usr/include/$(basename "$dir") 
+ done
+
+ elif [[ $GOARCH =~ arm64 ]]; then
+ sudo apt-get install -y llvm clang linux-libc-dev linux-headers-generic libbpf-dev libc6-dev nftables iproute2 gcc-aarch64-linux-gnu tree
+ for dir in /usr/include/aarch64-linux-gnu/*; do 
+ sudo ln -sfn "$dir" /usr/include/$(basename "$dir")
+ done
+ fi
+ # Mariner
+ else
+ sudo tdnf install -y llvm clang libbpf-devel nftables tree
+ for dir in /usr/include/aarch64-linux-gnu/*; do 
+ if [[ -d $dir ]]; then
+ sudo ln -sfn "$dir" /usr/include/$(basename "$dir") 
+ elif [[ -f "$dir" ]]; then
+ sudo ln -Tsfn "$dir" /usr/include/$(basename "$dir") 
+ fi
+ done
+ fi
+ displayName: "Install Binary Pre-Reqs"
+ workingDirectory: $(ACN_DIR)
+ continueOnError: true
+ env:
+ GOOS: ${{ parameters.os }}
+ GOARCH: ${{ parameters.arch }}
+
+- bash: |
+ make "$TARGET"
+ displayName: "Build Binary - ${{ parameters.target }}"
+ workingDirectory: $(ACN_DIR)
+ env:
+ REPO_ROOT: $(ACN_DIR)
+ TARGET: ${{ parameters.target }}
+ GOOS: ${{ parameters.os }}
+ GOARCH: ${{ parameters.arch }}
+
+- script: |
+ SOURCE_DIR="./output"
+ TARGET_DIR="$BUILD_ARTIFACTSTAGINGDIRECTORY"/out
+ tree "$SOURCE_DIR"
+
+ mkdir -p "$TARGET_DIR"
+ find "$SOURCE_DIR" -name '*.tgz*' -print -exec mv -t "$TARGET_DIR"/ {} +
+ find "$SOURCE_DIR" -name '*.zip' -print -exec mv -t "$TARGET_DIR"/ {} +
+
+ tree "$TARGET_DIR"
+ displayName: "Prepare Artifacts"
+ workingDirectory: $(ACN_DIR)

.pipelines/build/generate-manifest.steps.yaml

@@ -0,0 +1,23 @@
+parameters:
+- name: platforms
+ type: object
+ default: []
+
+
+steps:
+- bash: |
+ set -e
+ MANIFEST_DATA=$(echo "$IMAGE_PLATFORM_DATA" | \
+ jq -r '.[] | 
+ .args = [ (.platform | split("/")[0]), (.platform | split("/")[1]) ] | 
+ .args = [ ("--os " + .args[0] ), ("--arch " + .args[1] ) ] | 
+ if .osVersion then .args += ["--os-version " + .osVersion] else . end |
+ { image: .imageReference, annotate: .args }' | \
+ jq -rcs)
+ echo >&2 "##vso[task.setvariable variable=MANIFEST_JSON;isOutput=true]$MANIFEST_DATA"
+ echo "$MANIFEST_DATA" | jq -r .
+ displayName: "Populate Image Manifest Data"
+ name: data
+ env:
+ IMAGE_PLATFORM_DATA: '${{ convertToJson(parameters.platforms) }}'
+

.pipelines/build/image.steps.yaml

@@ -0,0 +1,89 @@
+parameters:
+- name: arch
+ type: string
+ default: ""
+
+- name: name
+ type: string
+ default: ""
+
+- name: os
+ type: string
+ default: ""
+
+- name: build_tag
+ type: string
+ default: ""
+
+- name: dockerfile_path
+ type: string
+ default: ""
+
+- name: archive_file
+ type: string
+ default: '$(name)-$(os)-$(platform)-$(Tag)'
+
+- name: source
+ type: string
+ default: drop_setup_env_source
+
+- name: extra_args
+ type: string
+ default: ''
+
+
+- name: default_args
+ type: object
+ default:
+ - "--target $(os) "
+ - "--platform $(os)/$(arch) "
+
+- name: common_build_args
+ type: object
+ default:
+ - "PLATFORM=$(os)/$(arch) "
+ - "ARCH=$(arch) "
+ - "OS=$(os) "
+ - "VERSION=$(Tag) "
+
+
+steps:
+- task: DownloadPipelineArtifact@2
+ inputs:
+ targetPath: $(Build.SourcesDirectory)/dst/${{ parameters.source }}
+ artifact: '${{ parameters.source }}'
+
+- task: onebranch.pipeline.containercontrol@1
+ displayName: "Login to ACR"
+ inputs:
+ command: login
+ endpoint: $(ACR_ARM_SERVICE_CONNECTION)
+
+# Build and push the Docker image
+- task: onebranch.pipeline.imagebuildinfo@1
+ displayName: Image Build
+ retryCountOnTaskFailure: 3
+ timeoutInMinutes: 30
+ inputs:
+ endpoint: $(ACR_ARM_SERVICE_CONNECTION)
+ registry: $(ACR).azurecr.io
+ repositoryName: $(os)-$(arch)/${{ parameters.name }}
+ os: '${{ parameters.os }}'
+ buildkit: 1
+ dockerFileRelPath: ${{ parameters.dockerfile_path }}/Dockerfile
+ dockerFileContextPath: ${{ parameters.source }}
+ enable_network: true
+ enable_pull: true
+ build_tag: ${{ parameters.build_tag }}
+ enable_acr_push: true
+
+ saveImageToPath: images/$(os)-$(arch)/${{ parameters.archive_file }}.tar.gz
+ #compress: true
+ #saveMetadataToPath: images/$(os)-$(arch)/metadata/${{ parameters.archive_file }}-metadata.json
+ #enable_isolated_acr_push: true
+
+ # Docker Build Arguments
+ ${{ if parameters.common_build_args }}:
+ arguments: --build-arg ${{ join('--build-arg ', parameters.common_build_args) }} ${{ parameters.extra_args }} ${{ join(' ', parameters.default_args) }}
+ ${{ else }}:
+ arguments: ${{ parameters.extra_args }} ${{ join(' ', parameters.default_args) }}

.pipelines/build/manifests.jobs.yaml

@@ -0,0 +1,38 @@
+parameters:
+- name: generate
+ type: jobList
+
+
+jobs:
+- ${{ each job_data in parameters.generate }}:
+ - job: ${{ job_data.job }}_generate_manifest
+ displayName: "Generate Image Manifest - ${{ job_data.job }}"
+ pool:
+ type: linux
+ variables:
+ ob_outputDirectory: $(Build.SourcesDirectory)/out
+ ob_git_checkout: false
+ steps:
+ - template: /.pipelines/build/generate-manifest.steps.yaml
+ parameters:
+ platforms: ${{ job_data.templateContext.platforms }}
+
+ - job: ${{ job_data.job }}_publish_manifest
+ displayName: "Publish Image Manifest - ${{ job_data.job }}"
+ dependsOn:
+ - ${{ job_data.job }}_generate_manifest
+ pool:
+ type: docker
+ os: linux
+ variables:
+ LinuxContainerImage: 'mcr.microsoft.com/onebranch/azurelinux/build:3.0'
+ ob_outputDirectory: $(Build.SourcesDirectory)/out
+ ob_git_checkout: false
+
+ MANIFEST_JSON: $[ dependencies.${{ job_data.job }}_generate_manifest.outputs['data.MANIFEST_JSON'] ]
+ steps:
+ - template: /.pipelines/build/publish-manifest.steps.yaml
+ parameters:
+ image_repository: ${{ job_data.templateContext.name }}
+ image_tag: ${{ job_data.templateContext.image_tag }}
+ manifest_data: $(MANIFEST_JSON)