Fix Multi Arch Container Image builds (#407)

* create buildx context

* build image on ci runs, but dont push when not a release

* use the context

* fetch plugins during container build

* setup experimental

* Revert experimental, it is set with env

* use docker buildx org for installing buildx and qemu

* fetch plugins before building container image

* update gcloud cli

* run buildx org install command

* gcloud org version typo

* sensu docker buildx orb has some issues, so lets base our implementation on their source

* Add qemu version

* create docker context

* Set gcloud sdk version

* use context when creating buildx builder

* fix gcloud syntax

* always fetch plugins

* use large builder for docker

* ignore entire artifacts dir

* use plan docker output

* fix plugin install by adding them to the stage build

* use larger instance and increase timeout for build

* we dont need fetch plugins for build anymore

* increase timeout

* build finished in 27 minutes, so increase timeout from 30 to 45m

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 
 orbs:
   win: circleci/windows@2.4.0
-  gcp-cli: circleci/gcp-cli@1.8.4
+  gcp-cli: circleci/gcp-cli@2.2.0
   codecov: codecov/codecov@1.0.2
   pr-comment: benjlevesque/pr-comment@0.1.4
 
@@ -34,6 +34,64 @@ jobs:
           root: ./artifacts
           paths:
             - "*"
+  build-docker:
+    resource_class: large
+    docker:
+      - image: circleci/buildpack-deps:stretch
+    steps:
+      - checkout
+      - setup_remote_docker:
+          version: 19.03.14
+      - attach_workspace:
+          at: ./artifacts
+      - run:
+          name: Install Docker buildx
+          command: |
+            mkdir -p ~/.docker/cli-plugins
+            curl -sSL -o ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.6.2/buildx-v0.6.2.linux-amd64
+            chmod a+x ~/.docker/cli-plugins/docker-buildx
+      - run:
+          name: Enable experimental Docker features
+          command: echo 'export DOCKER_CLI_EXPERIMENTAL="enabled"' >> $BASH_ENV
+      - run:
+          name: Create Docker context
+          command: docker context create tls-environment
+      - run:
+          name: Initialize Docker buildx
+          command: docker buildx install
+      - run:
+          name: Start multiarch/qemu-user-static container
+          command: >
+            docker run --rm --privileged
+            multiarch/qemu-user-static:5.2.0-2
+            --reset -p yes
+      - run:
+          name: Remove buildx multiarch container in case it exists
+          command: docker rm -f buildx_buildkit_docker-multiarch0 || continue
+      - run:
+          name: Create docker-multiarch builder
+          command: >
+            docker buildx create --name docker-multiarch tls-environment
+            --platform linux/386,linux/amd64,linux/arm/v5,linux/arm/v6,linux/arm/v7,linux/arm64,linux/mips64le,linux/ppc64le,linux/riscv64,linux/s390x
+      - run:
+          name: Inspect & bootstrap docker-multiarch builder
+          command: docker buildx inspect --builder docker-multiarch --bootstrap
+      - run:
+          name: Set docker-multiarch as default builder
+          command: docker buildx use docker-multiarch
+      - run:
+          name: "Build container image"
+          no_output_timeout: 45m
+          command: |
+            docker_tag=$(echo ${CIRCLE_TAG} | cut -b2- )
+            docker login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_TOKEN}
+            export DOCKER_CLI_EXPERIMENTAL=enabled
+
+            docker buildx build \
+              --progress=plain \
+              --platform linux/amd64,linux/arm64 \
+              --tag observiq/stanza:latest \
+              .
 
   fetch-plugins:
     executor: golang
@@ -72,29 +130,61 @@ jobs:
             ghr -t ${GHI_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} ${CIRCLE_TAG} ./artifacts/
 
   publish-docker-release:
+    # building amd64 + arm64 is compute intensive
+    resource_class: 2xlarge
     docker:
       - image: circleci/buildpack-deps:stretch
     steps:
       - checkout
-      - setup_remote_docker
+      - setup_remote_docker:
+          version: 19.03.14
       - attach_workspace:
           at: ./artifacts
       - run:
-          name: "Configure Docker multi arch builds"
+          name: Install Docker buildx
           command: |
             mkdir -p ~/.docker/cli-plugins
-            curl -fL https://github.com/docker/buildx/releases/download/v0.6.1/buildx-v0.6.1.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx
+            curl -sSL -o ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.6.2/buildx-v0.6.2.linux-amd64
             chmod a+x ~/.docker/cli-plugins/docker-buildx
-            docker buildx create --name stanza --use
-            docker buildx inspect --bootstrap
+      - run:
+          name: Enable experimental Docker features
+          command: echo 'export DOCKER_CLI_EXPERIMENTAL="enabled"' >> $BASH_ENV
+      - run:
+          name: Create Docker context
+          command: docker context create tls-environment
+      - run:
+          name: Initialize Docker buildx
+          command: docker buildx install
+      - run:
+          name: Start multiarch/qemu-user-static container
+          command: >
+            docker run --rm --privileged
+            multiarch/qemu-user-static:5.2.0-2
+            --reset -p yes
+      - run:
+          name: Remove buildx multiarch container in case it exists
+          command: docker rm -f buildx_buildkit_docker-multiarch0 || continue
+      - run:
+          name: Create docker-multiarch builder
+          command: >
+            docker buildx create --name docker-multiarch tls-environment
+            --platform linux/386,linux/amd64,linux/arm/v5,linux/arm/v6,linux/arm/v7,linux/arm64,linux/mips64le,linux/ppc64le,linux/riscv64,linux/s390x
+      - run:
+          name: Inspect & bootstrap docker-multiarch builder
+          command: docker buildx inspect --builder docker-multiarch --bootstrap
+      - run:
+          name: Set docker-multiarch as default builder
+          command: docker buildx use docker-multiarch
       - run:
           name: "Publish Release on Docker Hub"
+          no_output_timeout: 45m
           command: |
             docker_tag=$(echo ${CIRCLE_TAG} | cut -b2- )
             docker login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_TOKEN}
             export DOCKER_CLI_EXPERIMENTAL=enabled
 
             docker buildx build \
+              --progress=plain \
               --platform linux/amd64,linux/arm64 \
               --tag observiq/stanza:${docker_tag} \
               --tag observiq/stanza:latest \
@@ -441,6 +531,12 @@ workflows:
           filters:
             tags:
               only: /^v\d+\.\d+\.\d+.*/
+      - build-docker:
+          filters:
+            tags:
+              only: /^v\d+\.\d+\.\d+.*/
+          requires:
+            - build
       - fetch-plugins:
           filters:
             branches:

.dockerignore

@@ -1,4 +1,3 @@
-artifacts/stanza_darwin_amd64
-artifacts/stanza_windows_amd64
+artifacts/
 .git/
 dev/

Dockerfile

@@ -1,22 +1,31 @@
 FROM golang:1.16 as stage
 
+ARG plugins_url="https://github.com/observiq/stanza-plugins/releases/latest/download/stanza-plugins.zip"
+# arm cross builds do not have these symlinks in palce
+RUN \
+    ln -s /usr/bin/dpkg-split /usr/sbin/dpkg-split && \
+    ln -s /usr/bin/dpkg-deb /usr/sbin/dpkg-deb && \
+    ln -s /bin/tar /usr/sbin/tar && \
+    ln -s /bin/rm /usr/sbin/rm && \
+    echo "resolvconf resolvconf/linkify-resolvconf boolean false" | debconf-set-selections
+# unzip is required because tar does not work on arm
+RUN apt-get update && apt-get install unzip -y
+WORKDIR /stanza/artifacts
+RUN curl -fL "${plugins_url}" -o stanza-plugins.zip
+RUN unzip stanza-plugins.zip
 WORKDIR /stanza
 COPY . .
-RUN rm -rf artifacts/*
 RUN make build
-WORKDIR /stanza/artifacts
-# hack: "mv stanza_* stanza" gives an error because mv does not like '_*'
-RUN for f in stanza_*; do mv "$f" stanza; done
+RUN mv "artifacts/stanza_$(go env GOOS)_$(go env GOARCH)" artifacts/stanza
+
 
 FROM gcr.io/observiq-container-images/stanza-base:v1.1.0
 
 RUN mkdir -p /stanza_home
 ENV STANZA_HOME=/stanza_home
 RUN echo "pipeline:\n" >> /stanza_home/config.yaml
-
 COPY --from=stage /stanza/artifacts/stanza /stanza_home/stanza
-COPY ./artifacts/stanza-plugins.tar.gz /tmp/stanza-plugins.tar.gz
-RUN tar -zxvf /tmp/stanza-plugins.tar.gz -C /stanza_home/
+COPY --from=stage /stanza/artifacts/plugins /stanza_home/plugins
 ENTRYPOINT /stanza_home/stanza \
   --config /stanza_home/config.yaml \
   --database /stanza_home/stanza.db \