From b6356b6f3bf42a915c4e56fcc05b0c3681db493a Mon Sep 17 00:00:00 2001 From: Joseph Sirianni Date: Wed, 25 Aug 2021 12:39:18 -0400 Subject: [PATCH] Fix Multi Arch Container Image builds (#407) * create buildx context * build image on ci runs, but dont push when not a release * use the context * fetch plugins during container build * setup experimental * Revert experimental, it is set with env * use docker buildx org for installing buildx and qemu * fetch plugins before building container image * update gcloud cli * run buildx org install command * gcloud org version typo * sensu docker buildx orb has some issues, so lets base our implementation on their source * Add qemu version * create docker context * Set gcloud sdk version * use context when creating buildx builder * fix gcloud syntax * always fetch plugins * use large builder for docker * ignore entire artifacts dir * use plan docker output * fix plugin install by adding them to the stage build * use larger instance and increase timeout for build * we dont need fetch plugins for build anymore * increase timeout * build finished in 27 minutes, so increase timeout from 30 to 45m --- .circleci/config.yml | 108 ++++++++++++++++++++++++++++++++++++++++--- .dockerignore | 3 +- Dockerfile | 23 ++++++--- 3 files changed, 119 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dec37b420..e00a05c75 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ version: 2.1 orbs: win: circleci/windows@2.4.0 - gcp-cli: circleci/gcp-cli@1.8.4 + gcp-cli: circleci/gcp-cli@2.2.0 codecov: codecov/codecov@1.0.2 pr-comment: benjlevesque/pr-comment@0.1.4 @@ -34,6 +34,64 @@ jobs: root: ./artifacts paths: - "*" + build-docker: + resource_class: large + docker: + - image: circleci/buildpack-deps:stretch + steps: + - checkout + - setup_remote_docker: + version: 19.03.14 + - attach_workspace: + at: ./artifacts + - run: + name: Install Docker buildx + command: | + mkdir -p ~/.docker/cli-plugins + curl -sSL -o ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.6.2/buildx-v0.6.2.linux-amd64 + chmod a+x ~/.docker/cli-plugins/docker-buildx + - run: + name: Enable experimental Docker features + command: echo 'export DOCKER_CLI_EXPERIMENTAL="enabled"' >> $BASH_ENV + - run: + name: Create Docker context + command: docker context create tls-environment + - run: + name: Initialize Docker buildx + command: docker buildx install + - run: + name: Start multiarch/qemu-user-static container + command: > + docker run --rm --privileged + multiarch/qemu-user-static:5.2.0-2 + --reset -p yes + - run: + name: Remove buildx multiarch container in case it exists + command: docker rm -f buildx_buildkit_docker-multiarch0 || continue + - run: + name: Create docker-multiarch builder + command: > + docker buildx create --name docker-multiarch tls-environment + --platform linux/386,linux/amd64,linux/arm/v5,linux/arm/v6,linux/arm/v7,linux/arm64,linux/mips64le,linux/ppc64le,linux/riscv64,linux/s390x + - run: + name: Inspect & bootstrap docker-multiarch builder + command: docker buildx inspect --builder docker-multiarch --bootstrap + - run: + name: Set docker-multiarch as default builder + command: docker buildx use docker-multiarch + - run: + name: "Build container image" + no_output_timeout: 45m + command: | + docker_tag=$(echo ${CIRCLE_TAG} | cut -b2- ) + docker login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_TOKEN} + export DOCKER_CLI_EXPERIMENTAL=enabled + + docker buildx build \ + --progress=plain \ + --platform linux/amd64,linux/arm64 \ + --tag observiq/stanza:latest \ + . fetch-plugins: executor: golang @@ -72,29 +130,61 @@ jobs: ghr -t ${GHI_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} ${CIRCLE_TAG} ./artifacts/ publish-docker-release: + # building amd64 + arm64 is compute intensive + resource_class: 2xlarge docker: - image: circleci/buildpack-deps:stretch steps: - checkout - - setup_remote_docker + - setup_remote_docker: + version: 19.03.14 - attach_workspace: at: ./artifacts - run: - name: "Configure Docker multi arch builds" + name: Install Docker buildx command: | mkdir -p ~/.docker/cli-plugins - curl -fL https://github.com/docker/buildx/releases/download/v0.6.1/buildx-v0.6.1.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx + curl -sSL -o ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.6.2/buildx-v0.6.2.linux-amd64 chmod a+x ~/.docker/cli-plugins/docker-buildx - docker buildx create --name stanza --use - docker buildx inspect --bootstrap + - run: + name: Enable experimental Docker features + command: echo 'export DOCKER_CLI_EXPERIMENTAL="enabled"' >> $BASH_ENV + - run: + name: Create Docker context + command: docker context create tls-environment + - run: + name: Initialize Docker buildx + command: docker buildx install + - run: + name: Start multiarch/qemu-user-static container + command: > + docker run --rm --privileged + multiarch/qemu-user-static:5.2.0-2 + --reset -p yes + - run: + name: Remove buildx multiarch container in case it exists + command: docker rm -f buildx_buildkit_docker-multiarch0 || continue + - run: + name: Create docker-multiarch builder + command: > + docker buildx create --name docker-multiarch tls-environment + --platform linux/386,linux/amd64,linux/arm/v5,linux/arm/v6,linux/arm/v7,linux/arm64,linux/mips64le,linux/ppc64le,linux/riscv64,linux/s390x + - run: + name: Inspect & bootstrap docker-multiarch builder + command: docker buildx inspect --builder docker-multiarch --bootstrap + - run: + name: Set docker-multiarch as default builder + command: docker buildx use docker-multiarch - run: name: "Publish Release on Docker Hub" + no_output_timeout: 45m command: | docker_tag=$(echo ${CIRCLE_TAG} | cut -b2- ) docker login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_TOKEN} export DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build \ + --progress=plain \ --platform linux/amd64,linux/arm64 \ --tag observiq/stanza:${docker_tag} \ --tag observiq/stanza:latest \ @@ -441,6 +531,12 @@ workflows: filters: tags: only: /^v\d+\.\d+\.\d+.*/ + - build-docker: + filters: + tags: + only: /^v\d+\.\d+\.\d+.*/ + requires: + - build - fetch-plugins: filters: branches: diff --git a/.dockerignore b/.dockerignore index ddd32d7ec..657da95b5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ -artifacts/stanza_darwin_amd64 -artifacts/stanza_windows_amd64 +artifacts/ .git/ dev/ diff --git a/Dockerfile b/Dockerfile index a2824806d..5390b6502 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,31 @@ FROM golang:1.16 as stage +ARG plugins_url="https://github.com/observiq/stanza-plugins/releases/latest/download/stanza-plugins.zip" +# arm cross builds do not have these symlinks in palce +RUN \ + ln -s /usr/bin/dpkg-split /usr/sbin/dpkg-split && \ + ln -s /usr/bin/dpkg-deb /usr/sbin/dpkg-deb && \ + ln -s /bin/tar /usr/sbin/tar && \ + ln -s /bin/rm /usr/sbin/rm && \ + echo "resolvconf resolvconf/linkify-resolvconf boolean false" | debconf-set-selections +# unzip is required because tar does not work on arm +RUN apt-get update && apt-get install unzip -y +WORKDIR /stanza/artifacts +RUN curl -fL "${plugins_url}" -o stanza-plugins.zip +RUN unzip stanza-plugins.zip WORKDIR /stanza COPY . . -RUN rm -rf artifacts/* RUN make build -WORKDIR /stanza/artifacts -# hack: "mv stanza_* stanza" gives an error because mv does not like '_*' -RUN for f in stanza_*; do mv "$f" stanza; done +RUN mv "artifacts/stanza_$(go env GOOS)_$(go env GOARCH)" artifacts/stanza + FROM gcr.io/observiq-container-images/stanza-base:v1.1.0 RUN mkdir -p /stanza_home ENV STANZA_HOME=/stanza_home RUN echo "pipeline:\n" >> /stanza_home/config.yaml - COPY --from=stage /stanza/artifacts/stanza /stanza_home/stanza -COPY ./artifacts/stanza-plugins.tar.gz /tmp/stanza-plugins.tar.gz -RUN tar -zxvf /tmp/stanza-plugins.tar.gz -C /stanza_home/ +COPY --from=stage /stanza/artifacts/plugins /stanza_home/plugins ENTRYPOINT /stanza_home/stanza \ --config /stanza_home/config.yaml \ --database /stanza_home/stanza.db \