[CI] [GHA] Introduce Debian 10 ARM (#26156)

akashchi · web-flow · commit fcd113835b3a · 2024-08-29T08:26:22.000Z
### Tickets:
 - *148717*
diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag
@@ -1 +1 @@
-pr-26067
+pr-26156
diff --git a/.github/dockerfiles/ov_build/debian_10_arm/Dockerfile b/.github/dockerfiles/ov_build/debian_10_arm/Dockerfile
@@ -0,0 +1,89 @@
+FROM openvinogithubactions.azurecr.io/dockerio/library/debian:10.13
+
+USER root
+
+# APT configuration
+RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
+    echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    TZ="Europe/London"
+
+RUN apt-get update && \
+    apt-get install \
+        software-properties-common \
+        curl \
+        git \
+        gpg-agent \
+        tzdata \
+        # Pythons \
+        python3 \
+        python3-pip \
+        python3-dev \
+        python3-venv \
+        python3-distutils \
+        libhdf5-dev \
+        # For building Python from source
+        build-essential \
+        libffi-dev \
+        libgdbm-dev \
+        libc6-dev \
+        libssl-dev \
+        zlib1g-dev \
+        libbz2-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        libncurses5-dev \
+        libncursesw5-dev \
+        xz-utils \
+        tk-dev \
+        libxml2-dev \
+        libxmlsec1-dev \
+        liblzma-dev \
+        wget \
+        # Compilers
+        gcc-arm-linux-gnueabihf \
+        g++-arm-linux-gnueabihf \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install build dependencies
+ADD install_build_dependencies.sh /install_build_dependencies.sh
+RUN chmod +x /install_build_dependencies.sh && \
+    /install_build_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install sscache
+ARG SCCACHE_VERSION="v0.7.5"
+ENV SCCACHE_HOME="/opt/sccache" \
+    SCCACHE_PATH="/opt/sccache/sccache"
+
+RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
+    SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-aarch64-unknown-linux-musl.tar.gz" && \
+    curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
+    tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}
+
+ENV PATH="$SCCACHE_HOME:$PATH"
+
+# Setup Python
+RUN wget https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tar.xz
+
+RUN tar -xf Python-3.11.9.tar.xz && \
+    cd Python-3.11.9 && \
+    ./configure --enable-optimizations && \
+    make -j 8 && \
+    make altinstall
+
+# Setup pip
+ENV PIP_VERSION="24.0"
+RUN python3 -m pip install --upgrade pip==24.0
+RUN python3.11 -m pip install --upgrade pip==24.0
+
+# Use Python 3.11 as default
+# Using venv here because other methods to switch the default Python break both system and wheels build
+RUN python3.11 -m venv venv
+ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
+
+ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
diff --git a/.github/workflows/debian_10_arm.yml b/.github/workflows/debian_10_arm.yml
@@ -0,0 +1,111 @@
+name: Debian 10 ARM
+on:
+  workflow_dispatch:
+  pull_request:
+  merge_group:
+  push:
+    branches:
+      - master
+      - 'releases/**'
+
+concurrency:
+  # github.ref is not unique in post-commit
+  group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-debian-10-arm
+  cancel-in-progress: true
+
+permissions: read-all
+
+jobs:
+  Smart_CI:
+    runs-on: ubuntu-latest
+    outputs:
+      affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      changed_components: "${{ steps.smart_ci.outputs.changed_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
+    steps:
+      - name: checkout action
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          sparse-checkout: .github/actions/smart-ci
+
+      - name: Get affected components
+        id: smart_ci
+        uses: ./.github/actions/smart-ci
+        with:
+          repository: ${{ github.repository }}
+          pr: ${{ github.event.number }}
+          commit_sha: ${{ github.sha }}
+          ref_name: ${{ github.ref_name }}
+          component_pattern: "category: (.*)"
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          skip_when_only_listed_labels_set: 'docs'
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*'
+
+      - name: Show affected components
+        run: |
+          echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}"
+        shell: bash
+
+  Docker:
+    needs: Smart_CI
+    runs-on: aks-linux-16-cores-arm-docker-build
+    container:
+      image: openvinogithubactions.azurecr.io/docker_build:0.2
+      volumes:
+        - /mount:/mount
+    outputs:
+      images: "${{ steps.handle_docker.outputs.images }}"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - uses: ./.github/actions/handle_docker
+        id: handle_docker
+        with:
+          images: |
+            ov_build/debian_10_arm
+          registry: 'openvinogithubactions.azurecr.io'
+          dockerfiles_root_dir: '.github/dockerfiles'
+          changed_components: ${{ needs.smart_ci.outputs.changed_components }}
+
+  Build:
+    needs: [Docker, Smart_CI]
+    if: "!needs.smart_ci.outputs.skip_workflow"
+    uses: ./.github/workflows/job_build_linux.yml
+    with:
+      runner: 'aks-linux-16-cores-arm'
+      container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.debian_10_arm }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
+      affected-components: ${{ needs.smart_ci.outputs.affected_components }}
+      event-name: ${{ github.event_name }}
+      os: 'debian_10'
+      arch: 'arm'
+      build-js: false
+      build-debian-packages: false
+      build-contrib: false
+      cmake-options: |-
+        -DCMAKE_TOOLCHAIN_FILE=${OPENVINO_REPO}/cmake/arm.toolchain.cmake \
+        -DTHREADS_PTHREAD_ARG="-pthread" \
+        -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DENABLE_STRICT_DEPENDENCIES=OFF \
+        -DENABLE_CPPLINT=OFF \
+        -DENABLE_NCC_STYLE=OFF \
+        -DCMAKE_VERBOSE_MAKEFILE=ON \
+        -DENABLE_CONFORMANCE_PGQL=ON \
+        -DENABLE_LTO=ON \
+        -DENABLE_TESTS=ON \
+        -DENABLE_PYTHON=OFF
+
+  Overall_Status:
+    name: ci/gha_overall_status_debian_10_arm
+    needs: [Smart_CI, Build]
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check status of all jobs
+        if: >-
+          ${{
+            contains(needs.*.result, 'failure') ||
+            contains(needs.*.result, 'cancelled')
+          }}
+        run: exit 1
diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml
@@ -22,6 +22,30 @@ on:
         description: 'OS that is used for building in the form of "ubuntu_20_04"'
         type: string
         required: true
+      arch:
+        description: 'Target architecture'
+        type: string
+        default: 'x86_64'
+        required: false
+      cmake-options:
+        description: 'A string of options passed to CMake'
+        type: string
+        required: true
+      build-js:
+        description: 'Whether to build OpenVINO JS Bindings'
+        type: boolean
+        required: false
+        default: true
+      build-debian-packages:
+        description: 'Whether to build Debian packages'
+        type: boolean
+        required: false
+        default: true
+      build-contrib:
+        description: 'Whether to build OpenVINO Contrib'
+        type: boolean
+        required: false
+        default: true
 
 permissions: read-all
 
@@ -40,7 +64,6 @@ jobs:
     env:
       DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
       CMAKE_BUILD_TYPE: 'Release'
-      CMAKE_GENERATOR: 'Ninja Multi-Config'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
       SCCACHE_IGNORE_SERVER_IO_ERROR: 1
@@ -55,11 +78,11 @@ jobs:
       INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install
       DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install
       BUILD_DIR: /__w/openvino/openvino/openvino_build
-      SCCACHE_AZURE_KEY_PREFIX: ${{ inputs.os }}_x86_64_Release
+      SCCACHE_AZURE_KEY_PREFIX: ${{ inputs.os }}_${{ inputs.arch }}_Release
       ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime
       ARTIFACTS_SHARE: "/mount/build-artifacts"
       MANIFEST_PATH: '/__w/openvino/openvino/manifest.yml'
-      PRODUCT_TYPE: public_linux_${{ inputs.os }}_release
+      PRODUCT_TYPE: public_linux_${{ inputs.os }}_${{ inputs.arch }}_release
     steps:
       - name: Clone OpenVINO
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -131,17 +154,7 @@ jobs:
       - name: CMake configure - OpenVINO
         run: |
           cmake \
-            -G "${{ env.CMAKE_GENERATOR }}" \
-            -DENABLE_CPPLINT=OFF \
-            -DENABLE_NCC_STYLE=OFF \
-            -DENABLE_TESTS=ON \
-            -DENABLE_STRICT_DEPENDENCIES=OFF \
-            -DENABLE_SYSTEM_OPENCL=ON \
-            -DCMAKE_VERBOSE_MAKEFILE=ON \
-            -DCPACK_GENERATOR=TGZ \
-            -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \
-            -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \
+            ${{ inputs.cmake-options }} \
             -S ${OPENVINO_REPO} \
             -B ${BUILD_DIR}
 
@@ -182,6 +195,7 @@ jobs:
           popd
 
       - name: Build Debian packages
+        if: ${{ inputs.build-debian-packages }}
         run: |
           # Ubuntu 24 does not allow using the system Python directly so 
           # we have to use Python from the virtual environment created in Docker
@@ -199,6 +213,7 @@ jobs:
           cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package
 
       - name: Cmake & Build - OpenVINO Contrib
+        if: ${{ inputs.build-contrib }}
         run: |
           cmake \
             -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \
@@ -208,7 +223,7 @@ jobs:
           cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }}
 
       - name: CMake configure, build and install - OpenVINO JS API
-        if: ${{ fromJSON(inputs.affected-components).JS_API }}
+        if: ${{ fromJSON(inputs.affected-components).JS_API && inputs.build-js }}
         run: |
           cmake -UTBB* -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -S ${OPENVINO_REPO} -B ${BUILD_DIR}
           cmake --build ${BUILD_DIR} --parallel
@@ -234,7 +249,7 @@ jobs:
           if-no-files-found: 'error'
 
       - name: Upload openvino js package
-        if: ${{ fromJSON(inputs.affected-components).JS_API }}
+        if: ${{ fromJSON(inputs.affected-components).JS_API && inputs.build-js }}
         uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
           name: openvino_js_package
@@ -250,7 +265,7 @@ jobs:
           if-no-files-found: 'error'
 
       - name: Upload openvino debian packages
-        if: ${{ always() }}
+        if: ${{ inputs.build-debian-packages }}
         uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
           name: openvino_debian_packages
diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml
@@ -85,6 +85,16 @@ jobs:
       affected-components: ${{ needs.smart_ci.outputs.affected_components }}
       event-name: ${{ github.event_name }}
       os: 'ubuntu_20_04'
+      cmake-options: |- 
+            -G "Ninja Multi-Config" \
+            -DENABLE_CPPLINT=OFF \
+            -DENABLE_NCC_STYLE=OFF \
+            -DENABLE_TESTS=ON \
+            -DENABLE_STRICT_DEPENDENCIES=OFF \
+            -DENABLE_SYSTEM_OPENCL=ON \
+            -DCMAKE_VERBOSE_MAKEFILE=ON \
+            -DCPACK_GENERATOR=TGZ \
+            -DCMAKE_COMPILE_WARNING_AS_ERROR=ON
 
   Debian_Packages:
     name: Debian Packages
diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml
@@ -87,6 +87,16 @@ jobs:
       affected-components: ${{ needs.smart_ci.outputs.affected_components }}
       event-name: ${{ github.event_name }}
       os: 'ubuntu_22_04'
+      cmake-options: |- 
+            -G "Ninja Multi-Config" \
+            -DENABLE_CPPLINT=OFF \
+            -DENABLE_NCC_STYLE=OFF \
+            -DENABLE_TESTS=ON \
+            -DENABLE_STRICT_DEPENDENCIES=OFF \
+            -DENABLE_SYSTEM_OPENCL=ON \
+            -DCMAKE_VERBOSE_MAKEFILE=ON \
+            -DCPACK_GENERATOR=TGZ \
+            -DCMAKE_COMPILE_WARNING_AS_ERROR=ON
 
   Debian_Packages:
     name: Debian Packages
diff --git a/.github/workflows/ubuntu_22_dpcpp.yml b/.github/workflows/ubuntu_22_dpcpp.yml
@@ -74,6 +74,19 @@ jobs:
       affected-components: ${{ needs.smart_ci.outputs.affected_components }}
       event-name: ${{ github.event_name }}
       os: 'ubuntu_22_04_dpcpp'
+      build-js: false
+      build-debian-packages: false
+      build-contrib: false
+      cmake-options: |- 
+            -G "Ninja Multi-Config" \
+            -DENABLE_CPPLINT=OFF \
+            -DENABLE_NCC_STYLE=OFF \
+            -DENABLE_TESTS=ON \
+            -DENABLE_STRICT_DEPENDENCIES=OFF \
+            -DENABLE_SYSTEM_OPENCL=ON \
+            -DCMAKE_VERBOSE_MAKEFILE=ON \
+            -DCPACK_GENERATOR=TGZ \
+            -DCMAKE_COMPILE_WARNING_AS_ERROR=ON
 
   Overall_Status:
     name: ci/gha_overall_status_ubuntu_22.04_dpcpp
diff --git a/.github/workflows/ubuntu_24.yml b/.github/workflows/ubuntu_24.yml
@@ -83,6 +83,16 @@ jobs:
       affected-components: ${{ needs.smart_ci.outputs.affected_components }}
       event-name: ${{ github.event_name }}
       os: 'ubuntu_24_04'
+      cmake-options: |- 
+            -G "Ninja Multi-Config" \
+            -DENABLE_CPPLINT=OFF \
+            -DENABLE_NCC_STYLE=OFF \
+            -DENABLE_TESTS=ON \
+            -DENABLE_STRICT_DEPENDENCIES=OFF \
+            -DENABLE_SYSTEM_OPENCL=ON \
+            -DCMAKE_VERBOSE_MAKEFILE=ON \
+            -DCPACK_GENERATOR=TGZ \
+            -DCMAKE_COMPILE_WARNING_AS_ERROR=ON
 
   Debian_Packages:
     name: Debian Packages
