From c7627351407b3f70b587224bec906e422bff3781 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 10:21:10 +0100 Subject: [PATCH 01/60] add U20, 22, 24 to Docker building --- .../ov_build/ubuntu_24_04_x64/Dockerfile | 73 +++++++++++++++++++ .github/workflows/linux.yml | 33 ++++++--- 2 files changed, 97 insertions(+), 9 deletions(-) create mode 100644 .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile new file mode 100644 index 00000000000000..ed7e37c21334e0 --- /dev/null +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -0,0 +1,73 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:24.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install \ + curl \ + git \ + ca-certificates \ + gpg-agent \ + tzdata \ + libtbb2 \ + # Pythons + python3.8-dev \ + python3.8-venv \ + python3.8-distutils \ + python3.11-dev \ + python3.11-venv \ + python3.11-distutils \ + # For Java API + default-jdk \ + # Compiler \ + gcc-10 \ + g++-10 \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install build dependencies +ADD install_build_dependencies.sh /install_build_dependencies.sh +RUN chmod +x /install_build_dependencies.sh && \ + /install_build_dependencies.sh && \ + rm -rf /var/lib/apt/lists/* + +# Set gcc-10 as a default compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 + +# Install sscache +ARG SCCACHE_VERSION="v0.7.5" +ENV SCCACHE_HOME="/opt/sccache" \ + SCCACHE_PATH="/opt/sccache/sccache" + +RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ + SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \ + curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ + tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} + +# Setup pip +ENV PIP_VERSION="24.0" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py + +# Use Python 3.11 as default instead of Python 3.8 +# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +RUN python3.11 -m venv venv +ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" + +ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index b6f1d901af101e..f1caf7d056b580 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,4 +1,4 @@ -name: Linux (Ubuntu 20.04, Python 3.11) +name: Linux (Ubuntu 20.04, 22.04, 24.04, Python 3.11) on: schedule: # at 00:00 on Wednesday and Saturday @@ -71,21 +71,36 @@ jobs: with: images: | ov_build/ubuntu_20_04_x64 + ov_build/ubuntu_22_04_x64 + ov_build/ubuntu_24_04_x64 ov_build/ubuntu_20_04_x64_nvidia ov_test/ubuntu_20_04_x64 + ov_test/ubuntu_22_04_x64 registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} Build: + name: Build (Ubuntu ${{ matrix.os_version }}) needs: [Docker, Smart_CI] timeout-minutes: 150 defaults: run: shell: bash runs-on: aks-linux-16-cores-32gb + strategy: + max-parallel: 3 + fail-fast: false + matrix: + include: + - os_version: '20' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} + - os_version: '22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} + - os_version: '24' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_24_04_x64 }} container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} + image: ${{ matrix.image }} volumes: - /mount:/mount options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING @@ -107,7 +122,7 @@ jobs: INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install BUILD_DIR: /__w/openvino/openvino/openvino_build - SCCACHE_AZURE_KEY_PREFIX: ubuntu20_x86_64_Release + SCCACHE_AZURE_KEY_PREFIX: ubuntu${{ matrix.os_version }}_x86_64_Release ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime if: "!needs.smart_ci.outputs.skip_workflow" @@ -257,7 +272,7 @@ jobs: uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 if: always() with: - name: build_logs + name: build_logs_ubuntu_${{ matrix.os_version }} path: ${{ env.SCCACHE_ERROR_LOG }} if-no-files-found: 'ignore' @@ -265,7 +280,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_package + name: openvino_package_ubuntu_${{ matrix.os_version }} path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz if-no-files-found: 'error' @@ -273,7 +288,7 @@ jobs: if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_js_package + name: openvino_js_package_ubuntu_${{ matrix.os_version }} path: ${{ env.INSTALL_DIR_JS }} if-no-files-found: 'error' @@ -281,7 +296,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_developer_package + name: openvino_developer_package_ubuntu_${{ matrix.os_version }} path: ${{ env.BUILD_DIR }}/openvino_developer_package.tar.gz if-no-files-found: 'error' @@ -289,7 +304,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_debian_packages + name: openvino_debian_packages_ubuntu_${{ matrix.os_version }} path: ${{ env.BUILD_DIR }}/*.deb if-no-files-found: 'error' @@ -297,7 +312,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_tests + name: openvino_tests_ubuntu_${{ matrix.os_version }} path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz if-no-files-found: 'error' From 28f88a9557b636a9261319bf936bef7ac533903e Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 10:26:11 +0100 Subject: [PATCH 02/60] update docker tag --- .github/dockerfiles/docker_tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index 88270be301bc90..1a0c3296907b7c 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-25107 +pr-25202 From 443649e38a7c7e97e83322b9a61cc803cd3ee16c Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 10:36:58 +0100 Subject: [PATCH 03/60] rm unuses --- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index ed7e37c21334e0..03b4df8cfc0ca1 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -22,7 +22,6 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ - libtbb2 \ # Pythons python3.8-dev \ python3.8-venv \ From b170723cbbe73e7082616b8d3c7c4eeb7297d96d Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 11:15:21 +0100 Subject: [PATCH 04/60] newer pip --- .github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 2 +- .github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile | 2 +- .github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile | 2 +- .github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile index 011dc15895dc16..99e36e893cd6d9 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile @@ -59,7 +59,7 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index b13bfe1f2df316..1723c891410fbb 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -58,7 +58,7 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile index c192227085e901..35f00e5248a00a 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile @@ -75,7 +75,7 @@ RUN apt update && apt install -y \ rm -rf /var/lib/apt/lists/* # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile index 5eda853da56545..e5dcba97e79514 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile @@ -62,7 +62,7 @@ RUN dpkg --add-architecture riscv64 && \ apt-get install -y --no-install-recommends libpython3-dev:riscv64 # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index cb3e4cc639e0a9..5b529eacfb9dc2 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -59,7 +59,7 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile index e99b1bd028d7bf..14af17160066d1 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile @@ -58,7 +58,7 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 03b4df8cfc0ca1..39895100b76447 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -58,7 +58,7 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile index 68d80858dac63e..2b5b0724ca4b56 100644 --- a/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile @@ -37,7 +37,7 @@ RUN chmod +x /install_openvino_dependencies.sh && \ rm -rf /var/lib/apt/lists/* # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile index 0a151be1e6876d..73c10f0fbd865b 100644 --- a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile @@ -37,7 +37,7 @@ RUN chmod +x /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile index 1566c2305d04b7..903d8d21f27154 100644 --- a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile @@ -37,7 +37,7 @@ RUN chmod +x /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* # Setup pip -ENV PIP_VERSION="24.0" +ENV PIP_VERSION="24.1" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ From 519890c05b1e03328c7ade027563266ec66dfe45 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 11:27:24 +0100 Subject: [PATCH 05/60] install pip via apt --- .../dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile | 12 +++++++----- .../dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile | 12 +++++++----- .../dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 12 +++++++----- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index 1723c891410fbb..8b0ab295f4153e 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -26,9 +26,11 @@ RUN apt-get update && \ python3.8-dev \ python3.8-venv \ python3.8-distutils \ + python3.8-pip \ python3.11-dev \ python3.11-venv \ python3.11-distutils \ + python3.11-pip \ # For Java API default-jdk \ # Compiler \ @@ -58,11 +60,11 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.1" -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - rm -f get-pip.py +#ENV PIP_VERSION="24.1" +#RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ +# python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ +# python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ +# rm -f get-pip.py # Use Python 3.11 as default instead of Python 3.8 # Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index 5b529eacfb9dc2..c46fddfd73a4f1 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -27,9 +27,11 @@ RUN apt-get update && \ python3.8-dev \ python3.8-venv \ python3.8-distutils \ + python3.8-pip \ python3.11-dev \ python3.11-venv \ python3.11-distutils \ + python3.11-pip \ # For Java API default-jdk \ # Compiler \ @@ -59,11 +61,11 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.1" -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - rm -f get-pip.py +#ENV PIP_VERSION="24.1" +#RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ +# python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ +# python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ +# rm -f get-pip.py # Use Python 3.11 as default instead of Python 3.8 # Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 39895100b76447..4fec1d4ac26f12 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -26,9 +26,11 @@ RUN apt-get update && \ python3.8-dev \ python3.8-venv \ python3.8-distutils \ + python3.8-pip \ python3.11-dev \ python3.11-venv \ python3.11-distutils \ + python3.11-pip \ # For Java API default-jdk \ # Compiler \ @@ -58,11 +60,11 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.1" -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - rm -f get-pip.py +#ENV PIP_VERSION="24.1" +#RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ +# python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ +# python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ +# rm -f get-pip.py # Use Python 3.11 as default instead of Python 3.8 # Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build From f923856569de60bccf4885e0178e73f2dcd4bf7a Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 11:37:03 +0100 Subject: [PATCH 06/60] use system python --- .../ov_build/ubuntu_20_04_x64/Dockerfile | 28 ++++++++----------- .../ov_build/ubuntu_22_04_x64/Dockerfile | 28 ++++++++----------- .../ov_build/ubuntu_24_04_x64/Dockerfile | 28 ++++++++----------- .github/workflows/linux.yml | 7 ++--- 4 files changed, 39 insertions(+), 52 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index 8b0ab295f4153e..bec6a3828cd328 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -22,15 +22,11 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ - # Pythons - python3.8-dev \ - python3.8-venv \ - python3.8-distutils \ - python3.8-pip \ - python3.11-dev \ - python3.11-venv \ - python3.11-distutils \ - python3.11-pip \ + # Python + python3-dev \ + python3-venv \ + python3-distutils \ + python3-pip \ # For Java API default-jdk \ # Compiler \ @@ -59,16 +55,16 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} +ENV PATH="$SCCACHE_HOME:$PATH" # Setup pip -#ENV PIP_VERSION="24.1" -#RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ -# python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ -# python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ -# rm -f get-pip.py +ENV PIP_VERSION="24.1" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py # Use Python 3.11 as default instead of Python 3.8 # Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build -RUN python3.11 -m venv venv -ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" +#RUN python3.11 -m venv venv +#ENV PATH="/venv/bin:$PATH" ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index c46fddfd73a4f1..43982429b2e9e1 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -23,15 +23,11 @@ RUN apt-get update && \ gpg-agent \ tzdata \ libtbb2 \ - # Pythons - python3.8-dev \ - python3.8-venv \ - python3.8-distutils \ - python3.8-pip \ - python3.11-dev \ - python3.11-venv \ - python3.11-distutils \ - python3.11-pip \ + # Python + python3-dev \ + python3-venv \ + python3-distutils \ + python3-pip \ # For Java API default-jdk \ # Compiler \ @@ -60,16 +56,16 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} +ENV PATH="$SCCACHE_HOME:$PATH" # Setup pip -#ENV PIP_VERSION="24.1" -#RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ -# python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ -# python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ -# rm -f get-pip.py +ENV PIP_VERSION="24.1" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py # Use Python 3.11 as default instead of Python 3.8 # Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build -RUN python3.11 -m venv venv -ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" +#RUN python3.11 -m venv venv +#ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 4fec1d4ac26f12..870c5e26669788 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -22,15 +22,11 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ - # Pythons - python3.8-dev \ - python3.8-venv \ - python3.8-distutils \ - python3.8-pip \ - python3.11-dev \ - python3.11-venv \ - python3.11-distutils \ - python3.11-pip \ + # Python + python3-dev \ + python3-venv \ + python3-distutils \ + python3-pip \ # For Java API default-jdk \ # Compiler \ @@ -59,16 +55,16 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} +ENV PATH="$SCCACHE_HOME:$PATH" # Setup pip -#ENV PIP_VERSION="24.1" -#RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ -# python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ -# python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ -# rm -f get-pip.py +ENV PIP_VERSION="24.1" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py # Use Python 3.11 as default instead of Python 3.8 # Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build -RUN python3.11 -m venv venv -ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" +#RUN python3.11 -m venv venv +#ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f1caf7d056b580..9f99e0e9992f71 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,4 +1,4 @@ -name: Linux (Ubuntu 20.04, 22.04, 24.04, Python 3.11) +name: Linux (Ubuntu 20.04, 22.04, 24.04) on: schedule: # at 00:00 on Wednesday and Saturday @@ -237,14 +237,13 @@ jobs: - name: Build Debian packages run: | - /usr/bin/python3.8 -m pip install -U pip - /usr/bin/python3.8 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt + python3 -m pip install -U pip + python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt cmake -UPython* -U_Python* -UPYTHON* \ -UTBB* \ -DENABLE_SYSTEM_TBB=ON \ -DENABLE_PYTHON_PACKAGING=ON \ -DENABLE_TESTS=OFF \ - -DPython3_EXECUTABLE=/usr/bin/python3.8 \ -DCPACK_GENERATOR=DEB \ ${BUILD_DIR} cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package From 5c583ada0da83512a4b20633c11143f91eecaa5a Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 11:57:05 +0100 Subject: [PATCH 07/60] rm --- .github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile | 1 - .github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile | 1 - .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index bec6a3828cd328..c5f3610e0389b6 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -25,7 +25,6 @@ RUN apt-get update && \ # Python python3-dev \ python3-venv \ - python3-distutils \ python3-pip \ # For Java API default-jdk \ diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index 43982429b2e9e1..9603579612c4ec 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -26,7 +26,6 @@ RUN apt-get update && \ # Python python3-dev \ python3-venv \ - python3-distutils \ python3-pip \ # For Java API default-jdk \ diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 870c5e26669788..28a42a6d6c296a 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -25,7 +25,6 @@ RUN apt-get update && \ # Python python3-dev \ python3-venv \ - python3-distutils \ python3-pip \ # For Java API default-jdk \ From a0df984525e9a52b970b96e777b7b0633686787f Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 14:05:05 +0100 Subject: [PATCH 08/60] use venv for system python --- .../ov_build/ubuntu_24_04_x64/Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 28a42a6d6c296a..ffaafa5630fc8e 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -26,6 +26,7 @@ RUN apt-get update && \ python3-dev \ python3-venv \ python3-pip \ + python3-full \ # For Java API default-jdk \ # Compiler \ @@ -55,15 +56,14 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} ENV PATH="$SCCACHE_HOME:$PATH" -# Setup pip -ENV PIP_VERSION="24.1" -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - rm -f get-pip.py -# Use Python 3.11 as default instead of Python 3.8 -# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build -#RUN python3.11 -m venv venv -#ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" +# Create a virtual environment for the system Python as Python in Ubuntu 24 complains about installing +# packages into the system Python +RUN python3 -m venv venv +ENV PATH="/venv/bin:$PATH" + +# Setup pip +ENV PIP_VERSION="24.0" +RUN /venv/bin/python3 -m pip install pip==${PIP_VERSION} ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} From fff86c6f37c9661916854abaa4110916ffa9eeed Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 14:26:44 +0100 Subject: [PATCH 09/60] install protobuf on U24 --- .github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile | 2 +- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 2 ++ .github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile | 2 +- .github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile | 2 +- .github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile | 2 +- 10 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile index 99e36e893cd6d9..011dc15895dc16 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile @@ -59,7 +59,7 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.1" +ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index c5f3610e0389b6..806d0702fea704 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -56,7 +56,7 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ ENV PATH="$SCCACHE_HOME:$PATH" # Setup pip -ENV PIP_VERSION="24.1" +ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile index 35f00e5248a00a..c192227085e901 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile @@ -75,7 +75,7 @@ RUN apt update && apt install -y \ rm -rf /var/lib/apt/lists/* # Setup pip -ENV PIP_VERSION="24.1" +ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile index e5dcba97e79514..5eda853da56545 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile @@ -62,7 +62,7 @@ RUN dpkg --add-architecture riscv64 && \ apt-get install -y --no-install-recommends libpython3-dev:riscv64 # Setup pip -ENV PIP_VERSION="24.1" +ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index 9603579612c4ec..ec2bc87000a760 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -57,7 +57,7 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ ENV PATH="$SCCACHE_HOME:$PATH" # Setup pip -ENV PIP_VERSION="24.1" +ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile index 14af17160066d1..e99b1bd028d7bf 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile @@ -58,7 +58,7 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} # Setup pip -ENV PIP_VERSION="24.1" +ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index ffaafa5630fc8e..7be4070296d7d3 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -22,6 +22,8 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ + # ONNX + protobuf-compiler \ # Python python3-dev \ python3-venv \ diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile index 2b5b0724ca4b56..68d80858dac63e 100644 --- a/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile @@ -37,7 +37,7 @@ RUN chmod +x /install_openvino_dependencies.sh && \ rm -rf /var/lib/apt/lists/* # Setup pip -ENV PIP_VERSION="24.1" +ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile index 73c10f0fbd865b..0a151be1e6876d 100644 --- a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile @@ -37,7 +37,7 @@ RUN chmod +x /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* # Setup pip -ENV PIP_VERSION="24.1" +ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ diff --git a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile index 903d8d21f27154..1566c2305d04b7 100644 --- a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile @@ -37,7 +37,7 @@ RUN chmod +x /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* # Setup pip -ENV PIP_VERSION="24.1" +ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ From d38f31795172ed2382d5caa8d981a3b6e9e76f93 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 14:45:09 +0100 Subject: [PATCH 10/60] install setuptools and wheel --- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 7be4070296d7d3..f5c9210f9f2ed5 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -22,8 +22,6 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ - # ONNX - protobuf-compiler \ # Python python3-dev \ python3-venv \ @@ -66,6 +64,6 @@ ENV PATH="/venv/bin:$PATH" # Setup pip ENV PIP_VERSION="24.0" -RUN /venv/bin/python3 -m pip install pip==${PIP_VERSION} +RUN /venv/bin/python3 -m pip install pip==${PIP_VERSION} setuptools wheel ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} From 5a75480fb397f0bb9dbf8f7884de9dd49f75cef4 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 15:02:58 +0100 Subject: [PATCH 11/60] protobuf --- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index f5c9210f9f2ed5..6e2b79bdb4de23 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -22,6 +22,9 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ + # ONNX + protobuf-compiler \ + libprotobuf-dev \ # Python python3-dev \ python3-venv \ @@ -64,6 +67,6 @@ ENV PATH="/venv/bin:$PATH" # Setup pip ENV PIP_VERSION="24.0" -RUN /venv/bin/python3 -m pip install pip==${PIP_VERSION} setuptools wheel +RUN /venv/bin/python3 -m pip install --upgrade pip==${PIP_VERSION} setuptools wheel ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} From 1a6489681ee1e3f3c171cd253d61a652cdbbe360 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 15:21:34 +0100 Subject: [PATCH 12/60] set cmake args for onnx --- .github/workflows/linux.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9f99e0e9992f71..af27f3b170cc21 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -163,6 +163,11 @@ jobs: - name: Install python dependencies run: | + + if [[ ${{ matrix.os_version }} == '24' ]]; then + CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON + fi + # For Python API: build and wheel packaging python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt From 954b8cb6aa855b49ad571d4a19de36c3626a92b2 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 15:48:37 +0100 Subject: [PATCH 13/60] rm protobuf --- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 3 --- .github/workflows/linux.yml | 5 ----- 2 files changed, 8 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 6e2b79bdb4de23..3c89534c81c1ce 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -22,9 +22,6 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ - # ONNX - protobuf-compiler \ - libprotobuf-dev \ # Python python3-dev \ python3-venv \ diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index af27f3b170cc21..9f99e0e9992f71 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -163,11 +163,6 @@ jobs: - name: Install python dependencies run: | - - if [[ ${{ matrix.os_version }} == '24' ]]; then - CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON - fi - # For Python API: build and wheel packaging python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt From a82354cb8710a05fb999086504d8a2f286cd94f6 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 25 Jun 2024 16:27:06 +0100 Subject: [PATCH 14/60] do not use onnx from cache --- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 5 +++-- .github/workflows/linux.yml | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 3c89534c81c1ce..35b01766f312e4 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -1,4 +1,5 @@ -FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:24.04 +FROM ubuntu:24.04 +# FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:24.04 USER root @@ -64,6 +65,6 @@ ENV PATH="/venv/bin:$PATH" # Setup pip ENV PIP_VERSION="24.0" -RUN /venv/bin/python3 -m pip install --upgrade pip==${PIP_VERSION} setuptools wheel +RUN /venv/bin/python3 -m pip install --upgrade pip==${PIP_VERSION} ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9f99e0e9992f71..744fd9876d30a9 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -165,9 +165,13 @@ jobs: run: | # For Python API: build and wheel packaging python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - + # For running ONNX frontend unit tests - python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt + if [[ ${{ matrix.os_version }} == '24' ]]; then + python3 -m pip install --force-reinstall --no-cache-dir -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt + else + python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt + fi # For running TensorFlow frontend unit tests python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow/tests/requirements.txt From 364918d46c1feb576561b3537f2cf20de81d43d2 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 26 Jun 2024 09:32:53 +0100 Subject: [PATCH 15/60] use postfix for artifacts --- .../ov_build/ubuntu_24_04_x64/Dockerfile | 3 +- .github/workflows/job_debian_packages.yml | 7 +++- .github/workflows/linux.yml | 33 +++++++++++-------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 35b01766f312e4..5963d91d650ed6 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -1,5 +1,4 @@ -FROM ubuntu:24.04 -# FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:24.04 +FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:24.04 USER root diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index d4734805a2cb4d..3c35e45b0b574c 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -12,6 +12,11 @@ on: type: string required: false default: null + os: + description: 'OS version needed to download correct artifacts in the form of "_ubuntu_22"' + type: string + required: false + default: null permissions: read-all @@ -35,7 +40,7 @@ jobs: - name: Download OpenVINO debian packages uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: - name: openvino_debian_packages + name: openvino_debian_packages${{ inputs.os }} path: ${{ env.DEBIAN_PACKAGES_DIR }} # Needed as ${{ github.workspace }} is not working correctly when using Docker diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 744fd9876d30a9..cb750d933ff308 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -93,12 +93,12 @@ jobs: fail-fast: false matrix: include: - - os_version: '20' + - os: 'ubuntu_20' image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} - - os_version: '22' + - os: 'ubuntu_22' image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} - - os_version: '24' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_24_04_x64 }} +# - os: 'ubuntu_24' +# image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_24_04_x64 }} container: image: ${{ matrix.image }} volumes: @@ -122,7 +122,7 @@ jobs: INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install BUILD_DIR: /__w/openvino/openvino/openvino_build - SCCACHE_AZURE_KEY_PREFIX: ubuntu${{ matrix.os_version }}_x86_64_Release + SCCACHE_AZURE_KEY_PREFIX: ${{ matrix.os }}_x86_64_Release ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime if: "!needs.smart_ci.outputs.skip_workflow" @@ -167,7 +167,7 @@ jobs: python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt # For running ONNX frontend unit tests - if [[ ${{ matrix.os_version }} == '24' ]]; then + if [[ ${{ matrix.os }} == 'ubuntu_24' ]]; then python3 -m pip install --force-reinstall --no-cache-dir -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt else python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt @@ -275,7 +275,7 @@ jobs: uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 if: always() with: - name: build_logs_ubuntu_${{ matrix.os_version }} + name: build_logs_${{ matrix.os }} path: ${{ env.SCCACHE_ERROR_LOG }} if-no-files-found: 'ignore' @@ -283,7 +283,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_package_ubuntu_${{ matrix.os_version }} + name: openvino_package_${{ matrix.os }} path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz if-no-files-found: 'error' @@ -291,7 +291,7 @@ jobs: if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_js_package_ubuntu_${{ matrix.os_version }} + name: openvino_js_package_${{ matrix.os }} path: ${{ env.INSTALL_DIR_JS }} if-no-files-found: 'error' @@ -299,7 +299,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_developer_package_ubuntu_${{ matrix.os_version }} + name: openvino_developer_package_${{ matrix.os }} path: ${{ env.BUILD_DIR }}/openvino_developer_package.tar.gz if-no-files-found: 'error' @@ -307,7 +307,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_debian_packages_ubuntu_${{ matrix.os_version }} + name: openvino_debian_packages_${{ matrix.os }} path: ${{ env.BUILD_DIR }}/*.deb if-no-files-found: 'error' @@ -315,17 +315,22 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: openvino_tests_ubuntu_${{ matrix.os_version }} + name: openvino_tests_${{ matrix.os }} path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz if-no-files-found: 'error' Debian_Packages: name: Debian Packages - needs: Build + needs: [Build, Docker] uses: ./.github/workflows/job_debian_packages.yml + strategy: + matrix: + include: + - os: 'ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} with: runner: 'aks-linux-4-cores-16gb' - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + image: ${{ matrix.image }} Samples: needs: [ Docker, Build, Smart_CI ] From e31ac3cb74625a69fd36091096d10c9a0a31b921 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 26 Jun 2024 10:42:32 +0100 Subject: [PATCH 16/60] use Unix Makefiles for U24 onnx --- .github/workflows/linux.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index cb750d933ff308..9233f0ec13ee72 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -81,7 +81,7 @@ jobs: changed_components: ${{ needs.smart_ci.outputs.changed_components }} Build: - name: Build (Ubuntu ${{ matrix.os_version }}) + name: Build (${{ matrix.os }}) needs: [Docker, Smart_CI] timeout-minutes: 150 defaults: @@ -97,8 +97,8 @@ jobs: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} - os: 'ubuntu_22' image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} -# - os: 'ubuntu_24' -# image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_24_04_x64 }} + - os: 'ubuntu_24' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_24_04_x64 }} container: image: ${{ matrix.image }} volumes: @@ -168,6 +168,7 @@ jobs: # For running ONNX frontend unit tests if [[ ${{ matrix.os }} == 'ubuntu_24' ]]; then + export CMAKE_GENERATOR="Unix Makefiles" python3 -m pip install --force-reinstall --no-cache-dir -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt else python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt From 18416753ac9d4ded32c8544cf4c21232c1998c43 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 26 Jun 2024 12:22:08 +0100 Subject: [PATCH 17/60] provide missing arg --- .github/workflows/linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9233f0ec13ee72..6f29d4f67a2d5f 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -327,11 +327,12 @@ jobs: strategy: matrix: include: - - os: 'ubuntu_22' + - os: '_ubuntu_22' image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} with: runner: 'aks-linux-4-cores-16gb' image: ${{ matrix.image }} + os: ${{ matrix.os }} Samples: needs: [ Docker, Build, Smart_CI ] From becc14a367f1ce42689209f96e0287e086467286 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 26 Jun 2024 13:54:49 +0100 Subject: [PATCH 18/60] use approach with inputs --- .github/workflows/job_debian_packages.yml | 5 +- .github/workflows/linux.yml | 93 ++++++++++++++++++++++- 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index 3c35e45b0b574c..22d1564a5094dd 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -56,7 +56,10 @@ jobs: apt-get install --no-install-recommends -y gnupg wget ca-certificates wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - echo "deb https://apt.repos.intel.com/openvino/2024 ubuntu20 main" | tee /etc/apt/sources.list.d/intel-openvino-2024.list + + [[ $(lsb_release -r -s) == *"22"* ]] && ubuntu_version=22 || ubuntu_version=20 + echo "deb https://apt.repos.intel.com/openvino/2024 ubuntu$ubuntu_version main" | tee /etc/apt/sources.list.d/intel-openvino-2024.list + apt-get update -y apt-get install -y openvino fi diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 6f29d4f67a2d5f..3708a0ce80dc92 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -337,20 +337,32 @@ jobs: Samples: needs: [ Docker, Build, Smart_CI ] if: fromJSON(needs.smart_ci.outputs.affected_components).samples + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_samples_tests.yml with: runner: 'aks-linux-4-cores-16gb' image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} affected-components: ${{ needs.smart_ci.outputs.affected_components }} + os: ${{ matrix.os }} JS_API: name: OpenVINO JS API needs: [ Docker, Build, Smart_CI ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API uses: ./.github/workflows/job_openvino_js.yml with: runner: 'aks-linux-4-cores-16gb' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}"}' + os: ${{ matrix.os }} Conformance: needs: [ Build, Smart_CI ] @@ -358,7 +370,7 @@ jobs: defaults: run: shell: bash - runs-on: ubuntu-20.04-8-cores + runs-on: ubuntu-22.04-8-cores strategy: max-parallel: 2 fail-fast: false @@ -487,89 +499,143 @@ jobs: if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT || fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE needs: [ Build, Smart_CI, Docker ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_onnx_runtime.yml with: runner: 'aks-linux-16-cores-32gb' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' sccache-azure-key-prefix: 'ubuntu20_x86_64_onnxruntime' + os: ${{ matrix.os }} ONNX_Models: name: ONNX Models Tests if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test || fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test needs: [ Docker, Build, Smart_CI ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_onnx_models_tests.yml with: runner: 'aks-linux-16-cores-64gb' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' + os: ${{ matrix.os }} CXX_Unit_Tests: name: C++ unit tests needs: [ Docker, Build, Smart_CI ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_cxx_unit_tests.yml with: runner: 'aks-linux-4-cores-16gb' image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} affected-components: ${{ needs.smart_ci.outputs.affected_components }} + os: ${{ matrix.os }} Python_Unit_Tests: name: Python unit tests needs: [ Docker, Build, Smart_CI ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_python_unit_tests.yml with: runner: 'aks-linux-4-cores-16gb' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} + os: ${{ matrix.os }} TensorFlow_Layer_Tests: name: TensorFlow Layer Tests needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tensorflow_layer_tests.yml with: runner: 'aks-linux-4-cores-16gb' shell: bash container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} + os: ${{ matrix.os }} CPU_Functional_Tests: name: CPU functional tests if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test needs: [ Docker, Build, Smart_CI ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_cpu_functional_tests.yml with: runner: 'aks-linux-8-cores-32gb' image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} + os: ${{ matrix.os }} TensorFlow_Models_Tests_Precommit: name: TensorFlow Models tests if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-8-cores-16gb' model_scope: 'precommit' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' + os: ${{ matrix.os }} TensorFlow_Models_Tests_Nightly_TF_HUB: name: TensorFlow TF Hub Models tests if: ${{ github.event_name == 'schedule' }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-8-cores-64gb' model_scope: 'nightly_tf_hub' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' + os: ${{ matrix.os }} TensorFlow_Models_Tests_Nightly_HF: name: TensorFlow Hugging Face Models tests if: ${{ github.event_name == 'schedule' }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-8-cores-64gb' model_scope: 'nightly_hf' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' + os: ${{ matrix.os }} # TODO: Switch back to self-hosted runners # container: @@ -580,14 +646,25 @@ jobs: name: PyTorch Models tests if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test needs: [ Build, Smart_CI, Openvino_tokenizers ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_pytorch_models_tests.yml with: - runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }} + runner: ${{ github.event_name == 'schedule' && 'ubuntu-22.04-16-cores' || 'ubuntu-22.04-8-cores' }} event: ${{ github.event_name }} + os: ${{ matrix.os }} NVIDIA_Plugin: name: NVIDIA plugin needs: [ Docker, Build, Smart_CI ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} timeout-minutes: 15 defaults: run: @@ -667,12 +744,18 @@ jobs: Openvino_tokenizers: name: OpenVINO tokenizers extension needs: [ Build, Smart_CI, Docker ] + strategy: + matrix: + include: + - os: '_ubuntu_22' + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tokenizers.yml with: runner: 'aks-linux-4-cores-16gb' shell: bash container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} + os: ${{ matrix.os }} if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS iGPU: @@ -688,8 +771,9 @@ jobs: device: 'igpu' test_type: ${{ matrix.TEST_TYPE }} runner: "[ 'self-hosted', 'igpu' ]" - container: '{"image": "ubuntu:20.04", "volumes": ["/dev/dri:/dev/dri"], "options": "--group-add 109 --group-add 44 + container: '{"image": "ubuntu:22.04", "volumes": ["/dev/dri:/dev/dri"], "options": "--group-add 109 --group-add 44 --device /dev/dri:/dev/dri"}' + os: ${{ matrix.os }} if: fromJSON(needs.smart_ci.outputs.affected_components).GPU dGPU: @@ -705,8 +789,9 @@ jobs: device: 'dgpu' test_type: ${{ matrix.TEST_TYPE }} runner: "[ 'self-hosted', 'dgpu' ]" - container: '{"image": "ubuntu:20.04", "volumes": ["/dev/dri:/dev/dri"], "options": "--group-add 109 --group-add 44 + container: '{"image": "ubuntu:22.04", "volumes": ["/dev/dri:/dev/dri"], "options": "--group-add 109 --group-add 44 --device /dev/dri/card0:/dev/dri/card0 --device /dev/dri/renderD128:/dev/dri/renderD128"}' + os: ${{ matrix.os }} if: ${{ github.event_name == 'schedule' }} Overall_Status: From 07a4c808951211b5d2340a2b54fe162af63b4d90 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 26 Jun 2024 14:16:28 +0100 Subject: [PATCH 19/60] split workflows by os --- .../ubuntu_22_04_x64_nvidia/Dockerfile | 77 ++++ .github/workflows/job_build_linux.yml | 244 +++++++++++ .github/workflows/job_debian_packages.yml | 7 +- .github/workflows/ubuntu_20.yml | 136 ++++++ .../workflows/{linux.yml => ubuntu_22.yml} | 386 ++---------------- 5 files changed, 490 insertions(+), 360 deletions(-) create mode 100644 .github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile create mode 100644 .github/workflows/job_build_linux.yml create mode 100644 .github/workflows/ubuntu_20.yml rename .github/workflows/{linux.yml => ubuntu_22.yml} (55%) diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile new file mode 100644 index 00000000000000..d4a4c2d25bbbc9 --- /dev/null +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile @@ -0,0 +1,77 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/nvidia/cuda:11.8.0-runtime-ubuntu22.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install \ + curl \ + wget \ + git \ + ca-certificates \ + gpg-agent \ + tzdata \ + # Pythons + python3.8-dev \ + python3.8-venv \ + python3.8-distutils \ + python3.11-dev \ + python3.11-venv \ + python3.11-distutils \ + # For Java API + default-jdk \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install build dependencies +ADD install_build_dependencies.sh /install_build_dependencies.sh +RUN chmod +x /install_build_dependencies.sh && \ + /install_build_dependencies.sh && \ + rm -rf /var/lib/apt/lists/* + +# Install sscache +ARG SCCACHE_VERSION="v0.7.5" +ENV SCCACHE_HOME="/opt/sccache" \ + SCCACHE_PATH="/opt/sccache/sccache" + +RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ + SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \ + curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ + tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} + +# Install CUDA +RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb && \ + dpkg -i cuda-keyring_1.1-1_all.deb +RUN apt-get update && apt-get install -y cuda-runtime-11-8 cuda-11-8 && apt-get install -y \ + libcudnn8=8.9.4.*-1+cuda11.8 \ + libcudnn8-dev=8.9.4.*-1+cuda11.8 \ + libcudnn8-samples=8.9.4.*-1+cuda11.8 \ + libcutensor1=1.7.0.1-1 \ + libcutensor-dev=1.7.0.1-1 \ + cuda-drivers + +# Setup pip +ENV PIP_VERSION="24.0" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py + +# Use Python 3.11 as default instead of Python 3.8 +# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +RUN python3.11 -m venv venv +ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" + +ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml new file mode 100644 index 00000000000000..9d4de345c8ce57 --- /dev/null +++ b/.github/workflows/job_build_linux.yml @@ -0,0 +1,244 @@ +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' + type: string + required: false + default: '{"image": null}' + affected-components: + description: 'Components that are affected by changes in the commit defined by the Smart CI Action' + type: string + required: true + event-name: + description: 'Name of the event that triggered the workflow' + type: string + required: false + +permissions: read-all + +env: + PIP_CACHE_PATH: /mount/caches/pip/linux + PYTHON_VERSION: '3.11' + +jobs: + Build: + name: Build + timeout-minutes: 150 + runs-on: ${{ inputs.runner }} + container: ${{ fromJSON(inputs.container) }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + CMAKE_BUILD_TYPE: 'Release' + CMAKE_GENERATOR: 'Ninja Multi-Config' + CMAKE_CXX_COMPILER_LAUNCHER: sccache + CMAKE_C_COMPILER_LAUNCHER: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: 1 + SCCACHE_SERVER_PORT: 35555 + SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt + SCCACHE_LOG: warn + GITHUB_WORKSPACE: /__w/openvino/openvino + OPENVINO_REPO: /__w/openvino/openvino/openvino + OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib + INSTALL_DIR: /__w/openvino/openvino/openvino_install + INSTALL_DIR_JS: /__w/openvino/openvino/openvino_install/js + INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install + DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install + BUILD_DIR: /__w/openvino/openvino/openvino_build + SCCACHE_AZURE_KEY_PREFIX: ubuntu20_x86_64_Release + ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime + steps: + - name: Clone OpenVINO + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + path: ${{ env.OPENVINO_REPO }} + submodules: 'true' + + # Ticket: 139627 + - name: Checkout the latest OneDNN for GPU in nightly + if: ${{ github.event_name == 'schedule' }} + working-directory: ${{ env.OPENVINO_REPO }}/src/plugins/intel_gpu/thirdparty/onednn_gpu + run: | + git fetch origin + git checkout main + git rev-parse HEAD + + - name: Clone OpenVINO Contrib + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + repository: 'openvinotoolkit/openvino_contrib' + path: ${{ env.OPENVINO_CONTRIB_REPO }} + submodules: 'true' + ref: 'master' + + # + # Print system info + # + + - name: System info + uses: ./openvino/.github/actions/system_info + + # + # Dependencies + # + + - name: Install python dependencies + run: | + # For Python API: build and wheel packaging + python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt + + # For running ONNX frontend unit tests + python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt + + # For running TensorFlow frontend unit tests + python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow/tests/requirements.txt + + # For running TensorFlow Lite frontend unit tests + python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow_lite/tests/requirements.txt + + # For running Paddle frontend unit tests + python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/paddle/tests/requirements.txt + + # + # Build + # + + - name: CMake configure - OpenVINO + run: | + cmake \ + -G "${{ env.CMAKE_GENERATOR }}" \ + -DENABLE_CPPLINT=OFF \ + -DENABLE_NCC_STYLE=OFF \ + -DENABLE_TESTS=ON \ + -DENABLE_STRICT_DEPENDENCIES=OFF \ + -DENABLE_SYSTEM_OPENCL=ON \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCPACK_GENERATOR=TGZ \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ + -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \ + -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \ + -S ${OPENVINO_REPO} \ + -B ${BUILD_DIR} + + - name: Clean sccache stats + run: ${SCCACHE_PATH} --zero-stats + + - name: Cmake build - OpenVINO + run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: Show sccache stats + run: ${SCCACHE_PATH} --show-stats + + - name: Cmake install - OpenVINO + run: | + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${DEVELOPER_PACKAGE_DIR} -DCOMPONENT=developer_package -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake + + - name: Pack Artifacts + run: | + + # Add the ONNX Runtime version and skip tests list to the archive to use in the ONNX Runtime Job + # w/o the need to checkout repository + + cp -R ${ONNX_RUNTIME_UTILS} ${INSTALL_DIR} + + pushd ${INSTALL_DIR} + tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * + popd + + pushd ${DEVELOPER_PACKAGE_DIR} + tar -czvf ${BUILD_DIR}/openvino_developer_package.tar.gz * + popd + + pushd ${INSTALL_TEST_DIR} + tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz * + popd + + - name: Build Debian packages + run: | + python3 -m pip install -U pip + python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt + cmake -UPython* -U_Python* -UPYTHON* \ + -UTBB* \ + -DENABLE_SYSTEM_TBB=ON \ + -DENABLE_PYTHON_PACKAGING=ON \ + -DENABLE_TESTS=OFF \ + -DCPACK_GENERATOR=DEB \ + ${BUILD_DIR} + cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package + + - name: Cmake & Build - OpenVINO Contrib + run: | + cmake \ + -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \ + -DOPENVINO_EXTRA_MODULES="${OPENVINO_CONTRIB_REPO}/modules/java_api;${OPENVINO_CONTRIB_REPO}/modules/custom_operations" \ + -S ${OPENVINO_REPO} \ + -B ${BUILD_DIR} + cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: CMake configure, build and install - OpenVINO JS API + if: ${{ fromJSON(inputs.affected-components).JS_API }} + run: | + cmake -UTBB* -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -S ${OPENVINO_REPO} -B ${BUILD_DIR} + cmake --build ${BUILD_DIR} --parallel + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR_JS} -P ${BUILD_DIR}/cmake_install.cmake + + # + # Upload build artifacts and logs + # + - name: Upload build logs + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + if: always() + with: + name: build_logs + path: ${{ env.SCCACHE_ERROR_LOG }} + if-no-files-found: 'ignore' + + - name: Upload openvino package + if: ${{ always() }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: openvino_package + path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz + if-no-files-found: 'error' + + - name: Upload openvino js package + if: ${{ fromJSON(inputs.affected-components).JS_API }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: openvino_js_package + path: ${{ env.INSTALL_DIR_JS }} + if-no-files-found: 'error' + + - name: Upload openvino developer package + if: ${{ always() }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: openvino_developer_package + path: ${{ env.BUILD_DIR }}/openvino_developer_package.tar.gz + if-no-files-found: 'error' + + - name: Upload openvino debian packages + if: ${{ always() }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: openvino_debian_packages + path: ${{ env.BUILD_DIR }}/*.deb + if-no-files-found: 'error' + + - name: Upload openvino tests package + if: ${{ always() }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: openvino_tests + path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz + if-no-files-found: 'error' diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index 22d1564a5094dd..a59cec6cd1163c 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -12,11 +12,6 @@ on: type: string required: false default: null - os: - description: 'OS version needed to download correct artifacts in the form of "_ubuntu_22"' - type: string - required: false - default: null permissions: read-all @@ -40,7 +35,7 @@ jobs: - name: Download OpenVINO debian packages uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: - name: openvino_debian_packages${{ inputs.os }} + name: openvino_debian_packages path: ${{ env.DEBIAN_PACKAGES_DIR }} # Needed as ${{ github.workspace }} is not working correctly when using Docker diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml new file mode 100644 index 00000000000000..e5ce6cbf3b547c --- /dev/null +++ b/.github/workflows/ubuntu_20.yml @@ -0,0 +1,136 @@ +name: Linux (Ubuntu 20.04, Python 3.11) +on: + schedule: + # at 00:00 on Wednesday and Saturday + - cron: '0 0 * * 3,6' + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' + +concurrency: + # github.ref is not unique in post-commit + group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-ubuntu-20 + cancel-in-progress: true + +permissions: read-all + +env: + PIP_CACHE_PATH: /mount/caches/pip/linux + PYTHON_VERSION: '3.11' + +jobs: + Smart_CI: + runs-on: ubuntu-latest + outputs: + affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" + skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" + steps: + - name: checkout action + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + sparse-checkout: .github/actions/smart-ci + + - name: Get affected components + id: smart_ci + uses: ./.github/actions/smart-ci + with: + repository: ${{ github.repository }} + pr: ${{ github.event.number }} + commit_sha: ${{ github.sha }} + ref_name: ${{ github.ref_name }} + component_pattern: "category: (.*)" + repo_token: ${{ secrets.GITHUB_TOKEN }} + skip_when_only_listed_labels_set: 'docs' + skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg' + + - name: Show affected components + run: | + echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}" + shell: bash + + Docker: + needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_20_04_x64 + ov_build/ubuntu_20_04_x64_nvidia + ov_test/ubuntu_20_04_x64 + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + + Build: + needs: [Docker, Smart_CI] + if: "!needs.smart_ci.outputs.skip_workflow" + uses: ./.github/workflows/job_build_linux.yml + with: + runner: 'aks-linux-16-cores-32gb' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + event-name: ${{ github.event_name }} + + iGPU: + name: iGPU Tests + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_gpu_tests.yml + strategy: + max-parallel: 2 + fail-fast: false + matrix: + TEST_TYPE: ['unit', 'func'] + with: + device: 'igpu' + test_type: ${{ matrix.TEST_TYPE }} + runner: "[ 'self-hosted', 'igpu' ]" + container: '{"image": "ubuntu:20.04", "volumes": ["/dev/dri:/dev/dri"], "options": "--group-add 109 --group-add 44 + --device /dev/dri:/dev/dri"}' + if: fromJSON(needs.smart_ci.outputs.affected_components).GPU + + dGPU: + name: dGPU Tests + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_gpu_tests.yml + strategy: + max-parallel: 2 + fail-fast: false + matrix: + TEST_TYPE: ['unit', 'func'] + with: + device: 'dgpu' + test_type: ${{ matrix.TEST_TYPE }} + runner: "[ 'self-hosted', 'dgpu' ]" + container: '{"image": "ubuntu:20.04", "volumes": ["/dev/dri:/dev/dri"], "options": "--group-add 109 --group-add 44 + --device /dev/dri/card0:/dev/dri/card0 --device /dev/dri/renderD128:/dev/dri/renderD128"}' + if: ${{ github.event_name == 'schedule' }} + + Overall_Status: + name: ci/gha_overall_status + needs: [Smart_CI, Build, iGPU] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Check status of all jobs + if: >- + ${{ + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') + }} + run: exit 1 diff --git a/.github/workflows/linux.yml b/.github/workflows/ubuntu_22.yml similarity index 55% rename from .github/workflows/linux.yml rename to .github/workflows/ubuntu_22.yml index 3708a0ce80dc92..82dd3caf7c7ae1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/ubuntu_22.yml @@ -1,4 +1,4 @@ -name: Linux (Ubuntu 20.04, 22.04, 24.04) +name: Linux (Ubuntu 22.04, Python 3.11) on: schedule: # at 00:00 on Wednesday and Saturday @@ -13,7 +13,7 @@ on: concurrency: # github.ref is not unique in post-commit - group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux + group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-ubuntu-22 cancel-in-progress: true permissions: read-all @@ -70,299 +70,48 @@ jobs: id: handle_docker with: images: | - ov_build/ubuntu_20_04_x64 ov_build/ubuntu_22_04_x64 - ov_build/ubuntu_24_04_x64 - ov_build/ubuntu_20_04_x64_nvidia - ov_test/ubuntu_20_04_x64 + ov_build/ubuntu_22_04_x64_nvidia ov_test/ubuntu_22_04_x64 registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} Build: - name: Build (${{ matrix.os }}) needs: [Docker, Smart_CI] - timeout-minutes: 150 - defaults: - run: - shell: bash - runs-on: aks-linux-16-cores-32gb - strategy: - max-parallel: 3 - fail-fast: false - matrix: - include: - - os: 'ubuntu_20' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} - - os: 'ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} - - os: 'ubuntu_24' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_24_04_x64 }} - container: - image: ${{ matrix.image }} - volumes: - - /mount:/mount - options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING - env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - CMAKE_BUILD_TYPE: 'Release' - CMAKE_GENERATOR: 'Ninja Multi-Config' - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: 1 - SCCACHE_SERVER_PORT: 35555 - SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt - SCCACHE_LOG: warn - GITHUB_WORKSPACE: '/__w/openvino/openvino' - OPENVINO_REPO: /__w/openvino/openvino/openvino - OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib - INSTALL_DIR: /__w/openvino/openvino/openvino_install - INSTALL_DIR_JS: /__w/openvino/openvino/openvino_install/js - INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install - DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install - BUILD_DIR: /__w/openvino/openvino/openvino_build - SCCACHE_AZURE_KEY_PREFIX: ${{ matrix.os }}_x86_64_Release - ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime if: "!needs.smart_ci.outputs.skip_workflow" - - steps: - - name: Clone OpenVINO - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - path: ${{ env.OPENVINO_REPO }} - submodules: 'true' - - # Ticket: 139627 - - name: Checkout the latest OneDNN for GPU in nightly - if: ${{ github.event_name == 'schedule' }} - working-directory: ${{ env.OPENVINO_REPO }}/src/plugins/intel_gpu/thirdparty/onednn_gpu - run: | - git fetch origin - git checkout main - git rev-parse HEAD - - - name: Clone OpenVINO Contrib - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - repository: 'openvinotoolkit/openvino_contrib' - path: ${{ env.OPENVINO_CONTRIB_REPO }} - submodules: 'true' - ref: 'master' - - # - # Print system info - # - - - name: System info - uses: ./openvino/.github/actions/system_info - - # - # Dependencies - # - - - name: Install python dependencies - run: | - # For Python API: build and wheel packaging - python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - - # For running ONNX frontend unit tests - if [[ ${{ matrix.os }} == 'ubuntu_24' ]]; then - export CMAKE_GENERATOR="Unix Makefiles" - python3 -m pip install --force-reinstall --no-cache-dir -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt - else - python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt - fi - - # For running TensorFlow frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow/tests/requirements.txt - - # For running TensorFlow Lite frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow_lite/tests/requirements.txt - - # For running Paddle frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/paddle/tests/requirements.txt - - # - # Build - # - - - name: CMake configure - OpenVINO - run: | - cmake \ - -G "${{ env.CMAKE_GENERATOR }}" \ - -DENABLE_CPPLINT=OFF \ - -DENABLE_NCC_STYLE=OFF \ - -DENABLE_TESTS=ON \ - -DENABLE_STRICT_DEPENDENCIES=OFF \ - -DENABLE_SYSTEM_OPENCL=ON \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCPACK_GENERATOR=TGZ \ - -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ - -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \ - -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \ - -S ${OPENVINO_REPO} \ - -B ${BUILD_DIR} - - - name: Clean sccache stats - run: ${SCCACHE_PATH} --zero-stats - - - name: Cmake build - OpenVINO - run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: Show sccache stats - run: ${SCCACHE_PATH} --show-stats - - - name: Cmake install - OpenVINO - run: | - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${DEVELOPER_PACKAGE_DIR} -DCOMPONENT=developer_package -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake - - - name: Pack Artifacts - run: | - - # Add the ONNX Runtime version and skip tests list to the archive to use in the ONNX Runtime Job - # w/o the need to checkout repository - - cp -R ${ONNX_RUNTIME_UTILS} ${INSTALL_DIR} - - pushd ${INSTALL_DIR} - tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * - popd - - pushd ${DEVELOPER_PACKAGE_DIR} - tar -czvf ${BUILD_DIR}/openvino_developer_package.tar.gz * - popd - - pushd ${INSTALL_TEST_DIR} - tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz * - popd - - - name: Build Debian packages - run: | - python3 -m pip install -U pip - python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - cmake -UPython* -U_Python* -UPYTHON* \ - -UTBB* \ - -DENABLE_SYSTEM_TBB=ON \ - -DENABLE_PYTHON_PACKAGING=ON \ - -DENABLE_TESTS=OFF \ - -DCPACK_GENERATOR=DEB \ - ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package - - - name: Cmake & Build - OpenVINO Contrib - run: | - cmake \ - -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \ - -DOPENVINO_EXTRA_MODULES="${OPENVINO_CONTRIB_REPO}/modules/java_api;${OPENVINO_CONTRIB_REPO}/modules/custom_operations" \ - -S ${OPENVINO_REPO} \ - -B ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: CMake configure, build and install - OpenVINO JS API - if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API - run: | - cmake -UTBB* -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -S ${OPENVINO_REPO} -B ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR_JS} -P ${BUILD_DIR}/cmake_install.cmake - - # - # Upload build artifacts and logs - # - - name: Upload build logs - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - if: always() - with: - name: build_logs_${{ matrix.os }} - path: ${{ env.SCCACHE_ERROR_LOG }} - if-no-files-found: 'ignore' - - - name: Upload openvino package - if: ${{ always() }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_package_${{ matrix.os }} - path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz - if-no-files-found: 'error' - - - name: Upload openvino js package - if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_js_package_${{ matrix.os }} - path: ${{ env.INSTALL_DIR_JS }} - if-no-files-found: 'error' - - - name: Upload openvino developer package - if: ${{ always() }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_developer_package_${{ matrix.os }} - path: ${{ env.BUILD_DIR }}/openvino_developer_package.tar.gz - if-no-files-found: 'error' - - - name: Upload openvino debian packages - if: ${{ always() }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_debian_packages_${{ matrix.os }} - path: ${{ env.BUILD_DIR }}/*.deb - if-no-files-found: 'error' - - - name: Upload openvino tests package - if: ${{ always() }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_tests_${{ matrix.os }} - path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz - if-no-files-found: 'error' + uses: ./.github/workflows/job_build_linux.yml + with: + runner: 'aks-linux-16-cores-32gb' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + event-name: ${{ github.event_name }} Debian_Packages: name: Debian Packages - needs: [Build, Docker] + needs: Build uses: ./.github/workflows/job_debian_packages.yml - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} with: runner: 'aks-linux-4-cores-16gb' - image: ${{ matrix.image }} - os: ${{ matrix.os }} + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04' Samples: needs: [ Docker, Build, Smart_CI ] if: fromJSON(needs.smart_ci.outputs.affected_components).samples - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_samples_tests.yml with: runner: 'aks-linux-4-cores-16gb' - image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} + image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_22_04_x64 }} affected-components: ${{ needs.smart_ci.outputs.affected_components }} - os: ${{ matrix.os }} JS_API: name: OpenVINO JS API needs: [ Docker, Build, Smart_CI ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API uses: ./.github/workflows/job_openvino_js.yml with: runner: 'aks-linux-4-cores-16gb' - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}"}' - os: ${{ matrix.os }} + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}"}' Conformance: needs: [ Build, Smart_CI ] @@ -370,7 +119,7 @@ jobs: defaults: run: shell: bash - runs-on: ubuntu-22.04-8-cores + runs-on: ubuntu-20.04-8-cores strategy: max-parallel: 2 fail-fast: false @@ -499,179 +248,114 @@ jobs: if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT || fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE needs: [ Build, Smart_CI, Docker ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_onnx_runtime.yml with: runner: 'aks-linux-16-cores-32gb' - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' sccache-azure-key-prefix: 'ubuntu20_x86_64_onnxruntime' - os: ${{ matrix.os }} ONNX_Models: name: ONNX Models Tests if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test || fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test needs: [ Docker, Build, Smart_CI ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_onnx_models_tests.yml with: runner: 'aks-linux-16-cores-64gb' - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' - os: ${{ matrix.os }} + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"]}' CXX_Unit_Tests: name: C++ unit tests needs: [ Docker, Build, Smart_CI ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_cxx_unit_tests.yml with: runner: 'aks-linux-4-cores-16gb' - image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} + image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_22_04_x64 }} affected-components: ${{ needs.smart_ci.outputs.affected_components }} - os: ${{ matrix.os }} Python_Unit_Tests: name: Python unit tests needs: [ Docker, Build, Smart_CI ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_python_unit_tests.yml with: runner: 'aks-linux-4-cores-16gb' - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} - os: ${{ matrix.os }} TensorFlow_Layer_Tests: name: TensorFlow Layer Tests needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tensorflow_layer_tests.yml with: runner: 'aks-linux-4-cores-16gb' shell: bash - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} - os: ${{ matrix.os }} CPU_Functional_Tests: name: CPU functional tests if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test needs: [ Docker, Build, Smart_CI ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_cpu_functional_tests.yml with: runner: 'aks-linux-8-cores-32gb' - image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} - os: ${{ matrix.os }} + image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_22_04_x64 }} TensorFlow_Models_Tests_Precommit: name: TensorFlow Models tests if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-8-cores-16gb' model_scope: 'precommit' - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' - os: ${{ matrix.os }} + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"]}' TensorFlow_Models_Tests_Nightly_TF_HUB: name: TensorFlow TF Hub Models tests if: ${{ github.event_name == 'schedule' }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-8-cores-64gb' model_scope: 'nightly_tf_hub' - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' - os: ${{ matrix.os }} + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"]}' TensorFlow_Models_Tests_Nightly_HF: name: TensorFlow Hugging Face Models tests if: ${{ github.event_name == 'schedule' }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-8-cores-64gb' model_scope: 'nightly_hf' - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' - os: ${{ matrix.os }} + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"]}' # TODO: Switch back to self-hosted runners # container: - # image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + # image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04 # volumes: # - /mount:/mount PyTorch_Models_Tests: name: PyTorch Models tests if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test needs: [ Build, Smart_CI, Openvino_tokenizers ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_pytorch_models_tests.yml with: - runner: ${{ github.event_name == 'schedule' && 'ubuntu-22.04-16-cores' || 'ubuntu-22.04-8-cores' }} + runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }} event: ${{ github.event_name }} - os: ${{ matrix.os }} NVIDIA_Plugin: name: NVIDIA plugin needs: [ Docker, Build, Smart_CI ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} timeout-minutes: 15 defaults: run: shell: bash runs-on: aks-linux-16-cores-32gb container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64_nvidia }} + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64_nvidia }} volumes: - /mount:/mount options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING @@ -744,18 +428,12 @@ jobs: Openvino_tokenizers: name: OpenVINO tokenizers extension needs: [ Build, Smart_CI, Docker ] - strategy: - matrix: - include: - - os: '_ubuntu_22' - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }} uses: ./.github/workflows/job_tokenizers.yml with: runner: 'aks-linux-4-cores-16gb' shell: bash - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} - os: ${{ matrix.os }} if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS iGPU: @@ -773,8 +451,8 @@ jobs: runner: "[ 'self-hosted', 'igpu' ]" container: '{"image": "ubuntu:22.04", "volumes": ["/dev/dri:/dev/dri"], "options": "--group-add 109 --group-add 44 --device /dev/dri:/dev/dri"}' - os: ${{ matrix.os }} - if: fromJSON(needs.smart_ci.outputs.affected_components).GPU + # if: fromJSON(needs.smart_ci.outputs.affected_components).GPU + if: ${{ 'false' }} # Ticket: 143677 dGPU: name: dGPU Tests @@ -791,13 +469,13 @@ jobs: runner: "[ 'self-hosted', 'dgpu' ]" container: '{"image": "ubuntu:22.04", "volumes": ["/dev/dri:/dev/dri"], "options": "--group-add 109 --group-add 44 --device /dev/dri/card0:/dev/dri/card0 --device /dev/dri/renderD128:/dev/dri/renderD128"}' - os: ${{ matrix.os }} - if: ${{ github.event_name == 'schedule' }} + # if: ${{ github.event_name == 'schedule' }} + if: ${{ 'false' }} # Ticket: 143677 Overall_Status: name: ci/gha_overall_status needs: [Smart_CI, Build, Debian_Packages, Samples, Conformance, ONNX_Runtime, CXX_Unit_Tests, Python_Unit_Tests, TensorFlow_Layer_Tests, - CPU_Functional_Tests, TensorFlow_Models_Tests_Precommit, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers, iGPU] + CPU_Functional_Tests, TensorFlow_Models_Tests_Precommit, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers] if: ${{ always() }} runs-on: ubuntu-latest steps: From d47ee4b7b9cfa6fac7079bb4908ad5168c195538 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 26 Jun 2024 14:21:24 +0100 Subject: [PATCH 20/60] update docker tag --- .github/dockerfiles/docker_tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index 219e1c9dab4184..6eedba0f7e9055 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-25130 \ No newline at end of file +pr-25234 \ No newline at end of file From ec9d2102a0d78c1a40fc23fb29001f220d5615a6 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 26 Jun 2024 15:30:38 +0100 Subject: [PATCH 21/60] provide os --- .github/workflows/job_build_linux.yml | 6 +++++- .github/workflows/ubuntu_20.yml | 5 ++--- .github/workflows/ubuntu_22.yml | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index 9d4de345c8ce57..67750a82f3141c 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -18,6 +18,10 @@ on: description: 'Name of the event that triggered the workflow' type: string required: false + os: + description: 'OS that is used for building in the form of "ubuntu20"' + type: string + required: true permissions: read-all @@ -52,7 +56,7 @@ jobs: INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install BUILD_DIR: /__w/openvino/openvino/openvino_build - SCCACHE_AZURE_KEY_PREFIX: ubuntu20_x86_64_Release + SCCACHE_AZURE_KEY_PREFIX: ${{ inputs.os }}_x86_64_Release ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime steps: - name: Clone OpenVINO diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml index e5ce6cbf3b547c..1962d9555e8f52 100644 --- a/.github/workflows/ubuntu_20.yml +++ b/.github/workflows/ubuntu_20.yml @@ -71,8 +71,6 @@ jobs: with: images: | ov_build/ubuntu_20_04_x64 - ov_build/ubuntu_20_04_x64_nvidia - ov_test/ubuntu_20_04_x64 registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} @@ -86,6 +84,7 @@ jobs: container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} event-name: ${{ github.event_name }} + os: 'ubuntu20' iGPU: name: iGPU Tests @@ -122,7 +121,7 @@ jobs: if: ${{ github.event_name == 'schedule' }} Overall_Status: - name: ci/gha_overall_status + name: ci/gha_overall_status_ubuntu_20 needs: [Smart_CI, Build, iGPU] if: ${{ always() }} runs-on: ubuntu-latest diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index a250c9243e3ae7..ad3e989cb1f6b8 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -86,6 +86,7 @@ jobs: container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} event-name: ${{ github.event_name }} + os: 'ubuntu22' Debian_Packages: name: Debian Packages @@ -473,7 +474,7 @@ jobs: if: ${{ 'false' }} # Ticket: 143677 Overall_Status: - name: ci/gha_overall_status + name: ci/gha_overall_status_ubuntu_22 needs: [Smart_CI, Build, Debian_Packages, Samples, Conformance, ONNX_Runtime, CXX_Unit_Tests, Python_Unit_Tests, TensorFlow_Layer_Tests, CPU_Functional_Tests, TensorFlow_Models_Tests_Precommit, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers] if: ${{ always() }} From 22054dd47236fce020cef270a92965f6dc0bdca7 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 27 Jun 2024 11:40:30 +0100 Subject: [PATCH 22/60] use 3.11 --- .../ov_build/ubuntu_22_04_x64/Dockerfile | 21 ++++++++----------- .github/workflows/job_debian_packages.yml | 9 ++++---- .github/workflows/linux_arm64.yml | 2 +- .github/workflows/ubuntu_22.yml | 2 +- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index ec2bc87000a760..dfc40fbce2594b 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -23,15 +23,15 @@ RUN apt-get update && \ gpg-agent \ tzdata \ libtbb2 \ - # Python + # Pythons \ + python3.11-dev \ + python3.11-venv \ + python3.11-distutils \ python3-dev \ python3-venv \ python3-pip \ # For Java API default-jdk \ - # Compiler \ - gcc-10 \ - g++-10 \ && \ rm -rf /var/lib/apt/lists/* @@ -41,10 +41,6 @@ RUN chmod +x /install_build_dependencies.sh && \ /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* -# Set gcc-10 as a default compiler -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 - # Install sscache ARG SCCACHE_VERSION="v0.7.5" ENV SCCACHE_HOME="/opt/sccache" \ @@ -60,11 +56,12 @@ ENV PATH="$SCCACHE_HOME:$PATH" ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py -# Use Python 3.11 as default instead of Python 3.8 -# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build -#RUN python3.11 -m venv venv -#ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" +# Use Python 3.11 as default +# Using venv here 'cause other methods to switch the default Python on Ubuntu break both system and wheels build +RUN python3.11 -m venv venv +ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index fc407297117680..5bffce9739ead6 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -7,11 +7,11 @@ on: description: 'Machine on which the tests would run' type: string required: true - image: - description: 'Docker image in which the tests would run' + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' type: string required: false - default: null + default: '{"image": null}' permissions: read-all @@ -19,8 +19,7 @@ jobs: Debian_Packages: name: Debian Packages runs-on: ${{ inputs.runner }} - container: - image: ${{ inputs.image }} + container: ${{ fromJSON(inputs.container) }} defaults: run: shell: bash diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index 9156fa711e752a..84c71328d429b5 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -301,7 +301,7 @@ jobs: uses: ./.github/workflows/job_debian_packages.yml with: runner: 'aks-linux-16-cores-arm' - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_arm64 }}"}' Samples: needs: [ Build, Docker, Smart_CI ] diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index ad3e989cb1f6b8..222b358364c5d7 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -94,7 +94,7 @@ jobs: uses: ./.github/workflows/job_debian_packages.yml with: runner: 'aks-linux-4-cores-16gb' - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}"}' Samples: needs: [ Docker, Build, Smart_CI ] From 0822274f7f98348cb13e08d2facf98a5fe47dfc1 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 27 Jun 2024 13:02:04 +0100 Subject: [PATCH 23/60] add U24 --- .../ov_build/ubuntu_24_04_x64/Dockerfile | 7 -- .github/workflows/job_build_linux.yml | 10 +- .github/workflows/ubuntu_24.yml | 98 +++++++++++++++++++ 3 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/ubuntu_24.yml diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 5963d91d650ed6..7e47c08cf8fe8c 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -29,9 +29,6 @@ RUN apt-get update && \ python3-full \ # For Java API default-jdk \ - # Compiler \ - gcc-10 \ - g++-10 \ && \ rm -rf /var/lib/apt/lists/* @@ -41,10 +38,6 @@ RUN chmod +x /install_build_dependencies.sh && \ /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* -# Set gcc-10 as a default compiler -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 - # Install sscache ARG SCCACHE_VERSION="v0.7.5" ENV SCCACHE_HOME="/opt/sccache" \ diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index 67750a82f3141c..a8460cb54417dd 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -67,7 +67,7 @@ jobs: # Ticket: 139627 - name: Checkout the latest OneDNN for GPU in nightly - if: ${{ github.event_name == 'schedule' }} + if: ${{ inputs.event-name == 'schedule' }} working-directory: ${{ env.OPENVINO_REPO }}/src/plugins/intel_gpu/thirdparty/onednn_gpu run: | git fetch origin @@ -99,7 +99,13 @@ jobs: python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt # For running ONNX frontend unit tests - python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt + if [[ ${{ inputs.os }} == 'ubuntu24' ]]; then + # Should be removed after https://github.com/openvinotoolkit/openvino/pull/24242 is merged + export CMAKE_GENERATOR="Unix Makefiles" + python3 -m pip install --force-reinstall --no-cache-dir -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt + else + python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt + fi # For running TensorFlow frontend unit tests python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow/tests/requirements.txt diff --git a/.github/workflows/ubuntu_24.yml b/.github/workflows/ubuntu_24.yml new file mode 100644 index 00000000000000..a6746166f6a675 --- /dev/null +++ b/.github/workflows/ubuntu_24.yml @@ -0,0 +1,98 @@ +name: Linux (Ubuntu 24.04, Python 3.12) +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' + +concurrency: + # github.ref is not unique in post-commit + group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-ubuntu-24 + cancel-in-progress: true + +permissions: read-all + +env: + PIP_CACHE_PATH: /mount/caches/pip/linux + PYTHON_VERSION: '3.11' + +jobs: + Smart_CI: + runs-on: ubuntu-latest + outputs: + affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" + skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" + steps: + - name: checkout action + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + sparse-checkout: .github/actions/smart-ci + + - name: Get affected components + id: smart_ci + uses: ./.github/actions/smart-ci + with: + repository: ${{ github.repository }} + pr: ${{ github.event.number }} + commit_sha: ${{ github.sha }} + ref_name: ${{ github.ref_name }} + component_pattern: "category: (.*)" + repo_token: ${{ secrets.GITHUB_TOKEN }} + skip_when_only_listed_labels_set: 'docs' + skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg' + + - name: Show affected components + run: | + echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}" + shell: bash + + Docker: + needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_24_04_x64 + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + + Build: + needs: [Docker, Smart_CI] + if: "!needs.smart_ci.outputs.skip_workflow" + uses: ./.github/workflows/job_build_linux.yml + with: + runner: 'aks-linux-16-cores-32gb' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_24_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + event-name: ${{ github.event_name }} + os: 'ubuntu24' + + Overall_Status: + name: ci/gha_overall_status_ubuntu_24 + needs: [Smart_CI, Build] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Check status of all jobs + if: >- + ${{ + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') + }} + run: exit 1 From fdfc9e07a8d7da19f1d1399cb9e07d26b4845473 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 27 Jun 2024 13:13:45 +0100 Subject: [PATCH 24/60] compiler --- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 7e47c08cf8fe8c..5963d91d650ed6 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -29,6 +29,9 @@ RUN apt-get update && \ python3-full \ # For Java API default-jdk \ + # Compiler \ + gcc-10 \ + g++-10 \ && \ rm -rf /var/lib/apt/lists/* @@ -38,6 +41,10 @@ RUN chmod +x /install_build_dependencies.sh && \ /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* +# Set gcc-10 as a default compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 + # Install sscache ARG SCCACHE_VERSION="v0.7.5" ENV SCCACHE_HOME="/opt/sccache" \ From 562b3da18faa7ca9ae2098429ad8d0d3b030ef0b Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 27 Jun 2024 14:24:54 +0100 Subject: [PATCH 25/60] fix image names --- .github/workflows/ubuntu_22.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 222b358364c5d7..ee6ab10f1fb5af 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -120,7 +120,7 @@ jobs: defaults: run: shell: bash - runs-on: ubuntu-20.04-8-cores + runs-on: ubuntu-22.04-8-cores strategy: max-parallel: 2 fail-fast: false @@ -253,7 +253,7 @@ jobs: with: runner: 'aks-linux-16-cores-32gb' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' - sccache-azure-key-prefix: 'ubuntu20_x86_64_onnxruntime' + sccache-azure-key-prefix: 'ubuntu22_x86_64_onnxruntime' ONNX_Models: name: ONNX Models Tests @@ -344,7 +344,7 @@ jobs: needs: [ Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: - runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }} + runner: ${{ github.event_name == 'schedule' && 'ubuntu-22.04-16-cores' || 'ubuntu-22.04-8-cores' }} event: ${{ github.event_name }} NVIDIA_Plugin: @@ -376,7 +376,7 @@ jobs: OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib NVIDIA_BUILD_DIR: /__w/openvino/openvino/nvidia_plugin_build DEBIAN_FRONTEND: 'noninteractive' - SCCACHE_AZURE_KEY_PREFIX: ubuntu20_x86_64_Release + SCCACHE_AZURE_KEY_PREFIX: ubuntu22_x86_64_Release if: fromJSON(needs.smart_ci.outputs.affected_components).NVIDIA steps: From 20cf3d795360bb0ac58b238865558452b65e3e82 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 27 Jun 2024 14:29:18 +0100 Subject: [PATCH 26/60] use sys python across all Ubuntus --- .../ov_build/ubuntu_20_04_x64/Dockerfile | 5 ----- .../ov_build/ubuntu_22_04_x64/Dockerfile | 11 +---------- .../ov_test/ubuntu_22_04_x64/Dockerfile | 15 +++++---------- .github/workflows/ubuntu_20.yml | 3 +-- .github/workflows/ubuntu_22.yml | 4 ++-- 5 files changed, 9 insertions(+), 29 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index 806d0702fea704..f4260560a409ec 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -61,9 +61,4 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py -# Use Python 3.11 as default instead of Python 3.8 -# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build -#RUN python3.11 -m venv venv -#ENV PATH="/venv/bin:$PATH" - ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index dfc40fbce2594b..c747ef481949b3 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -23,10 +23,7 @@ RUN apt-get update && \ gpg-agent \ tzdata \ libtbb2 \ - # Pythons \ - python3.11-dev \ - python3.11-venv \ - python3.11-distutils \ + # Pythons python3-dev \ python3-venv \ python3-pip \ @@ -56,12 +53,6 @@ ENV PATH="$SCCACHE_HOME:$PATH" ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py -# Use Python 3.11 as default -# Using venv here 'cause other methods to switch the default Python on Ubuntu break both system and wheels build -RUN python3.11 -m venv venv -ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" - ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile index 1566c2305d04b7..544c5deb07c7c3 100644 --- a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile @@ -22,10 +22,11 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ - # Python - python3.11-dev \ - python3.11-venv \ - python3.11-distutils \ + # Python \ + python3 \ + python3-dev \ + python3-venv \ + python3-distutils \ libhdf5-dev \ && \ rm -rf /var/lib/apt/lists/* @@ -40,13 +41,7 @@ RUN chmod +x /install_build_dependencies.sh && \ ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py -# Use Python 3.11 as default -# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build -RUN python3.11 -m venv venv -ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" - ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} ENV PIP_INSTALL_PATH=/venv/lib/python3.11/site-packages diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml index 1962d9555e8f52..0bbc0e32d6b864 100644 --- a/.github/workflows/ubuntu_20.yml +++ b/.github/workflows/ubuntu_20.yml @@ -1,4 +1,4 @@ -name: Linux (Ubuntu 20.04, Python 3.11) +name: Linux (Ubuntu 20.04, Python 3.8) on: schedule: # at 00:00 on Wednesday and Saturday @@ -20,7 +20,6 @@ permissions: read-all env: PIP_CACHE_PATH: /mount/caches/pip/linux - PYTHON_VERSION: '3.11' jobs: Smart_CI: diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index ee6ab10f1fb5af..735994c85b83b0 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -1,4 +1,4 @@ -name: Linux (Ubuntu 22.04, Python 3.11) +name: Linux (Ubuntu 22.04, Python 3.10) on: schedule: # at 00:00 on Wednesday and Saturday @@ -20,7 +20,7 @@ permissions: read-all env: PIP_CACHE_PATH: /mount/caches/pip/linux - PYTHON_VERSION: '3.11' + PYTHON_VERSION: '3.10' jobs: Smart_CI: From 0e27dde92dfef5067343b8bcbc2e9c7a57f127a8 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 27 Jun 2024 15:28:37 +0100 Subject: [PATCH 27/60] use 3.11 in U22 across all workflow --- .../ov_build/ubuntu_22_04_x64/Dockerfile | 11 ++++++++++- .../ov_test/ubuntu_22_04_x64/Dockerfile | 15 ++++++++++----- .github/workflows/ubuntu_22.yml | 4 ++-- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index c747ef481949b3..dfc40fbce2594b 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -23,7 +23,10 @@ RUN apt-get update && \ gpg-agent \ tzdata \ libtbb2 \ - # Pythons + # Pythons \ + python3.11-dev \ + python3.11-venv \ + python3.11-distutils \ python3-dev \ python3-venv \ python3-pip \ @@ -53,6 +56,12 @@ ENV PATH="$SCCACHE_HOME:$PATH" ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py +# Use Python 3.11 as default +# Using venv here 'cause other methods to switch the default Python on Ubuntu break both system and wheels build +RUN python3.11 -m venv venv +ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" + ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile index 544c5deb07c7c3..1566c2305d04b7 100644 --- a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile @@ -22,11 +22,10 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ - # Python \ - python3 \ - python3-dev \ - python3-venv \ - python3-distutils \ + # Python + python3.11-dev \ + python3.11-venv \ + python3.11-distutils \ libhdf5-dev \ && \ rm -rf /var/lib/apt/lists/* @@ -41,7 +40,13 @@ RUN chmod +x /install_build_dependencies.sh && \ ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py +# Use Python 3.11 as default +# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +RUN python3.11 -m venv venv +ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" + ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} ENV PIP_INSTALL_PATH=/venv/lib/python3.11/site-packages diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 735994c85b83b0..ee6ab10f1fb5af 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -1,4 +1,4 @@ -name: Linux (Ubuntu 22.04, Python 3.10) +name: Linux (Ubuntu 22.04, Python 3.11) on: schedule: # at 00:00 on Wednesday and Saturday @@ -20,7 +20,7 @@ permissions: read-all env: PIP_CACHE_PATH: /mount/caches/pip/linux - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' jobs: Smart_CI: From 68a3da6d4cb7f314882d2dbe015c5b295edfc667 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 27 Jun 2024 16:23:34 +0100 Subject: [PATCH 28/60] use proper name --- .github/workflows/ubuntu_22.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index ee6ab10f1fb5af..fcb91fadc314c7 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -120,7 +120,7 @@ jobs: defaults: run: shell: bash - runs-on: ubuntu-22.04-8-cores + runs-on: ubuntu-latest-8-cores strategy: max-parallel: 2 fail-fast: false @@ -344,7 +344,7 @@ jobs: needs: [ Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: - runner: ${{ github.event_name == 'schedule' && 'ubuntu-22.04-16-cores' || 'ubuntu-22.04-8-cores' }} + runner: ${{ github.event_name == 'schedule' && 'ubuntu-latest-16-cores' || 'ubuntu-latest-8-cores' }} event: ${{ github.event_name }} NVIDIA_Plugin: From 4b7b953ad4966cc165c330bb6c80f9edee1e1570 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 27 Jun 2024 16:57:19 +0100 Subject: [PATCH 29/60] use dedicated names --- .github/workflows/ubuntu_22.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index fcb91fadc314c7..ee6ab10f1fb5af 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -120,7 +120,7 @@ jobs: defaults: run: shell: bash - runs-on: ubuntu-latest-8-cores + runs-on: ubuntu-22.04-8-cores strategy: max-parallel: 2 fail-fast: false @@ -344,7 +344,7 @@ jobs: needs: [ Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: - runner: ${{ github.event_name == 'schedule' && 'ubuntu-latest-16-cores' || 'ubuntu-latest-8-cores' }} + runner: ${{ github.event_name == 'schedule' && 'ubuntu-22.04-16-cores' || 'ubuntu-22.04-8-cores' }} event: ${{ github.event_name }} NVIDIA_Plugin: From 4109000fbd6873c5a9c1218dd0005a5fea4c0339 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 28 Jun 2024 09:12:54 +0100 Subject: [PATCH 30/60] update runner; update wf name in docs --- .github/workflows/ubuntu_22.yml | 2 +- docs/dev/ci/github_actions/adding_tests.md | 2 +- docs/dev/ci/github_actions/caches.md | 4 ++-- docs/dev/ci/github_actions/docker_images.md | 4 ++-- docs/dev/ci/github_actions/overview.md | 8 ++++---- docs/dev/ci/github_actions/reusable_workflows.md | 6 +++--- docs/dev/ci/github_actions/runners.md | 6 +++--- docs/dev/ci/github_actions/smart_ci.md | 2 +- .../functional/plugin/conformance/test_runner/README.md | 2 +- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index ee6ab10f1fb5af..550a449c06c2a9 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -120,7 +120,7 @@ jobs: defaults: run: shell: bash - runs-on: ubuntu-22.04-8-cores + runs-on: ubuntu-22.04-16-cores strategy: max-parallel: 2 fail-fast: false diff --git a/docs/dev/ci/github_actions/adding_tests.md b/docs/dev/ci/github_actions/adding_tests.md index 1078c5c9efe3c6..a31cd4ded0c49c 100644 --- a/docs/dev/ci/github_actions/adding_tests.md +++ b/docs/dev/ci/github_actions/adding_tests.md @@ -61,7 +61,7 @@ more about workflows and job organization. If new tests do not align with any existing job across all workflows, it is possible to create a dedicated job for them. -An example dedicated job for a single set of tests from [`linux.yml`](./../../../../.github/workflows/linux.yml): +An example dedicated job for a single set of tests from [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml): ```yaml NVIDIA_Plugin: name: NVIDIA plugin diff --git a/docs/dev/ci/github_actions/caches.md b/docs/dev/ci/github_actions/caches.md index 2ecf6993bdf37e..686e9083b2234b 100644 --- a/docs/dev/ci/github_actions/caches.md +++ b/docs/dev/ci/github_actions/caches.md @@ -35,7 +35,7 @@ This cache is used for sharing small dependencies or artifacts between runs. Refer to the [GitHub Actions official documentation](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows) for a complete reference. -The `CPU functional tests` job in the [`linux.yml`](./../../../../.github/workflows/linux.yml) +The `CPU functional tests` job in the [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml) workflow uses this cache for sharing test execution time to speed up the subsequent runs. First, the artifacts are saved with `actions/cache/save` with a particular key `${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp-${{ github.sha }}`: @@ -107,7 +107,7 @@ path in the Docker container where the resources will be available. * Used in jobs that involve Python usage * onnx models for tests * Accessible at the path: `/mount/onnxtestdata` - * Used in the `ONNX Models tests` job in the [`linux.yml`](./../../../../.github/workflows/linux.yml) workflow + * Used in the `ONNX Models tests` job in the [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml) workflow * Linux RISC-V with Conan build artifacts * Used in the [`linux_riscv.yml`](./../../../../.github/workflows/linux_riscv.yml) workflow diff --git a/docs/dev/ci/github_actions/docker_images.md b/docs/dev/ci/github_actions/docker_images.md index c9950f6f9a0bf5..ee2ddcc74845f8 100644 --- a/docs/dev/ci/github_actions/docker_images.md +++ b/docs/dev/ci/github_actions/docker_images.md @@ -10,7 +10,7 @@ Docker images are specified for each job using the `container` key. See the [GHA documentation](https://docs.github.com/en/actions/using-jobs/running-jobs-in-a-container) for syntax reference. -An example `Build` job from the [`linux.yml`](./../../../../.github/workflows/linux.yml) workflow: +An example `Build` job from the [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml) workflow: ```yaml Build: ... @@ -57,7 +57,7 @@ Examples: The Docker image choice depends on the nature of the job and the configuration being tested. -An example `Build` job from the [`linux.yml`](./../../../../.github/workflows/linux.yml) workflow: +An example `Build` job from the [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml) workflow: ```yaml Build: ... diff --git a/docs/dev/ci/github_actions/overview.md b/docs/dev/ci/github_actions/overview.md index 05dc980ebb5b3d..746ac2926c7284 100644 --- a/docs/dev/ci/github_actions/overview.md +++ b/docs/dev/ci/github_actions/overview.md @@ -32,7 +32,7 @@ They include: You can find all workflows for this repository in the [workflows folder](../../../../.github/workflows). The three main ones, providing most coverage for different operating systems, are: -* [Linux](../../../../.github/workflows/linux.yml) +* [Linux](../../../../.github/workflows/ubuntu_22.yml) * [Windows](../../../../.github/workflows/windows.yml) * [macOS](../../../../.github/workflows/mac.yml) @@ -56,7 +56,7 @@ and see what and how to [obtain additional actions](https://github.com/marketpla Workflows run whenever they are triggered by predefined [events](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows). These triggers **are not** mutually exclusive and multiple can be used by one workflow. The OpenVINO repository has three, and as you may see in the example below, they are all -included in the [Linux workflow](../../../../.github/workflows/linux.yml). They are: +included in the [Linux workflow](../../../../.github/workflows/ubuntu_22.yml). They are: * `on: schedule` - schedule trigger * This trigger runs the workflow on a specified interval (e.g., nightly). @@ -129,7 +129,7 @@ The workflow structures for Linux, Windows, and macOS are mostly the same: **NOTE**: some workflows may use the same structure, while others may lack the last 3 steps, with tests coming right after the `Build` step. -Overview of the [Linux workflow](../../../../.github/workflows/linux.yml). +Overview of the [Linux workflow](../../../../.github/workflows/ubuntu_22.yml). There are several jobs present: ```yaml @@ -167,7 +167,7 @@ Each job has several keys that describe its environment. Consider checking a com This section describes the specifics of the OpenVINO CI environment. -Overview of the [Linux workflow's](../../../../.github/workflows/linux.yml) `Python_Unit_Tests` job: +Overview of the [Linux workflow's](../../../../.github/workflows/ubuntu_22.yml) `Python_Unit_Tests` job: ```yaml Python_Unit_Tests: name: Python unit tests diff --git a/docs/dev/ci/github_actions/reusable_workflows.md b/docs/dev/ci/github_actions/reusable_workflows.md index a36991fc3ebb78..a804c587690ee8 100644 --- a/docs/dev/ci/github_actions/reusable_workflows.md +++ b/docs/dev/ci/github_actions/reusable_workflows.md @@ -17,11 +17,11 @@ This document describes the setup used in the OpenVINO GitHub Actions. You can find all workflows for this repository in the [workflows folder](../../../../.github/workflows). There are two categories of workflows: -* files starting with the OS name, for example: [`linux.yml`](./../../../../.github/workflows/linux.yml), [`windows_conditional_compilation.yml`](./../../../../.github/workflows/windows_conditional_compilation.yml). These are validation workflows that include building and testing of the corresponding OS, +* files starting with the OS name, for example: [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml), [`windows_conditional_compilation.yml`](./../../../../.github/workflows/windows_conditional_compilation.yml). These are validation workflows that include building and testing of the corresponding OS, architecture and set of tools. Read more on the [workflows page](./overview.md#structure-of-the-workflows). * files starting with the word `job`, for example: [`job_cxx_unit_tests.yml`](./../../../../.github/workflows/job_cxx_unit_tests.yml), [`job_samples_tests.yml`](./../../../../.github/workflows/job_samples_tests.yml). These workflows are _reusable workflows_ used as jobs in several other workflows. -For example, the [`job_python_unit_tests.yml`](./../../../../.github/workflows/job_python_unit_tests.yml) reusable workflow is used in the [`linux.yml`](./../../../../.github/workflows/linux.yml), [`linux_arm64.yml`](./../../../../.github/workflows/linux_arm64.yml), +For example, the [`job_python_unit_tests.yml`](./../../../../.github/workflows/job_python_unit_tests.yml) reusable workflow is used in the [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml), [`linux_arm64.yml`](./../../../../.github/workflows/linux_arm64.yml), [`mac.yml`](./../../../../.github/workflows/mac.yml) and [`mac_arm64.yml`](./../../../../.github/workflows/mac_arm64.yml) workflows as a `Python_Unit_Tests` job: ```yaml Python_Unit_Tests: @@ -38,7 +38,7 @@ For example, the [`job_python_unit_tests.yml`](./../../../../.github/workflows/j A reusable workflow should be referenced as a `job`. The [`job_python_unit_tests.yml`](./../../../../.github/workflows/job_python_unit_tests.yml) -reusable workflow example in the [`linux.yml`](./../../../../.github/workflows/linux.yml) workflow: +reusable workflow example in the [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml) workflow: ```yaml Python_Unit_Tests: name: Python unit tests diff --git a/docs/dev/ci/github_actions/runners.md b/docs/dev/ci/github_actions/runners.md index 30c85ab47241a8..85710063549fed 100644 --- a/docs/dev/ci/github_actions/runners.md +++ b/docs/dev/ci/github_actions/runners.md @@ -12,7 +12,7 @@ longer workflows (such as builds or functional tests) should use the self-hosted The runners are specified for each job using the `runs-on` key. -An example `Build` job from the [`linux.yml`](./../../../../.github/workflows/linux.yml) +An example `Build` job from the [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml) workflow, using the `aks-linux-16-cores-32gb` runner group: ```yaml @@ -121,7 +121,7 @@ available images and how to choose one in the [OpenVINO Docker Image Overview](. The configuration of a runner required for a job (building, testing, other tasks) depends on the nature of the job. Jobs that are more memory and/or CPU-intensive require a more robust configuration. -The `Build` job in the [`linux.yml`](./../../../../.github/workflows/linux.yml) workflow uses +The `Build` job in the [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml) workflow uses the `aks-linux-16-cores-32gb` group as specified in the `runs-on` key: ```yaml Build: @@ -133,7 +133,7 @@ Build: The `aks-linux-16-cores-32gb` group has machines with 16-core CPU and 32 GB of RAM. These resources are suitable for using in parallel by the build tools in the `Build` job. -The `C++ unit tests` job in the [`linux.yml`](./../../../../.github/workflows/linux.yml) workflow uses the `aks-linux-4-cores-16gb` group: +The `C++ unit tests` job in the [`ubuntu_22.yml`](./../../../../.github/workflows/ubuntu_22.yml) workflow uses the `aks-linux-4-cores-16gb` group: ```yaml CXX_Unit_Tests: name: C++ unit tests diff --git a/docs/dev/ci/github_actions/smart_ci.md b/docs/dev/ci/github_actions/smart_ci.md index b98917661546e2..d9f17595f682af 100644 --- a/docs/dev/ci/github_actions/smart_ci.md +++ b/docs/dev/ci/github_actions/smart_ci.md @@ -192,7 +192,7 @@ corresponding components. ### Adding Validation for a Component If you want to add a new validation job to test your new component or choose an existing one, go to the desired workflow in [.github/workflows](../../../../.github/workflows). The main ones include -[linux.yml](../../../../.github/workflows/linux.yml), [windows.yml](../../../../.github/workflows/windows.yml) and +[ubuntu_22.yml](../../../../.github/workflows/ubuntu_22.yml), [windows.yml](../../../../.github/workflows/windows.yml) and [mac.yml](../../../../.github/workflows/mac.yml). If Smart CI is enabled for the pipeline, you will find the Smart_CI job in the beginning of the workflow: ```yaml diff --git a/src/tests/functional/plugin/conformance/test_runner/README.md b/src/tests/functional/plugin/conformance/test_runner/README.md index 6579d893673bfd..f0954c4e3b78be 100644 --- a/src/tests/functional/plugin/conformance/test_runner/README.md +++ b/src/tests/functional/plugin/conformance/test_runner/README.md @@ -195,7 +195,7 @@ The `run_conformace.py` script has an optional `--expected_failures` argument wh You can find the files with the most up-to-date expected failures for different devices and conformance types [here](./../../../../../tests/test_utils/functional_test_utils/layer_tests_summary/skip_configs). -These files are used in [the Linux GitHub workflow](./../../../../../../.github/workflows/linux.yml) for test skip. +These files are used in [the Linux GitHub workflow](./../../../../../../.github/workflows/ubuntu_22.yml) for test skip. You can update the file(s) you need with either new passing tests, i.e., when something is fixed, or with new failing tests to skip them. The changes will be reflected in the GitHub actions pipeline, in the `Conformance_Tests` job. From ea8d2284625f1fb2b892e4972c8dd96288aede88 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 28 Jun 2024 10:49:06 +0100 Subject: [PATCH 31/60] rm unused Python from nvidia image --- .../ov_build/ubuntu_22_04_x64_nvidia/Dockerfile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile index d4a4c2d25bbbc9..eeae5adef4c276 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile @@ -23,10 +23,7 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ - # Pythons - python3.8-dev \ - python3.8-venv \ - python3.8-distutils \ + # Python python3.11-dev \ python3.11-venv \ python3.11-distutils \ @@ -65,12 +62,11 @@ RUN apt-get update && apt-get install -y cuda-runtime-11-8 cuda-11-8 && apt-get # Setup pip ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py -# Use Python 3.11 as default instead of Python 3.8 -# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +# Use Python 3.11 as default +# Using venv here 'cause other methods to switch the default Python on Ubuntu 22 break both system and wheels build RUN python3.11 -m venv venv ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" From 307616278590f5914ace9b93b2b2cb7e4d16cc13 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 28 Jun 2024 16:11:53 +0100 Subject: [PATCH 32/60] change name --- .github/workflows/ubuntu_22.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 550a449c06c2a9..6a2bd8b9fc75da 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -474,7 +474,7 @@ jobs: if: ${{ 'false' }} # Ticket: 143677 Overall_Status: - name: ci/gha_overall_status_ubuntu_22 + name: ci/gha_overall_status needs: [Smart_CI, Build, Debian_Packages, Samples, Conformance, ONNX_Runtime, CXX_Unit_Tests, Python_Unit_Tests, TensorFlow_Layer_Tests, CPU_Functional_Tests, TensorFlow_Models_Tests_Precommit, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers] if: ${{ always() }} From 8d2f2cd0eb52a3d0612174928717ef45d7b81dce Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 28 Jun 2024 17:17:45 +0100 Subject: [PATCH 33/60] add missing dependency --- .github/workflows/ubuntu_22.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 6a2bd8b9fc75da..9d20cb4a61b772 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -90,7 +90,7 @@ jobs: Debian_Packages: name: Debian Packages - needs: Build + needs: [Docker, Build] uses: ./.github/workflows/job_debian_packages.yml with: runner: 'aks-linux-4-cores-16gb' From 5f35fbb5cea238cf7a1a33de6d3b8df65d56d5e9 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 28 Jun 2024 17:47:26 +0100 Subject: [PATCH 34/60] use clean image --- .github/workflows/ubuntu_22.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 9d20cb4a61b772..711c3cb496f1b2 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -94,7 +94,7 @@ jobs: uses: ./.github/workflows/job_debian_packages.yml with: runner: 'aks-linux-4-cores-16gb' - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}"}' + container: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04' Samples: needs: [ Docker, Build, Smart_CI ] From 170402562d76a188bc5d5d286a68040dffe50b7e Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 1 Jul 2024 08:59:13 +0100 Subject: [PATCH 35/60] return bare image --- .github/workflows/job_debian_packages.yml | 9 +++++---- .github/workflows/linux_arm64.yml | 2 +- .github/workflows/ubuntu_22.yml | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index 5bffce9739ead6..fc407297117680 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -7,11 +7,11 @@ on: description: 'Machine on which the tests would run' type: string required: true - container: - description: 'JSON to be converted to the value of the "container" configuration for the job' + image: + description: 'Docker image in which the tests would run' type: string required: false - default: '{"image": null}' + default: null permissions: read-all @@ -19,7 +19,8 @@ jobs: Debian_Packages: name: Debian Packages runs-on: ${{ inputs.runner }} - container: ${{ fromJSON(inputs.container) }} + container: + image: ${{ inputs.image }} defaults: run: shell: bash diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index 84c71328d429b5..9156fa711e752a 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -301,7 +301,7 @@ jobs: uses: ./.github/workflows/job_debian_packages.yml with: runner: 'aks-linux-16-cores-arm' - container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_arm64 }}"}' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' Samples: needs: [ Build, Docker, Smart_CI ] diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 711c3cb496f1b2..b44bc57c16228d 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -94,7 +94,7 @@ jobs: uses: ./.github/workflows/job_debian_packages.yml with: runner: 'aks-linux-4-cores-16gb' - container: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04' Samples: needs: [ Docker, Build, Smart_CI ] From 2cbae3a21a58df8f1492f47ad5509b4db6fe0fba Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 1 Jul 2024 10:48:47 +0100 Subject: [PATCH 36/60] use system Python for Debian packages --- .../dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile | 3 +++ .github/workflows/job_build_linux.yml | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index dfc40fbce2594b..0ad2a7d75cf88d 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -64,4 +64,7 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ RUN python3.11 -m venv venv ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" +# Create venv for Python 3.10 (System) for Debian packages +RUN python3 -m venv venv3.10 + ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index a8460cb54417dd..14ae21e7b017a3 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -175,13 +175,20 @@ jobs: - name: Build Debian packages run: | - python3 -m pip install -U pip - python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt + + # In Ubuntu 22, we use Python 3.11 for tests and build, i.e., not system 3.10 + # while we use system Python for other OSes + # Debian packages should be built with the system Python + [[ ${{ inputs.os }} == "ubuntu22" ]] && python_exec_path=/venv3.10/bin/python3 || ubuntu_version=/usr/bin/python3 + + $python_exec_path -m pip install -U pip + $python_exec_path -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt cmake -UPython* -U_Python* -UPYTHON* \ -UTBB* \ -DENABLE_SYSTEM_TBB=ON \ -DENABLE_PYTHON_PACKAGING=ON \ -DENABLE_TESTS=OFF \ + -DPython3_EXECUTABLE=$python_exec_path \ -DCPACK_GENERATOR=DEB \ ${BUILD_DIR} cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package From 8cd195a99cc243220b32aeda72065081a7270705 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 1 Jul 2024 11:05:24 +0100 Subject: [PATCH 37/60] fix var name --- .github/workflows/job_build_linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index 14ae21e7b017a3..bc430ffdbe2399 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -179,7 +179,7 @@ jobs: # In Ubuntu 22, we use Python 3.11 for tests and build, i.e., not system 3.10 # while we use system Python for other OSes # Debian packages should be built with the system Python - [[ ${{ inputs.os }} == "ubuntu22" ]] && python_exec_path=/venv3.10/bin/python3 || ubuntu_version=/usr/bin/python3 + [[ ${{ inputs.os }} == "ubuntu22" ]] && python_exec_path=/venv3.10/bin/python3 || python_exec_path=/usr/bin/python3 $python_exec_path -m pip install -U pip $python_exec_path -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt From ec0f8ee5b230986f4f3b78efe214a1109f0d343a Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 1 Jul 2024 11:58:50 +0100 Subject: [PATCH 38/60] change order --- .github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index 0ad2a7d75cf88d..61208a63ebfe98 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -59,12 +59,12 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py +# Create venv for Python 3.10 (System) for Debian packages +RUN python3 -m venv venv3.10 + # Use Python 3.11 as default # Using venv here 'cause other methods to switch the default Python on Ubuntu break both system and wheels build RUN python3.11 -m venv venv ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" -# Create venv for Python 3.10 (System) for Debian packages -RUN python3 -m venv venv3.10 - ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} From e91e8517072ff657f4aa29db6d0996208fd35319 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 1 Jul 2024 12:00:40 +0100 Subject: [PATCH 39/60] use path to sys python --- .../ov_build/ubuntu_22_04_x64/Dockerfile | 3 --- .github/workflows/job_build_linux.yml | 13 ++++--------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index 61208a63ebfe98..dfc40fbce2594b 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -59,9 +59,6 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py -# Create venv for Python 3.10 (System) for Debian packages -RUN python3 -m venv venv3.10 - # Use Python 3.11 as default # Using venv here 'cause other methods to switch the default Python on Ubuntu break both system and wheels build RUN python3.11 -m venv venv diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index bc430ffdbe2399..471ac4ef3e2329 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -175,20 +175,15 @@ jobs: - name: Build Debian packages run: | - - # In Ubuntu 22, we use Python 3.11 for tests and build, i.e., not system 3.10 - # while we use system Python for other OSes - # Debian packages should be built with the system Python - [[ ${{ inputs.os }} == "ubuntu22" ]] && python_exec_path=/venv3.10/bin/python3 || python_exec_path=/usr/bin/python3 - - $python_exec_path -m pip install -U pip - $python_exec_path -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt + # Use system Python + /usr/bin/python3 -m pip install -U pip + /usr/bin/python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt cmake -UPython* -U_Python* -UPYTHON* \ -UTBB* \ -DENABLE_SYSTEM_TBB=ON \ -DENABLE_PYTHON_PACKAGING=ON \ -DENABLE_TESTS=OFF \ - -DPython3_EXECUTABLE=$python_exec_path \ + -DPython3_EXECUTABLE=/usr/bin/python3 \ -DCPACK_GENERATOR=DEB \ ${BUILD_DIR} cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package From 43db64de49273bfdaf28dbb1409fcbd738d7f5c9 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 1 Jul 2024 12:46:55 +0100 Subject: [PATCH 40/60] adapt for U24 --- .github/workflows/job_build_linux.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index 471ac4ef3e2329..b910bb739ec2cd 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -175,15 +175,17 @@ jobs: - name: Build Debian packages run: | - # Use system Python - /usr/bin/python3 -m pip install -U pip - /usr/bin/python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt + # Ubuntu 24 does not allow using the system Python directly so + # we have to use Python from the virtual environment created in Docker + [[ ${{ inputs.os }} == "ubuntu24" ]] && python_exec=/venv/bin/python3 || python_exec=/usr/bin/python3 + $python_exec -m pip install -U pip + $python_exec -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt cmake -UPython* -U_Python* -UPYTHON* \ -UTBB* \ -DENABLE_SYSTEM_TBB=ON \ -DENABLE_PYTHON_PACKAGING=ON \ -DENABLE_TESTS=OFF \ - -DPython3_EXECUTABLE=/usr/bin/python3 \ + -DPython3_EXECUTABLE=$python_exec \ -DCPACK_GENERATOR=DEB \ ${BUILD_DIR} cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package From c1d8bbcf89345a97ded7f3128c94446f7f38bc00 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 2 Jul 2024 13:05:25 +0100 Subject: [PATCH 41/60] rm iGPU from required --- .github/workflows/ubuntu_20.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml index 0bbc0e32d6b864..ba95cf4c7efb93 100644 --- a/.github/workflows/ubuntu_20.yml +++ b/.github/workflows/ubuntu_20.yml @@ -121,7 +121,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_ubuntu_20 - needs: [Smart_CI, Build, iGPU] + needs: [Smart_CI, Build] if: ${{ always() }} runs-on: ubuntu-latest steps: From d39f36cca2a47c5c694c8fdddd4809be657bbfc3 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 3 Jul 2024 10:01:27 +0100 Subject: [PATCH 42/60] return as required --- .github/workflows/ubuntu_20.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml index ba95cf4c7efb93..0bbc0e32d6b864 100644 --- a/.github/workflows/ubuntu_20.yml +++ b/.github/workflows/ubuntu_20.yml @@ -121,7 +121,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_ubuntu_20 - needs: [Smart_CI, Build] + needs: [Smart_CI, Build, iGPU] if: ${{ always() }} runs-on: ubuntu-latest steps: From f6d9bacce33bd94b59be8e9a50051f6ed08c9d4e Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 3 Jul 2024 10:01:49 +0100 Subject: [PATCH 43/60] rm --- .github/workflows/ubuntu_22.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 4ca152fc7f8669..b44bc57c16228d 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -476,7 +476,7 @@ jobs: Overall_Status: name: ci/gha_overall_status needs: [Smart_CI, Build, Debian_Packages, Samples, Conformance, ONNX_Runtime, CXX_Unit_Tests, Python_Unit_Tests, TensorFlow_Layer_Tests, - CPU_Functional_Tests, TensorFlow_Models_Tests_Precommit, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers, iGPU] + CPU_Functional_Tests, TensorFlow_Models_Tests_Precommit, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers] if: ${{ always() }} runs-on: ubuntu-latest steps: From 0c30c568566c7688edd06ed7f0ba50588b5ce51f Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 11 Jul 2024 09:21:04 +0100 Subject: [PATCH 44/60] review --- .github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index f4260560a409ec..b530946bf7dfd6 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -22,7 +22,7 @@ RUN apt-get update && \ ca-certificates \ gpg-agent \ tzdata \ - # Python + # Python (3.8) python3-dev \ python3-venv \ python3-pip \ From 7d6d962879a2c36f3ea628067db814f926e93687 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 26 Jul 2024 13:27:57 +0100 Subject: [PATCH 45/60] rm unused --- .github/workflows/ubuntu_22.yml | 261 -------------------------------- 1 file changed, 261 deletions(-) diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 8e07dd77d03c8d..5fbac8731d03fa 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -79,7 +79,6 @@ jobs: Build: needs: [Docker, Smart_CI] -<<<<<<< HEAD:.github/workflows/ubuntu_22.yml if: "!needs.smart_ci.outputs.skip_workflow" uses: ./.github/workflows/job_build_linux.yml with: @@ -88,266 +87,6 @@ jobs: affected-components: ${{ needs.smart_ci.outputs.affected_components }} event-name: ${{ github.event_name }} os: 'ubuntu22' -======= - timeout-minutes: 150 - defaults: - run: - shell: bash - runs-on: aks-linux-16-cores-32gb - container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} - volumes: - - /mount:/mount - options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING - env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - CMAKE_BUILD_TYPE: 'Release' - CMAKE_GENERATOR: 'Ninja Multi-Config' - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: 1 - SCCACHE_SERVER_PORT: 35555 - SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt - SCCACHE_LOG: warn - GITHUB_WORKSPACE: '/__w/openvino/openvino' - OPENVINO_REPO: /__w/openvino/openvino/openvino - OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib - INSTALL_DIR: /__w/openvino/openvino/openvino_install - INSTALL_DIR_JS: /__w/openvino/openvino/openvino_install/js - INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install - DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install - BUILD_DIR: /__w/openvino/openvino/openvino_build - SCCACHE_AZURE_KEY_PREFIX: ubuntu20_x86_64_Release - ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime - ARTIFACTS_SHARE: "/mount/build-artifacts" - MANIFEST_PATH: '/__w/openvino/openvino/manifest.yml' - PRODUCT_TYPE: 'public_linux_ubuntu_20_04_release' - if: "!needs.smart_ci.outputs.skip_workflow" - - steps: - - name: Clone OpenVINO - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - path: ${{ env.OPENVINO_REPO }} - submodules: 'true' - - # Ticket: 139627 - - name: Checkout the latest OneDNN for GPU in nightly - if: ${{ github.event_name == 'schedule' }} - working-directory: ${{ env.OPENVINO_REPO }}/src/plugins/intel_gpu/thirdparty/onednn_gpu - run: | - git fetch origin - git checkout main - git rev-parse HEAD - - - name: Clone OpenVINO Contrib - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - repository: 'openvinotoolkit/openvino_contrib' - path: ${{ env.OPENVINO_CONTRIB_REPO }} - submodules: 'true' - ref: 'master' - - - name: Generate product manifest and set CI_BUILD_NUMBER & CI_BUILD_DEV_TAG - id: create_manifest - uses: ./openvino/.github/actions/create_manifest - with: - repos: | - ${{ env.OPENVINO_REPO }} - ${{ env.OPENVINO_CONTRIB_REPO }} - product_type: ${{ env.PRODUCT_TYPE }} - target_arch: 'intel64' - build_type: 'release' - save_to: ${{ env.MANIFEST_PATH }} - - # - # Print system info - # - - - name: System info - uses: ./openvino/.github/actions/system_info - - # - # Dependencies - # - - - name: Install python dependencies - run: | - # For Python API: build and wheel packaging - python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - - # For running ONNX frontend unit tests - python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt - - # For running TensorFlow frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow/tests/requirements.txt - - # For running TensorFlow Lite frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow_lite/tests/requirements.txt - - # For running Paddle frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/paddle/tests/requirements.txt - - # - # Build - # - - - name: CMake configure - OpenVINO - run: | - cmake \ - -G "${{ env.CMAKE_GENERATOR }}" \ - -DENABLE_CPPLINT=OFF \ - -DENABLE_NCC_STYLE=OFF \ - -DENABLE_TESTS=ON \ - -DENABLE_STRICT_DEPENDENCIES=OFF \ - -DENABLE_SYSTEM_OPENCL=ON \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCPACK_GENERATOR=TGZ \ - -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ - -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \ - -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \ - -S ${OPENVINO_REPO} \ - -B ${BUILD_DIR} - - - name: Clean sccache stats - run: ${SCCACHE_PATH} --zero-stats - - - name: Cmake build - OpenVINO - run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: Show sccache stats - run: ${SCCACHE_PATH} --show-stats - - - name: Cmake install - OpenVINO - run: | - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${DEVELOPER_PACKAGE_DIR} -DCOMPONENT=developer_package -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake - - - name: Pack Artifacts - run: | - - # Add the ONNX Runtime version and skip tests list to the archive to use in the ONNX Runtime Job - # w/o the need to checkout repository - - cp -R ${ONNX_RUNTIME_UTILS} ${INSTALL_DIR} - - pushd ${INSTALL_DIR} - tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * - popd - - pushd ${DEVELOPER_PACKAGE_DIR} - tar -czvf ${BUILD_DIR}/openvino_developer_package.tar.gz * - popd - - pushd ${INSTALL_TEST_DIR} - tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz * - popd - - - name: Build Debian packages - run: | - /usr/bin/python3.8 -m pip install -U pip - /usr/bin/python3.8 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - cmake -UPython* -U_Python* -UPYTHON* \ - -UTBB* \ - -DENABLE_SYSTEM_TBB=ON \ - -DENABLE_PYTHON_PACKAGING=ON \ - -DENABLE_TESTS=OFF \ - -DPython3_EXECUTABLE=/usr/bin/python3.8 \ - -DCPACK_GENERATOR=DEB \ - ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package - - - name: Cmake & Build - OpenVINO Contrib - run: | - cmake \ - -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \ - -DOPENVINO_EXTRA_MODULES="${OPENVINO_CONTRIB_REPO}/modules/java_api;${OPENVINO_CONTRIB_REPO}/modules/custom_operations" \ - -S ${OPENVINO_REPO} \ - -B ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: CMake configure, build and install - OpenVINO JS API - if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API - run: | - cmake -UTBB* -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -S ${OPENVINO_REPO} -B ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR_JS} -P ${BUILD_DIR}/cmake_install.cmake - - # - # Upload build artifacts and logs - # - - name: Upload build logs - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - if: always() - with: - name: build_logs - path: ${{ env.SCCACHE_ERROR_LOG }} - if-no-files-found: 'ignore' - - - name: Upload openvino package - if: ${{ always() }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_package - path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz - if-no-files-found: 'error' - - - name: Upload openvino js package - if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_js_package - path: ${{ env.INSTALL_DIR_JS }} - if-no-files-found: 'error' - - - name: Upload openvino developer package - if: ${{ always() }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_developer_package - path: ${{ env.BUILD_DIR }}/openvino_developer_package.tar.gz - if-no-files-found: 'error' - - - name: Upload openvino debian packages - if: ${{ always() }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_debian_packages - path: ${{ env.BUILD_DIR }}/*.deb - if-no-files-found: 'error' - - - name: Upload openvino tests package - if: ${{ always() }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - with: - name: openvino_tests - path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz - if-no-files-found: 'error' ->>>>>>> upstream/master:.github/workflows/linux.yml - - - name: Prepare debian packages for storage on share - if: ${{ always() }} - continue-on-error: true - run: | - pushd ${{ env.BUILD_DIR }} - mkdir deb && mv *.deb deb/ - popd - - - name: Store artifacts to a shared drive - id: store_artifacts - if: ${{ always() }} - uses: ./openvino/.github/actions/store_artifacts - with: - artifacts: | - ${{ env.BUILD_DIR }}/openvino_package.tar.gz - ${{ env.BUILD_DIR }}/openvino_developer_package.tar.gz - ${{ env.BUILD_DIR }}/openvino_tests.tar.gz - ${{ env.BUILD_DIR }}/deb - ${{ env.MANIFEST_PATH }} - storage_dir: ${{ env.PRODUCT_TYPE }} - storage_root: ${{ env.ARTIFACTS_SHARE }} Debian_Packages: name: Debian Packages From 09596612ed78dce994c2588873f8c43b7fc08ed0 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 26 Jul 2024 13:31:09 +0100 Subject: [PATCH 46/60] rm unsued env var --- .github/workflows/job_build_linux.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index e5623fb7238f7e..330da8ee7f235a 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -27,7 +27,6 @@ permissions: read-all env: PIP_CACHE_PATH: /mount/caches/pip/linux - PYTHON_VERSION: '3.11' jobs: Build: From f754441154753ea58241e6cc415461d4bf4f09f5 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 26 Jul 2024 13:51:55 +0100 Subject: [PATCH 47/60] rm conditions, store artefacts for all Ubuntus --- .github/workflows/job_build_linux.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index 330da8ee7f235a..4e8136d6e9691f 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -85,7 +85,6 @@ jobs: ref: 'master' - name: Generate product manifest and set CI_BUILD_NUMBER & CI_BUILD_DEV_TAG - if: ${{ inputs.os == 'ubuntu22' }} # TODO: enable for other OSes id: create_manifest uses: ./openvino/.github/actions/create_manifest with: @@ -273,7 +272,7 @@ jobs: if-no-files-found: 'error' - name: Prepare debian packages for storage on share - if: ${{ inputs.os == 'ubuntu22' }} # TODO: enable for other OSes + if: ${{ always() }} continue-on-error: true run: | pushd ${{ env.BUILD_DIR }} @@ -282,7 +281,7 @@ jobs: - name: Store artifacts to a shared drive id: store_artifacts - if: ${{ inputs.os == 'ubuntu22' }} # TODO: enable for other OSes + if: ${{ always() }} uses: ./openvino/.github/actions/store_artifacts with: artifacts: | From fb3ad846c5d6054faccb6009648f342ec8203618 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 29 Jul 2024 10:53:22 +0100 Subject: [PATCH 48/60] use underscores for OS name --- .github/workflows/job_build_linux.yml | 8 ++++---- .github/workflows/ubuntu_20.yml | 2 +- .github/workflows/ubuntu_22.yml | 2 +- .github/workflows/ubuntu_24.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index 4e8136d6e9691f..93ab7f87f65c35 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -19,7 +19,7 @@ on: type: string required: false os: - description: 'OS that is used for building in the form of "ubuntu20"' + description: 'OS that is used for building in the form of "ubuntu_20_04"' type: string required: true @@ -59,7 +59,7 @@ jobs: ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime ARTIFACTS_SHARE: "/mount/build-artifacts" MANIFEST_PATH: '/__w/openvino/openvino/manifest.yml' - PRODUCT_TYPE: public_linux_${{ inputs.os }}_04_release + PRODUCT_TYPE: public_linux_${{ inputs.os }}_release steps: - name: Clone OpenVINO uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -113,7 +113,7 @@ jobs: python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt # For running ONNX frontend unit tests - if [[ ${{ inputs.os }} == 'ubuntu24' ]]; then + if [[ ${{ inputs.os }} == 'ubuntu_24_04' ]]; then # Should be removed after https://github.com/openvinotoolkit/openvino/pull/24242 is merged export CMAKE_GENERATOR="Unix Makefiles" python3 -m pip install --force-reinstall --no-cache-dir -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt @@ -191,7 +191,7 @@ jobs: run: | # Ubuntu 24 does not allow using the system Python directly so # we have to use Python from the virtual environment created in Docker - [[ ${{ inputs.os }} == "ubuntu24" ]] && python_exec=/venv/bin/python3 || python_exec=/usr/bin/python3 + [[ ${{ inputs.os }} == "ubuntu_24_04" ]] && python_exec=/venv/bin/python3 || python_exec=/usr/bin/python3 $python_exec -m pip install -U pip $python_exec -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt cmake -UPython* -U_Python* -UPYTHON* \ diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml index 0bbc0e32d6b864..7eafc8a03fb56b 100644 --- a/.github/workflows/ubuntu_20.yml +++ b/.github/workflows/ubuntu_20.yml @@ -83,7 +83,7 @@ jobs: container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} event-name: ${{ github.event_name }} - os: 'ubuntu20' + os: 'ubuntu_20_04' iGPU: name: iGPU Tests diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 5fbac8731d03fa..71bc6e2c705223 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -86,7 +86,7 @@ jobs: container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} event-name: ${{ github.event_name }} - os: 'ubuntu22' + os: 'ubuntu_22_04' Debian_Packages: name: Debian Packages diff --git a/.github/workflows/ubuntu_24.yml b/.github/workflows/ubuntu_24.yml index a6746166f6a675..f09e6656e7c332 100644 --- a/.github/workflows/ubuntu_24.yml +++ b/.github/workflows/ubuntu_24.yml @@ -81,7 +81,7 @@ jobs: container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_24_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} event-name: ${{ github.event_name }} - os: 'ubuntu24' + os: 'ubuntu_24_04' Overall_Status: name: ci/gha_overall_status_ubuntu_24 From b55868eb28f7ed0bb4b4ab23fcba4b0fa4469cf5 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 29 Jul 2024 11:09:11 +0100 Subject: [PATCH 49/60] use python3 explicetly --- .github/actions/create_manifest/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/create_manifest/action.yml b/.github/actions/create_manifest/action.yml index 66d59930e93712..1acc1ebff98fae 100644 --- a/.github/actions/create_manifest/action.yml +++ b/.github/actions/create_manifest/action.yml @@ -35,7 +35,7 @@ runs: id: create_manifest shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} run: >- - python ${{ env.ACTION_PATH }}/create_manifest.py + python3 ${{ env.ACTION_PATH }}/create_manifest.py --target_arch "${{ inputs.target_arch }}" --build_type "${{ inputs.build_type }}" --save_to "${{ inputs.save_to }}" --product_type "${{ inputs.product_type }}" -r "${{ inputs.repos }}" env: From 6c8be1c5e66d870e0d315dd9ab236e2a5def6c2b Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 29 Jul 2024 15:44:19 +0100 Subject: [PATCH 50/60] use py3 explicitly --- .github/actions/store_artifacts/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/store_artifacts/action.yml b/.github/actions/store_artifacts/action.yml index d9c4184a622cce..a60938f20e745f 100644 --- a/.github/actions/store_artifacts/action.yml +++ b/.github/actions/store_artifacts/action.yml @@ -31,7 +31,7 @@ runs: id: copy_artifacts shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} run: >- - python ${{ env.ACTION_PATH }}/store_artifacts.py + python3 ${{ env.ACTION_PATH }}/store_artifacts.py --storage_dir "${{ inputs.storage_dir }}" --storage_root "${{ inputs.storage_root }}" -a "${{ inputs.artifacts }}" env: From 9e76ab8a634e94257237b92cd325f6710c95115d Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 10:30:27 +0100 Subject: [PATCH 51/60] add Debian and Samples jobs to U20/24 workflows; use better Ubuntu versions parsing --- .../ov_build/ubuntu_20_04_x64/Dockerfile | 5 --- .../ov_build/ubuntu_22_04_x64/Dockerfile | 4 -- .../ubuntu_22_04_x64_nvidia/Dockerfile | 1 - .../ov_build/ubuntu_24_04_x64/Dockerfile | 11 ----- .../ov_test/ubuntu_24_04_x64/Dockerfile | 44 +++++++++++++++++++ .github/workflows/job_debian_packages.yml | 5 ++- .github/workflows/ubuntu_20.yml | 18 ++++++++ .github/workflows/ubuntu_24.yml | 20 ++++++++- 8 files changed, 85 insertions(+), 23 deletions(-) create mode 100644 .github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index b530946bf7dfd6..4c76d0c869d443 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -19,13 +19,8 @@ RUN apt-get update && \ apt-get install \ curl \ git \ - ca-certificates \ gpg-agent \ tzdata \ - # Python (3.8) - python3-dev \ - python3-venv \ - python3-pip \ # For Java API default-jdk \ # Compiler \ diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile index dfc40fbce2594b..808ff49b243134 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile @@ -19,7 +19,6 @@ RUN apt-get update && \ apt-get install \ curl \ git \ - ca-certificates \ gpg-agent \ tzdata \ libtbb2 \ @@ -27,9 +26,6 @@ RUN apt-get update && \ python3.11-dev \ python3.11-venv \ python3.11-distutils \ - python3-dev \ - python3-venv \ - python3-pip \ # For Java API default-jdk \ && \ diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile index eeae5adef4c276..508b3815c8e317 100644 --- a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile @@ -20,7 +20,6 @@ RUN apt-get update && \ curl \ wget \ git \ - ca-certificates \ gpg-agent \ tzdata \ # Python diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 5963d91d650ed6..70110826d48043 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -19,19 +19,12 @@ RUN apt-get update && \ apt-get install \ curl \ git \ - ca-certificates \ gpg-agent \ tzdata \ # Python - python3-dev \ - python3-venv \ - python3-pip \ python3-full \ # For Java API default-jdk \ - # Compiler \ - gcc-10 \ - g++-10 \ && \ rm -rf /var/lib/apt/lists/* @@ -41,10 +34,6 @@ RUN chmod +x /install_build_dependencies.sh && \ /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* -# Set gcc-10 as a default compiler -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 - # Install sscache ARG SCCACHE_VERSION="v0.7.5" ENV SCCACHE_HOME="/opt/sccache" \ diff --git a/.github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile new file mode 100644 index 00000000000000..adbeee914483ed --- /dev/null +++ b/.github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile @@ -0,0 +1,44 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:24.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + apt-get update && \ + apt-get install \ + curl \ + git \ + gpg-agent \ + tzdata \ + # Python + python3-full \ + libhdf5-dev \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install build dependencies +ADD install_build_dependencies.sh /install_build_dependencies.sh +RUN chmod +x /install_build_dependencies.sh && \ + /install_build_dependencies.sh && \ + rm -rf /var/lib/apt/lists/* + +# Create a virtual environment for the system Python as Python in Ubuntu 24 complains about installing +# packages into the system Python +RUN python3 -m venv venv +ENV PATH="/venv/bin:$PATH" + +# Setup pip +ENV PIP_VERSION="24.0" +RUN /venv/bin/python3 -m pip install --upgrade pip==${PIP_VERSION} + +ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} +ENV PIP_INSTALL_PATH=/venv/lib/python3/site-packages diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index 331f1262c5ba31..61968ad5834433 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -52,7 +52,10 @@ jobs: wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - [[ $(lsb_release -r -s) == *"22"* ]] && ubuntu_version=22 || ubuntu_version=20 + # Yields only the number, e.g., "20" + ubuntu_version=$(lsb_release -r -s) + ubuntu_version=${ubuntu_version%.*} + echo "deb https://apt.repos.intel.com/openvino/2024 ubuntu$ubuntu_version main" | tee /etc/apt/sources.list.d/intel-openvino-2024.list apt-get update -y diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml index 7eafc8a03fb56b..5e87a02cec7665 100644 --- a/.github/workflows/ubuntu_20.yml +++ b/.github/workflows/ubuntu_20.yml @@ -70,6 +70,7 @@ jobs: with: images: | ov_build/ubuntu_20_04_x64 + ov_test/ubuntu_20_04_x64 registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} @@ -85,6 +86,23 @@ jobs: event-name: ${{ github.event_name }} os: 'ubuntu_20_04' + Debian_Packages: + name: Debian Packages + needs: [ Docker, Build ] + uses: ./.github/workflows/job_debian_packages.yml + with: + runner: 'aks-linux-4-cores-16gb' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + + Samples: + needs: [ Docker, Build, Smart_CI ] + if: fromJSON(needs.smart_ci.outputs.affected_components).samples + uses: ./.github/workflows/job_samples_tests.yml + with: + runner: 'aks-linux-4-cores-16gb' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + iGPU: name: iGPU Tests needs: [ Build, Smart_CI ] diff --git a/.github/workflows/ubuntu_24.yml b/.github/workflows/ubuntu_24.yml index f09e6656e7c332..5480d82a97053f 100644 --- a/.github/workflows/ubuntu_24.yml +++ b/.github/workflows/ubuntu_24.yml @@ -68,6 +68,7 @@ jobs: with: images: | ov_build/ubuntu_24_04_x64 + ov_test/ubuntu_24_04_x64 registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} @@ -83,9 +84,26 @@ jobs: event-name: ${{ github.event_name }} os: 'ubuntu_24_04' + Debian_Packages: + name: Debian Packages + needs: [ Docker, Build ] + uses: ./.github/workflows/job_debian_packages.yml + with: + runner: 'aks-linux-4-cores-16gb' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:24.04' + + Samples: + needs: [ Docker, Build, Smart_CI ] + if: fromJSON(needs.smart_ci.outputs.affected_components).samples + uses: ./.github/workflows/job_samples_tests.yml + with: + runner: 'aks-linux-4-cores-16gb' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_24_04_x64 }}", "volumes": ["/mount:/mount"]}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + Overall_Status: name: ci/gha_overall_status_ubuntu_24 - needs: [Smart_CI, Build] + needs: [Smart_CI, Build, Debian_Packages, Samples] if: ${{ always() }} runs-on: ubuntu-latest steps: From f679da1c4cf0f9ee33a473cf4e0e3925b7f424aa Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 10:31:09 +0100 Subject: [PATCH 52/60] add missing required jobs --- .github/workflows/ubuntu_20.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml index 5e87a02cec7665..7820b897d78dbd 100644 --- a/.github/workflows/ubuntu_20.yml +++ b/.github/workflows/ubuntu_20.yml @@ -139,7 +139,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_ubuntu_20 - needs: [Smart_CI, Build, iGPU] + needs: [Smart_CI, Build, iGPU, Debian_Packages, Samples] if: ${{ always() }} runs-on: ubuntu-latest steps: From 8879558cb041203bb3ace1d6d101aa11936d7db3 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 11:18:19 +0100 Subject: [PATCH 53/60] add missing deps --- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 3 +++ .github/workflows/job_debian_packages.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 70110826d48043..6070748f60aa51 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -22,6 +22,9 @@ RUN apt-get update && \ gpg-agent \ tzdata \ # Python + python3-dev \ + python3-venv \ + python3-pip \ python3-full \ # For Java API default-jdk \ diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index 61968ad5834433..6933179edda230 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -48,7 +48,7 @@ jobs: if [[ "${{ runner.arch }}" == "X64" ]]; then # Install debian packages from previous release - apt-get install --no-install-recommends -y gnupg wget ca-certificates + apt-get install --no-install-recommends -y gnupg wget ca-certificates lsb-release wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB From 1e9286325c473ca2b0428be9999b5a0a6ab952c7 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 12:24:05 +0100 Subject: [PATCH 54/60] add cmake args --- .github/workflows/job_build_linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index 93ab7f87f65c35..56e39ca76d095e 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -116,6 +116,7 @@ jobs: if [[ ${{ inputs.os }} == 'ubuntu_24_04' ]]; then # Should be removed after https://github.com/openvinotoolkit/openvino/pull/24242 is merged export CMAKE_GENERATOR="Unix Makefiles" + export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON" python3 -m pip install --force-reinstall --no-cache-dir -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt else python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt From ddd33362fb9f9f7924bbae53595bb8a3a171b365 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 13:26:43 +0100 Subject: [PATCH 55/60] return gcc-10 --- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index 6070748f60aa51..a8fa0ddfd3245e 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -28,6 +28,9 @@ RUN apt-get update && \ python3-full \ # For Java API default-jdk \ + # Compiler + gcc-10 \ + g++-10 \ && \ rm -rf /var/lib/apt/lists/* @@ -37,7 +40,11 @@ RUN chmod +x /install_build_dependencies.sh && \ /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* -# Install sscache +# Set gcc-10 as a default compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 + +# Install sccache ARG SCCACHE_VERSION="v0.7.5" ENV SCCACHE_HOME="/opt/sccache" \ SCCACHE_PATH="/opt/sccache/sccache" From b747c15da38f465b392bc5760ba05c847e71d73a Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 14:26:38 +0100 Subject: [PATCH 56/60] rm --- .github/workflows/job_build_linux.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index 56e39ca76d095e..93ab7f87f65c35 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -116,7 +116,6 @@ jobs: if [[ ${{ inputs.os }} == 'ubuntu_24_04' ]]; then # Should be removed after https://github.com/openvinotoolkit/openvino/pull/24242 is merged export CMAKE_GENERATOR="Unix Makefiles" - export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON" python3 -m pip install --force-reinstall --no-cache-dir -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt else python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt From e7152c3f4c0126338eca17ae6363e0af709ba884 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 15:17:19 +0100 Subject: [PATCH 57/60] use system py --- .../ov_test/ubuntu_20_04_x64/Dockerfile | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile index 0a151be1e6876d..bd707943563742 100644 --- a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile @@ -19,13 +19,8 @@ RUN apt-get update && \ apt-get install \ curl \ git \ - ca-certificates \ gpg-agent \ tzdata \ - # Python - python3.11-dev \ - python3.11-venv \ - python3.11-distutils \ libhdf5-dev \ && \ rm -rf /var/lib/apt/lists/* @@ -43,10 +38,10 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py -# Use Python 3.11 as default instead of Python 3.8 -# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build -RUN python3.11 -m venv venv -ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" +# Setup pip +ENV PIP_VERSION="24.0" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} -ENV PIP_INSTALL_PATH=/venv/lib/python3.11/site-packages From 2c84f4d28d89277ad0cee6db86187a6ff0183e88 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 15:31:30 +0100 Subject: [PATCH 58/60] rm unused --- .github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile index bd707943563742..67cec0b1ce806d 100644 --- a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile @@ -31,13 +31,6 @@ RUN chmod +x /install_build_dependencies.sh && \ /install_build_dependencies.sh && \ rm -rf /var/lib/apt/lists/* -# Setup pip -ENV PIP_VERSION="24.0" -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ - rm -f get-pip.py - # Setup pip ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ From e59a45f862c80b949b0d1c524369e99359de8623 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 17:01:47 +0100 Subject: [PATCH 59/60] use install deps script --- .github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile index adbeee914483ed..1af94764570995 100644 --- a/.github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile @@ -25,10 +25,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -# Install build dependencies -ADD install_build_dependencies.sh /install_build_dependencies.sh -RUN chmod +x /install_build_dependencies.sh && \ - /install_build_dependencies.sh && \ +# Install openvino dependencies +ADD scripts/install_dependencies/install_openvino_dependencies.sh /install_openvino_dependencies.sh +RUN chmod +x /install_openvino_dependencies.sh && \ + /install_openvino_dependencies.sh && \ rm -rf /var/lib/apt/lists/* # Create a virtual environment for the system Python as Python in Ubuntu 24 complains about installing From 5247b079640d442a820ff97859344b9300a9f425 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 31 Jul 2024 17:35:26 +0100 Subject: [PATCH 60/60] add comment about gcc-10 on U24; use intall ov deps script in test Dockerfiles --- .github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile | 2 +- .github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile | 8 ++++---- .github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile index a8fa0ddfd3245e..29510fcab065a8 100644 --- a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && \ python3-full \ # For Java API default-jdk \ - # Compiler + # Compiler. TODO: remove and use the system one; if the system one is used, onnx fails to build gcc-10 \ g++-10 \ && \ diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile index 67cec0b1ce806d..f3b63234824960 100644 --- a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile @@ -25,10 +25,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -# Install build dependencies -ADD install_build_dependencies.sh /install_build_dependencies.sh -RUN chmod +x /install_build_dependencies.sh && \ - /install_build_dependencies.sh && \ +# Install openvino dependencies +ADD scripts/install_dependencies/install_openvino_dependencies.sh /install_openvino_dependencies.sh +RUN chmod +x /install_openvino_dependencies.sh && \ + /install_openvino_dependencies.sh && \ rm -rf /var/lib/apt/lists/* # Setup pip diff --git a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile index 1566c2305d04b7..c077c44a1d6f3d 100644 --- a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile @@ -30,10 +30,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -# Install build dependencies -ADD install_build_dependencies.sh /install_build_dependencies.sh -RUN chmod +x /install_build_dependencies.sh && \ - /install_build_dependencies.sh && \ +# Install openvino dependencies +ADD scripts/install_dependencies/install_openvino_dependencies.sh /install_openvino_dependencies.sh +RUN chmod +x /install_openvino_dependencies.sh && \ + /install_openvino_dependencies.sh && \ rm -rf /var/lib/apt/lists/* # Setup pip