Merge branch 'master' into cvs_76333_python_transformations

Author: itikhono

.github/actions/create_manifest/create_manifest.py

@@ -99,8 +99,12 @@ def generate_manifest(repos: list, product_type: str, event_type: str, build_typ
     custom_branch_name = f'-{trigger_repo.branch}' if trigger_repo.branch != 'master' else ''
     run_number_postfix = f'-{os.environ.get("GITHUB_RUN_NUMBER")}' if os.environ.get("GITHUB_RUN_NUMBER") else ''
     product_version = f"{ov_version}{run_number_postfix}-{trigger_repo.revision[:11]}{custom_branch_name}"
-    ci_build_dev_tag = f'dev{trigger_repo.commit_time.strftime("%Y%m%d")}'
-    wheel_product_version = f'{ov_version}.{ci_build_dev_tag}'
+
+    merge_queue_target_branch = next(iter(re.findall(f'^gh-readonly-queue/(.*)/', trigger_repo.branch)), None)
+    target_branch = merge_queue_target_branch or trigger_repo.target_branch or trigger_repo.branch
+    is_release_branch = re.match('^releases/.+$', target_branch)
+    ci_build_dev_tag = f'dev{trigger_repo.commit_time.strftime("%Y%m%d")}' if not is_release_branch else ''
+    wheel_product_version = f'{ov_version}.{ci_build_dev_tag}' if not is_release_branch else ov_version
 
     set_github_output('CI_BUILD_NUMBER', product_version, 'GITHUB_ENV')
     set_github_output('CI_BUILD_DEV_TAG', ci_build_dev_tag, 'GITHUB_ENV')

.github/dockerfiles/docker_tag

@@ -1 +1 @@
-pr-25912
+pr-26086

.github/dockerfiles/ov_build/debian_10_arm/Dockerfile

@@ -0,0 +1,89 @@
+FROM openvinogithubactions.azurecr.io/dockerio/library/debian:10.13
+
+USER root
+
+# APT configuration
+RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
+    echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    TZ="Europe/London"
+
+RUN apt-get update && \
+    apt-get install \
+        software-properties-common \
+        curl \
+        git \
+        gpg-agent \
+        tzdata \
+        # Pythons \
+        python3 \
+        python3-pip \
+        python3-dev \
+        python3-venv \
+        python3-distutils \
+        libhdf5-dev \
+        # For building Python from source
+        build-essential \
+        libffi-dev \
+        libgdbm-dev \
+        libc6-dev \
+        libssl-dev \
+        zlib1g-dev \
+        libbz2-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        libncurses5-dev \
+        libncursesw5-dev \
+        xz-utils \
+        tk-dev \
+        libxml2-dev \
+        libxmlsec1-dev \
+        liblzma-dev \
+        wget \
+        # Compilers
+        gcc-arm-linux-gnueabihf \
+        g++-arm-linux-gnueabihf \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install build dependencies
+ADD install_build_dependencies.sh /install_build_dependencies.sh
+RUN chmod +x /install_build_dependencies.sh && \
+    /install_build_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install sscache
+ARG SCCACHE_VERSION="v0.7.5"
+ENV SCCACHE_HOME="/opt/sccache" \
+    SCCACHE_PATH="/opt/sccache/sccache"
+
+RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
+    SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-aarch64-unknown-linux-musl.tar.gz" && \
+    curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
+    tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}
+
+ENV PATH="$SCCACHE_HOME:$PATH"
+
+# Setup Python
+RUN wget https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tar.xz
+
+RUN tar -xf Python-3.11.9.tar.xz && \
+    cd Python-3.11.9 && \
+    ./configure --enable-optimizations && \
+    make -j 8 && \
+    make altinstall
+
+# Setup pip
+ENV PIP_VERSION="24.0"
+RUN python3 -m pip install --upgrade pip==24.0
+RUN python3.11 -m pip install --upgrade pip==24.0
+
+# Use Python 3.11 as default
+# Using venv here because other methods to switch the default Python break both system and wheels build
+RUN python3.11 -m venv venv
+ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
+
+ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}

.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile

@@ -71,3 +71,11 @@ RUN python3.11 -m venv venv
 ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
 
 ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
+
+# Install Node
+ENV NODE_VERSION=21.7.3
+ENV NVM_DIR=/.nvm
+RUN mkdir -p $NVM_DIR
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
+ENV PATH="$NVM_DIR/versions/node/v${NODE_VERSION}/bin/:${PATH}"

.github/dockerfiles/ov_build/ubuntu_22_04_android_arm64/Dockerfile .github/dockerfiles/ov_build/ubuntu_22_04_android/Dockerfile

@@ -61,3 +61,11 @@ RUN python3.11 -m venv venv
 ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
 
 ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
+
+# Install Node
+ENV NODE_VERSION=21.7.3
+ENV NVM_DIR=/.nvm
+RUN mkdir -p $NVM_DIR
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
+ENV PATH="$NVM_DIR/versions/node/v${NODE_VERSION}/bin/:${PATH}"

.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile

@@ -0,0 +1,95 @@
+FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04
+
+USER root
+
+# APT configuration
+RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
+    echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    TZ="Europe/London"
+
+RUN apt-get update && \
+    apt-get install software-properties-common && \
+    add-apt-repository --yes --no-update ppa:git-core/ppa && \
+    add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get install \
+        wget \
+        curl \
+        git \
+        ca-certificates \
+        gpg-agent \
+        tzdata \
+        libtbb2 \
+        # Pythons \
+        python3.11-dev \
+        python3.11-venv \
+        python3.11-distutils \
+        default-jdk \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+
+# Install OneAPI Toolkit
+RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/intel-oneapi-archive-keyring.gpg > /dev/null
+RUN echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list
+
+RUN apt-get update && \
+    apt-get install \
+        intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install build dependencies
+ADD install_build_dependencies.sh /install_build_dependencies.sh
+RUN chmod +x /install_build_dependencies.sh && \
+    /install_build_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install sscache
+ARG SCCACHE_VERSION="v0.7.5"
+ENV SCCACHE_HOME="/opt/sccache" \
+    SCCACHE_PATH="/opt/sccache/sccache"
+
+RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
+    SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \
+    curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
+    tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}
+
+# Setup pip
+ENV PIP_VERSION="24.0"
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
+    python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
+    rm -f get-pip.py
+
+# Use Python 3.11 as default
+# Using venv here 'cause other methods to switch the default Python on Ubuntu break both system and wheels build
+RUN python3.11 -m venv venv
+ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
+
+ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
+
+# OneAPI env
+ENV ONEAPI_ROOT=/opt/intel/oneapi
+ENV PKG_CONFIG_PATH=/opt/intel/oneapi/compiler/2024.2/lib/pkgconfig
+ENV DIAGUTIL_PATH=/opt/intel/oneapi/dpcpp-ct/2024.2/etc/dpct/sys_check/sys_check.sh:/opt/intel/oneapi/debugger/2024.2/etc/debugger/sys_check/sys_check.py:/opt/intel/oneapi/compiler/2024.2/etc/compiler/sys_check/sys_check.sh
+ENV MANPATH=/opt/intel/oneapi/debugger/2024.2/share/man:/opt/intel/oneapi/compiler/2024.2/share/man:
+ENV GDB_INFO=/opt/intel/oneapi/debugger/2024.2/share/info/
+ENV CMAKE_PREFIX_PATH=/opt/intel/oneapi/compiler/2024.2
+ENV CMPLR_ROOT=/opt/intel/oneapi/compiler/2024.2
+ENV INFOPATH=/opt/intel/oneapi/debugger/2024.2/share/info
+ENV LIBRARY_PATH=/opt/intel/oneapi/compiler/2024.2/lib
+ENV OCL_ICD_FILENAMES=/opt/intel/oneapi/compiler/2024.2/lib/libintelocl.so
+ENV LD_LIBRARY_PATH=/opt/intel/oneapi/debugger/2024.2/opt/debugger/lib:/opt/intel/oneapi/compiler/2024.2/opt/compiler/lib:/opt/intel/oneapi/compiler/2024.2/lib
+ENV NLSPATH=/opt/intel/oneapi/mkl/2024.2/share/locale/%l_%t/%N:/opt/intel/oneapi/compiler/2024.2/lib/compiler/locale/%l_%t/%N
+ENV PATH=$PATH:/opt/intel/oneapi/dev-utilities/2024.2/bin:/opt/intel/oneapi/debugger/2024.2/opt/debugger/bin:/opt/intel/oneapi/compiler/2024.2/bin
+ENV INTEL_PYTHONHOME=/opt/intel/oneapi/debugger/2024.2/opt/debugger
+ENV CPATH=/opt/intel/oneapi/dpl/2022.6/include:/opt/intel/oneapi/dev-utilities/2024.2/include:
+
+# Set Intel DPC++ as a default compiler
+ENV CC=icx
+ENV CXX=icpx

.github/dockerfiles/ov_build/ubuntu_22_04_x64_dpcpp/Dockerfile

@@ -66,5 +66,9 @@
     {
         "error_text": "Unexpected HTTP response: 520",
         "ticket": 147958
+    },
+    {
+        "error_text": "Unable to make request: ECONNRESET",
+        "ticket": 148702
     }
 ]

.github/scripts/workflow_rerun/errors_to_look_for.json

@@ -58,7 +58,7 @@ jobs:
         id: handle_docker
         with:
           images: |
-            ov_build/ubuntu_22_04_android_arm64
+            ov_build/ubuntu_22_04_android
           registry: 'openvinogithubactions.azurecr.io'
           dockerfiles_root_dir: '.github/dockerfiles'
           changed_components: ${{ needs.smart_ci.outputs.changed_components }}
@@ -71,7 +71,7 @@ jobs:
         shell: bash
     runs-on: aks-linux-16-cores-32gb
     container:
-      image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_android_arm64 }}
+      image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_android }}
       volumes:
         - /mount:/mount
       options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING
@@ -91,6 +91,7 @@ jobs:
       ANDROID_NDK_HOME: '/deps/android_tools/ndk-bundle'
       ANDROID_SDK_VERSION: 29
       ANDROID_ABI_CONFIG: arm64-v8a
+      VCPKG_TARGET_TRIPLET: arm64-android
       VCPKG_DEFAULT_BINARY_CACHE: '/mount/caches/ccache/android_arm64/vcpkg_cache'
       VCPKG_FORCE_SYSTEM_BINARIES: '1'
       SCCACHE_AZURE_KEY_PREFIX: android_arm64
@@ -154,7 +155,7 @@ jobs:
             -DENABLE_SYSTEM_FLATBUFFERS=ON \
             -DANDROID_ABI=${{ env.ANDROID_ABI_CONFIG }} \
             -DANDROID_PLATFORM=${{ env.ANDROID_SDK_VERSION }} \
-            -DVCPKG_TARGET_TRIPLET=arm64-android \
+            -DVCPKG_TARGET_TRIPLET=${{ env.VCPKG_TARGET_TRIPLET }} \
             -DVCPKG_HOST_TRIPLET=x64-linux-release \
             -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \
             -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake \
@@ -177,15 +178,15 @@ jobs:
       # Upload build logs
       #
       - name: Upload build logs
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         if: always()
         with:
           name: build_logs
           path: ${{ env.SCCACHE_ERROR_LOG }}
           if-no-files-found: 'ignore'
 
   Overall_Status:
-    name: ci/gha_overall_status_android
+    name: ci/gha_overall_status_android_arm64
     needs: [Smart_CI, Build]
     if: ${{ always() }}
     runs-on: ubuntu-latest