[GHA] OV provider interface updates (openvinotoolkit#26636)

Author: akladiev

.github/actions/common/artifact_utils.py

@@ -3,7 +3,7 @@
 import argparse
 import os
 from pathlib import Path
-from .constants import EventType, ProductType
+from .constants import EventType, ProductType, PlatformKey
 
 
 def add_common_args(parser: argparse.ArgumentParser):
@@ -12,10 +12,14 @@ def add_common_args(parser: argparse.ArgumentParser):
                         default=os.getenv('GITHUB_BASE_REF') or os.getenv('GITHUB_REF_NAME'))
     parser.add_argument('-e', '--event_name', help='Name of GitHub event', required=False,
                         default=os.getenv('GITHUB_EVENT_NAME'))
-    parser.add_argument('--storage_dir', help='Subdirectory name for artifacts, same as product type', required=True,
-                        choices=[product_type.value for product_type in ProductType])
     parser.add_argument('--storage_root', help='Root path of the artifacts storage', required=False,
                         default=os.getenv('ARTIFACTS_SHARE'))
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument('-d', '--storage_dir', help='Subdirectory name for artifacts, same as product type',
+                       choices=[platform_key.value for platform_key in ProductType])
+    group.add_argument('-p', '--platform', type=str,
+                       help='Platform for which to restore artifacts. Used if storage_dir is not set',
+                       choices=[product_type.value for product_type in PlatformKey])
 
 
 def get_event_type(event_name: str = os.getenv('GITHUB_EVENT_NAME')) -> str:

.github/actions/common/constants.py

@@ -18,3 +18,27 @@ class EventType(Enum):
     'public_windows_vs2019_Debug',
 )
 ProductType = Enum('ProductType', {t.upper(): t for t in productTypes})
+
+
+platformKeys = (
+    'centos7_x86_64',
+    'debian10_armhf',
+    'rhel8_x86_64',
+    'ubuntu20_arm64',
+    'ubuntu20_x86_64',
+    'ubuntu22_x86_64',
+    'ubuntu24_x86_64',
+    'macos_12_6_arm64',
+    'macos_12_6_x86_64',
+    'windows_x86_64',
+)
+PlatformKey = Enum('PlatformKey', {t.upper(): t for t in platformKeys})
+
+PlatformMapping = {
+    PlatformKey.DEBIAN10_ARMHF: ProductType.PUBLIC_LINUX_DEBIAN_10_ARM_RELEASE,
+    PlatformKey.UBUNTU20_X86_64: ProductType.PUBLIC_LINUX_UBUNTU_20_04_X86_64_RELEASE,
+    PlatformKey.UBUNTU20_ARM64: ProductType.PUBLIC_LINUX_UBUNTU_20_04_ARM64_RELEASE,
+    PlatformKey.UBUNTU22_X86_64: ProductType.PUBLIC_LINUX_UBUNTU_22_04_X86_64_RELEASE,
+    PlatformKey.UBUNTU24_X86_64: ProductType.PUBLIC_LINUX_UBUNTU_24_04_X86_64_RELEASE,
+    PlatformKey.WINDOWS_X86_64: ProductType.PUBLIC_WINDOWS_VS2019_RELEASE,
+}

.github/actions/openvino_provider/action.yml

@@ -2,7 +2,8 @@ name: 'OpenVINO provider'
 description: 'Provides OpenVINO build artifacts from a requested source'
 inputs:
   revision:
-    description: 'Version of OpenVINO to provide - latest_available | HEAD | specific commit hash'
+    description: 'Version of OpenVINO to provide - latest_available_commit | HEAD | specific commit hash | 
+                  latest_nightly | specific package version (e.g. 2024.4.0rc2)'
     required: true
   branch_name:
     description: 'Branch of OpenVINO to take the revision from if no specific hash was provided. 
@@ -11,21 +12,25 @@ inputs:
   ov_artifact_name:
     description: "Name under which to upload provided OpenVINO build artifacts, set automatically by default"
     required: false
+  platform:
+    description: "Platform for which to get artifacts: centos7 | debian10 | rhel8 | ubuntu20 | 
+      ubuntu20 | ubuntu22 | ubuntu24 | macos | macos_12_6 | windows"
+    required: false
+  arch:
+    description: "Architecture for which to get artifacts: x86_64 | armhf | arm64"
+    default: "x86_64"
+    required: false
+  install_dir:
+    description: 'Local path to install OpenVINO package to. If not specified, package is installed to GitHub workspace 
+                  and uploaded to GitHub'
+    required: false
   commit_packages_to_provide:
     description: "Comma-separated list of OpenVINO packages from post-commit to provide ('all' to get everything)"
     required: false
-  commit_build_key:
-    description: 'public_linux_ubuntu_20_04_release | public_linux_ubuntu_22_04_release | 
-                  public_linux_ubuntu_24_04_release | public_windows_vs2019_release'
-    required: false
   commit_share_path:
     description: 'Path to share holding OpenVINO post-commit artifacts'
     required: false
     default: '/mount/build-artifacts'
-  nightly_build_key:
-    description: 'centos7_x86_64 | debian9_armhf | rhel8_x86_64 | ubuntu20_arm64 | ubuntu20_x86_64 | ubuntu22_x86_64 | 
-                  ubuntu24_x86_64 | macos_12_6_arm64 | macos_12_6_x86_64 | windows_x86_64'
-    required: false
   nightly_package_source:
     description: 'Source URL hosting OpenVINO nightly archives'
     required: false
@@ -34,40 +39,44 @@ inputs:
     description: 'Source URL hosting OpenVINO wheels, for passing to pip as --extra-index-url'
     required: false
     default: 'https://storage.openvinotoolkit.org/simple/wheels'
-  package_url:
-    description: 'Direct URL to OpenVINO archive to use, if neither post-commit nor nightly archives are required'
-    required: false
+
 outputs:
   ov_artifact_name:
     description: "Name of the artifact to upload OpenVINO build artifacts under"
-    value: ${{ steps.openvino_download.outputs.ov_artifact_name || steps.openvino_commit_output.outputs.ov_artifact_name }}
-  ov_package_path:
-    description: "Path to OpenVINO core package relative to the root of uploaded artifacts"
-    value: ${{ steps.openvino_download.outputs.ov_package_path || steps.openvino_commit_output.outputs.ov_package_path }}
-  ov_wheel_input:
-    description: "Input for pip to install OpenVINO python wheel"
-    value: ${{ steps.openvino_wheel.outputs.ov_wheel_input || steps.openvino_commit_wheel.outputs.ov_wheel_input }}
+    value: ${{ steps.openvino_s3_download.outputs.ov_artifact_name || steps.openvino_commit_output.outputs.ov_artifact_name }}
   ov_wheel_source:
     description: "Pip option for custom OV wheel location (--find-links or --extra-index-url)"
-    value: ${{ steps.openvino_wheel.outputs.ov_wheel_source || steps.openvino_commit_wheel.outputs.ov_wheel_source }}
+    value: ${{ steps.openvino_s3_wheel.outputs.ov_wheel_source || steps.openvino_commit_wheel.outputs.ov_wheel_source }}
+  ov_version:
+    description: "OpenVINO product version"
+    value: ${{ steps.openvino_s3_wheel.outputs.ov_version || steps.openvino_commit_wheel.outputs.ov_version }}
 
 runs:
   using: "composite"
   steps:
-    - name: Verify parameters
-      if: ${{ !inputs.package_url && !inputs.nightly_build_key && !inputs.commit_build_key }}
+    - name: Identify artifacts source
+      id: get_source
       shell: bash
       run: |
-        echo "Please, set one parameter for action - nightly_build_key, commit_build_key or package_url"
-        exit 1
+        artifacts_source="postcommit"
+        if [[ "${{ inputs.revision }}" == "latest_nightly" || "${{ inputs.revision }}" =~ ^[0-9]{4}\.[0-9]+\.*[0-9]* ]]; then
+          artifacts_source="s3"
+        fi
+        echo "artifacts_source=$artifacts_source" >> $GITHUB_OUTPUT
+
+    - name: Get action revision
+      id: get_action_revision
+      run: echo "action_ref=${{ env.ACTION_REF }}" >> $GITHUB_OUTPUT
+      shell: bash
+      env:
+        ACTION_REF: ${{ github.action_ref || github.base_ref || github.event.merge_group.base_ref || github.ref }}
 
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         repository: 'openvinotoolkit/openvino'
-        ref: ${{ inputs.branch_name || github.base_ref || github.event.merge_group.base_ref || github.ref }}
+        ref: ${{ steps.get_action_revision.outputs.action_ref }}
         sparse-checkout: .github/actions
 
-    # --- Post-commit case ---
     - name: Clone OpenVINO to get HEAD commit
       if: inputs.revision == 'HEAD'
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -84,18 +93,25 @@ runs:
         head=$(git -C openvino rev-parse HEAD)
         echo "head=$head" >> $GITHUB_OUTPUT
 
+    - name: Create install dir
+      if: inputs.install_dir
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.install_dir }}
+
+    # --- Post-commit case ---
     - name: Download post-commit OpenVINO archives
       id: openvino_commit_download
-      if: inputs.commit_build_key
+      if: steps.get_source.outputs.artifacts_source == 'postcommit'
       uses: ./.github/actions/restore_artifacts
       with:
-        storage_dir: ${{ inputs.commit_build_key }}
+        platform: ${{ inputs.platform }}_${{ inputs.arch }}
         storage_root: ${{ inputs.commit_share_path }}
         event_name: "commit"
         trigger_repo_sha: ${{ env.OV_REVISION }}
         branch_name: ${{ inputs.branch_name }}
-        artifacts_key: ${{ inputs.ov_artifact_name || inputs.commit_build_key }}
         to_restore: ${{ inputs.commit_packages_to_provide }}
+        target_dir: ${{ inputs.install_dir }}
       env:
         OV_REVISION: ${{ inputs.revision == 'HEAD' && steps.get_openvino_head_commit.outputs.head || inputs.revision }}
 
@@ -104,79 +120,98 @@ runs:
       if: steps.openvino_commit_download.outcome == 'success'
       shell: bash
       run: |
-        ov_artifact_name=${{ inputs.ov_artifact_name || steps.openvino_commit_download.outputs.restored_artifacts_key }}
-        cd ${{ steps.openvino_commit_download.outputs.artifacts_workspace_path }}
+        ov_artifact_name=${{ inputs.ov_artifact_name || inputs.platform }}
+        workspace_path=${{ steps.openvino_commit_download.outputs.artifacts_path }}
+        cd $workspace_path
         ov_package_path=$(find . -name 'openvino_package*' -printf "%P\n" -quit)
         [ -z "$ov_package_path" ] && echo "No openvino_package found in restored artifacts" && exit 1
+        mv $ov_package_path/* . && rm -rf $ov_package_path
         echo "ov_artifact_name=$ov_artifact_name" >> $GITHUB_OUTPUT
-        echo "ov_package_path=$ov_package_path" >> $GITHUB_OUTPUT
+        echo "ov_package_path=$workspace_path" >> $GITHUB_OUTPUT
 
     - name: Get commit wheel
       id: openvino_commit_wheel
       if: steps.openvino_commit_download.outcome == 'success'
       shell: bash
       run: |
-        cd ${{ steps.openvino_commit_download.outputs.artifacts_workspace_path }}
+        artifacts_path=${{ steps.openvino_commit_download.outputs.artifacts_path }}
+        cd $artifacts_path
         version=$(yq eval '.components.dldt.custom_params.wheel_product_version' manifest.yml)
-        wheel_path=${{ steps.openvino_commit_output.outputs.ov_package_path }}/tools
-        default_find_links_cmd="--find-links=./$wheel_path"
+        wheel_path=${{ inputs.install_dir && '$artifacts_path/tools' || './tools' }}
+        default_find_links_cmd="--find-links=$wheel_path"
         find_links_cmd=$([[ -n "$PIP_FIND_LINKS" ]] && echo "" || echo "$default_find_links_cmd")
-        wheel_input="$find_links_cmd openvino==$version"
-        echo "ov_wheel_input=$wheel_input" >> $GITHUB_OUTPUT
+        echo "ov_version=$version" >> $GITHUB_OUTPUT
         echo "ov_wheel_source=$default_find_links_cmd" >> $GITHUB_OUTPUT
 
+    - name: Upload commit OpenVINO archives
+      if: steps.openvino_commit_download.outcome == 'success' && !inputs.install_dir
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ steps.openvino_commit_output.outputs.ov_artifact_name }}
+        path: ${{ steps.openvino_commit_output.outputs.ov_package_path }}
+        if-no-files-found: 'error'
 
-    # --- S3 case (nightly or direct URL) ---
-    - name: Verify parameters
-      if: (inputs.nightly_build_key || inputs.package_url) && inputs.revision != 'latest_available'
+    # --- S3 case ---
+    - name: Get a specific S3 package URL
+      id: get_s3_package
+      if: steps.get_source.outputs.artifacts_source == 's3' && inputs.revision != 'latest_nightly'
       shell: bash
-      run: |
-        echo "Nightly and package_url build sources provide only 'latest' artifacts, specific revision not supported"
-        exit 1
+      run: >-
+        python3 -m venv venv && . venv/bin/activate &&
+        pip install -r $GITHUB_ACTION_PATH/requirements.txt &&
+        python $GITHUB_ACTION_PATH/get_s3_package.py --product openvino --platform ${{ inputs.platform }} --arch ${{ inputs.arch }} 
+        --version ${{ inputs.revision }}
 
-    - name: Download OpenVINO archives
-      id: openvino_download
-      if: inputs.nightly_build_key || inputs.package_url
+    - name: Download nightly OpenVINO archives
+      id: openvino_s3_download
+      if: steps.get_source.outputs.artifacts_source == 's3'
       shell: bash
       run: |
-        if [ ${{ inputs.package_url }} ]; then
-          ov_package_url=${{ inputs.package_url }}
+        if [ ${{ steps.get_s3_package.outcome }} != 'skipped' ]; then
+          ov_package_url=${{ steps.get_s3_package.outputs.package_url }}
         else
-          ov_package_url=$(curl -s ${{ inputs.nightly_package_source }} | jq -r '.${{ inputs.nightly_build_key }}')
+          ov_package_url=$(curl -s ${{ inputs.nightly_package_source }} | jq -r '.${{ inputs.platform }}_${{ inputs.arch }}')
         fi
+        cd ${{ inputs.install_dir || env.GITHUB_WORKSPACE }}
         package_basename=$(basename $ov_package_url)
         wget $ov_package_url --progress=bar:force:noscroll -O $package_basename
-        package_folder=${package_basename%.*} && mkdir $package_folder
-        [[ "$package_basename" == *.zip ]] && unzip "$package_basename" -d $package_folder || tar -xvf "$package_basename" -C $package_folder
+        package_folder=${package_basename%.*}
+        [[ "$package_basename" == *.zip ]] && unzip "$package_basename" || tar -xvf "$package_basename"
+        rm $package_basename
+        if [ ${{ inputs.install_dir }} ]; then
+          mv $package_folder/* . && rm -rf $package_folder
+        fi
         ov_artifact_name=$([ ${{ inputs.ov_artifact_name }} ] && echo ${{ inputs.ov_artifact_name }} || echo $package_folder)
+        echo "ov_package_url=$ov_package_url" >> $GITHUB_OUTPUT
         echo "ov_artifact_name=$ov_artifact_name" >> $GITHUB_OUTPUT
         echo "ov_package_path=$package_folder" >> $GITHUB_OUTPUT
 
     - name: Upload OpenVINO archives
-      if: steps.openvino_download.outcome == 'success'
+      if: steps.openvino_s3_download.outcome == 'success' && !inputs.install_dir
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ steps.openvino_download.outputs.ov_artifact_name }}
-        path: ${{ steps.openvino_download.outputs.ov_package_path }}
+        name: ${{ steps.openvino_s3_download.outputs.ov_artifact_name }}
+        path: ${{ steps.openvino_s3_download.outputs.ov_package_path }}
         if-no-files-found: 'error'
 
     - name: Get wheel
-      id: openvino_wheel
-      if: steps.openvino_download.outcome == 'success'
+      id: openvino_s3_wheel
+      if: steps.openvino_s3_download.outcome == 'success'
       shell: bash
       run: |
-        rc_version=$(echo ${{ inputs.package_url }} |
+        rc_version=$(echo ${{ steps.openvino_s3_download.outputs.ov_package_url }} |
           grep -oP '(?<=pre-release\/)\d{4}\.\d+\.\d+rc\d+' || true)
 
-        release_version=$(echo ${{ steps.openvino_download.outputs.ov_package_path }} |
+        release_version=$(echo ${{ steps.openvino_s3_download.outputs.ov_package_path }} |
           grep -oP '_\K\d{4}\.\d+\.\d+')
         
-        dev_version=$(echo ${{ steps.openvino_download.outputs.ov_package_path }} |
+        dev_version=$(echo ${{ steps.openvino_s3_download.outputs.ov_package_path }} |
           grep -oP "${release_version//./\\.}\.dev\d{8}" || true)
 
         version=${rc_version:-${dev_version:-$release_version}}
+
         extra_index="--pre --extra-index-url ${{ inputs.pip_extra_url }}/nightly \
           --extra-index-url ${{ inputs.pip_extra_url }}/pre-release"
-        wheel_input="$extra_index openvino==$version"
-        echo "ov_wheel_input=$wheel_input" >> $GITHUB_OUTPUT
+
+        echo "ov_version=$version" >> $GITHUB_OUTPUT
         echo "ov_wheel_source=$extra_index" >> $GITHUB_OUTPUT

.github/actions/openvino_provider/get_s3_package.py

@@ -0,0 +1,77 @@
+import requests
+import re
+import argparse
+import logging
+import sys
+
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parents[1]))
+from common import action_utils
+
+
+# Function to download the JSON file
+def load_json_file(url):
+    response = requests.get(url)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        raise Exception(f"Failed to download the file, status code: {response.status_code}")
+
+
+# Function to recursively gather all file paths from the JSON structure
+def gather_all_files(node, base_path=''):
+    all_files = []
+    if 'children' in node:
+        for child in node['children']:
+            new_base = f"{base_path}/{child['name']}"
+            if 'children' in child:
+                all_files += gather_all_files(child, new_base)
+            else:
+                all_files.append(new_base)
+    return all_files
+
+
+# Function to filter files based on the product, version, platform, architecture, and folder
+def filter_files_by_criteria(files, product, version_pattern, platform, arch, folder):
+    matching_files = []
+    for file in files:
+        if re.search(
+                fr"{product}/packages/{folder}(/?.*/)?{version_pattern}/.*{platform}.*{arch}.*\.(tar\.gz|tgz|zip)$",
+                file):
+            matching_files.append(file)
+    return matching_files
+
+
+# Main function to load the JSON, gather file paths, and filter based on criteria
+def main(product, version_pattern, platform, arch, folder):
+    action_utils.init_logger()
+    logger = logging.getLogger(__name__)
+
+    url = 'https://storage.openvinotoolkit.org/filetree.json'
+    filetree = load_json_file(url)
+    all_files = gather_all_files(filetree)
+    matching_files = filter_files_by_criteria(all_files, product, version_pattern, platform, arch, folder)
+    if matching_files:
+        logger.info(f"Matching packages: {sorted(matching_files)}")
+        package_url = f"https://storage.openvinotoolkit.org{sorted(matching_files)[-1]}"
+        logger.info(f"Returning package URL: {package_url}")
+        action_utils.set_github_output("package_url", package_url)
+    else:
+        logger.error("No matching files found.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description='Search OpenVINO archives based on product, version, platform, architecture, and folder.')
+    parser.add_argument('--product', required=True, choices=['openvino', 'openvino_genai', 'openvino_tokenizers'],
+                        help='Product name')
+    parser.add_argument('--version', required=True, help='Version pattern (supports regex)')
+    parser.add_argument('--platform', default='ubuntu22', help='Platform (default: ubuntu22)')
+    parser.add_argument('--arch', default='x86_64', help='Architecture (default: x86_64)')
+    parser.add_argument('--folder', default='(.*/)?', help='Folder type (e.g., pre-release, nightly, default: (.*)?')
+
+    args = parser.parse_args()
+
+    main(args.product, args.version, args.platform, args.arch, args.folder)

.github/actions/openvino_provider/requirements.txt

@@ -0,0 +1 @@
+requests~=2.32