job_pytorch_models_tests.yml

name: PyTorch Models tests

on:
  workflow_call:
    inputs:
      runner:
        description: 'Machine on which the tests would run'
        type: string
        required: true
      container:
        description: 'JSON to be converted to the value of the "container" configuration for the job'
        type: string
        required: false
        default: '{"image": null}'
      event:
        description: 'Event that triggered the workflow. E.g., "schedule" for nightly runs'
        type: string
        required: true

jobs:
  PyTorch_Models_Tests:
    name: PyTorch Models tests
    timeout-minutes: ${{ inputs.event == 'schedule' && 400 || 30 }}
    runs-on: ${{ inputs.runner }}
    container: ${{ fromJSON(inputs.container) }}
    defaults:
      run:
        shell: bash
    env:
      DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
      OPENVINO_REPO: ${{ github.workspace }}/openvino
      INSTALL_DIR: ${{ github.workspace }}/install
      INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
      MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests
    steps:
      - name: Check sudo
        if: ${{ runner.os == 'Linux' }}
        run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi

      - name: Set apt retries
        run: |
          if [ "$(id -u)" -eq 0 ]; then
            echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
          else
            sudo sh -c "echo 'Acquire::Retries \"10\";' >> /etc/apt/apt.conf.d/80-retries"
          fi

      - name: Download OpenVINO package
        uses: actions/download-artifact@v3
        with:
          name: openvino_package
          path: ${{ env.INSTALL_DIR }}

      - name: Download OpenVINO tokenizers extension
        uses: actions/download-artifact@v3
        with:
          name: openvino_tokenizers_wheel
          path: ${{ env.INSTALL_DIR }}

      - name: Download OpenVINO tests package
        uses: actions/download-artifact@v3
        with:
          name: openvino_tests
          path: ${{ env.INSTALL_TEST_DIR }}

      # Needed as ${{ github.workspace }} is not working correctly when using Docker
      - name: Setup Variables
        run: |
          echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV"
          echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV"
          echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV"
          echo "MODEL_HUB_TESTS_INSTALL_DIR=$GITHUB_WORKSPACE/install/tests/model_hub_tests" >> "$GITHUB_ENV"

      - name: Extract OpenVINO packages
        run: |
          pushd ${INSTALL_DIR}
            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
          popd

          pushd ${INSTALL_TEST_DIR}
            tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR}
          popd

      - name: Fetch setup_python action
        uses: actions/checkout@v4
        with:
          sparse-checkout: |
            .github/actions/setup_python/action.yml
          sparse-checkout-cone-mode: false
          path: 'openvino'

      - name: Install dependencies
        if: ${{ runner.os == 'Linux' }}
        run: |
          # install git (required to build pip deps from the sources)
          # install 'g++' to build 'detectron2' and 'natten' wheels
          sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates

      - name: Setup Python 3.11
        uses: ./openvino/.github/actions/setup_python
        with:
          version: '3.11'
          should-setup-pip-paths: 'false'
          self-hosted-runner: ${{ contains(inputs.runner, 'aks') }}

      - name: Install OpenVINO Python wheels
        run: |
          python3 -m pip install ${INSTALL_DIR}/tools/openvino-*
          python3 -m pip install ${INSTALL_DIR}/openvino_tokenizers-*

      - name: Install PyTorch tests requirements
        run: |
          python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements.txt
          python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements_secondary.txt
          echo "Available storage:"
          df -h
        env:
          CPLUS_INCLUDE_PATH: ${{ env.Python_ROOT_DIR }}/include/python${{ env.PYTHON_VERSION }}

      - name: PyTorch Models Tests
        run: |
          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v
        env:
          TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}}
          TEST_DEVICE: CPU
          USE_SYSTEM_CACHE: False
          OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log

      - name: Reformat unsupported ops file
        if: '!cancelled()'
        run: |
          python3 ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/scripts/process_op_report.py ${INSTALL_TEST_DIR}/TEST-torch_unsupported_ops.log
  
      - name: Available storage after tests
        run: |
          echo "Available storage:"
          df -h

      - name: Upload Test Results
        uses: actions/upload-artifact@v3
        if: ${{ !cancelled() }}
        with:
          name: test-results-torch-models
          path: |
            ${{ env.INSTALL_TEST_DIR }}/TEST-torch*
          if-no-files-found: 'error'