[TF FE][OV Tokenizers] Deploy use of OpenVINO Tokenizers in FE layer tests (openvinotoolkit#22896)

**Details:** We are going to test OpenVINO Tokenizers conversion
extensions by FE layer tests. It gives better quality integration
between openvino and openvino-tokenizers. For example, operation on
string tensors such as tf.raw_ops.StringLower, tf.raw_ops.StringSplitV2
will supported via OV tokenizers near feature until we move them into
core part.

**Ticket:** 132667

---------

Signed-off-by: Kazantsev, Roman <roman.kazantsev@intel.com>

Author: rkazants

.github/workflows/job_python_unit_tests.yml

@@ -52,6 +52,13 @@ jobs:
           name: openvino_tests
           path: ${{ env.INSTALL_TEST_DIR }}
 
+      - name: Download OpenVINO tokenizers extension
+        if: ${{ runner.os != 'macOS' && runner.arch != 'ARM64' }} # Ticket: 126287
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_tokenizers_wheel
+          path: ${{ env.INSTALL_DIR }}
+
       # Needed as ${{ github.workspace }} is not working correctly when using Docker
       - name: Setup Variables
         run: |
@@ -98,6 +105,10 @@ jobs:
           # Install the core OV wheel
           python3 -m pip install ${INSTALL_DIR}/tools/openvino-*.whl
 
+          if [[ "${{ runner.arch }}" != "ARM64" ]]; then
+            python3 -m pip install ${INSTALL_DIR}/openvino_tokenizers-*.whl
+          fi
+
           extras_to_install="caffe,kaldi,onnx,tensorflow2,pytorch"
 
           if [[ "${{ runner.arch }}" != "ARM64" ]]; then

.github/workflows/linux.yml

@@ -454,7 +454,7 @@ jobs:
 
   Python_Unit_Tests:
     name: Python unit tests
-    needs: [ Build, Smart_CI ]
+    needs: [ Build, Smart_CI, Openvino_tokenizers ]
     uses: ./.github/workflows/job_python_unit_tests.yml
     with:
       runner: 'aks-linux-4-cores-16gb'

.github/workflows/windows.yml

@@ -484,7 +484,7 @@ jobs:
 
   Python_Unit_Tests:
     name: Python unit tests
-    needs: [ Build, Smart_CI ]
+    needs: [ Build, Smart_CI, Openvino_tokenizers ]
     timeout-minutes: 75
     defaults:
       run:
@@ -505,6 +505,12 @@ jobs:
           name: openvino_package
           path: ${{ env.INSTALL_DIR }}
 
+      - name: Download OpenVINO tokenizers extension
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_tokenizers_wheel
+          path: ${{ env.INSTALL_DIR }}
+
       - name: Download OpenVINO tests package
         uses: actions/download-artifact@v3
         with:
@@ -542,6 +548,10 @@ jobs:
           $ovCoreWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\tools" -Filter openvino-*.whl | % { $_.FullName }
           python3 -m pip install "$ovCoreWheelPath"
 
+          # Find and install the core OV Tokenizers wheel
+          $ovCoreWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}" -Filter openvino_tokenizers-*.whl | % { $_.FullName }
+          python3 -m pip install "$ovCoreWheelPath"
+
           # Find and install the dev OV wheel
           $ovDevWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\tools" -Filter openvino_dev*.whl | % { $_.FullName }
           python3 -m pip install "$ovDevWheelPath[mxnet,caffe,kaldi,onnx,tensorflow2,pytorch]"

tests/layer_tests/common/layer_utils.py

@@ -1,11 +1,22 @@
 # Copyright (C) 2018-2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
+import platform
 import subprocess
 import sys
 
 from common.utils.multiprocessing_utils import multiprocessing_run
 from openvino.runtime import Core, get_version as ie2_get_version
 
+try:
+    # noinspection PyUnresolvedReferences
+    import openvino_tokenizers  # do not delete, needed for validation of OpenVINO tokenizers extensions
+except:
+    # TODO 132908: add build OpenVINO Tokenizers in GHA for MacOS and ARM64
+    # TODO 132909: add build OpenVINO Tokenizers in Jenkins for layer_ubuntu20_release tests
+    assert platform.system() in ('Linux', 'Darwin') or platform.machine() in ('arm', 'armv7l',
+                                                                              'aarch64',
+                                                                              'arm64', 'ARM64')
+
 
 def shell(cmd, env=None, cwd=None):
     if sys.platform.startswith('linux') or sys.platform == 'darwin':
@@ -32,6 +43,7 @@ def infer(self, input_data, config=None, infer_timeout=10):
         self.res = multiprocessing_run(self.fw_infer, [input_data, config], self.name, infer_timeout)
         return self.res
 
+
 class InferAPI(BaseInfer):
     def __init__(self, model, weights, device, use_legacy_frontend):
         super().__init__('OpenVINO')

tests/layer_tests/common/utils/common_utils.py

@@ -3,6 +3,7 @@
 
 import logging
 import os
+import platform
 import shutil
 import subprocess
 import sys
@@ -57,6 +58,17 @@ def generate_ir_python_api(coverage=False, **kwargs):
         serialize(ov_model, out_dir)
     else:
         from openvino import convert_model, save_model
+        try:
+            # noinspection PyUnresolvedReferences
+            import openvino_tokenizers  # do not delete, needed for validation of OpenVINO tokenizers extensions
+        except:
+            # TODO 132908: add build OpenVINO Tokenizers in GHA for MacOS and ARM64
+            # TODO 132909: add build OpenVINO Tokenizers in Jenkins for layer_ubuntu20_release tests
+            assert platform.system() in ('Linux', 'Darwin') or platform.machine() in ('arm', 'armv7l',
+                                                                                      'aarch64',
+                                                                                      'arm64', 'ARM64')
+            # CI Jenkins job and ARM64 has no openvino_tokenizers available
+            pass
 
         # cleanup parameters for convert
         if 'output_dir' in kwargs: