install openvino_tokenizers for genai_python_lib

Author: pavel-esir

.github/workflows/genai_python_lib.yml

@@ -16,18 +16,25 @@ jobs:
       - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
       - run: source ./ov/setupvars.sh && cmake --build ./build/ --config Release -j
       - run: python -m pip install --pre openvino --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly  # Can't load CentOS libraries from the archive
+      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers]
       - run: PYTHONPATH=./src/python/ python -c "from openvino_genai import LLMPipeline"
       - run: source ./ov/setupvars.sh && python -m pip install --pre . --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
-      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers]
       - run: python -c "from openvino_genai import LLMPipeline"
+      - name: check paths
+        run: |
+          python -c "
+          import openvino_genai
+          print("openvino_genai path: ", openvino_genai.__path__)
+          import openvino_tokenizers
+          print("openvino_tokenizers path: ", openvino_tokenizers.__path__)
+          "
       - name: GenAI Python API tests
         run: |
           source ./ov/setupvars.sh
-          cd ./tests/
+          cd ./tests/python_tests/
           python -m pip install -r requirements.txt
-          models=$(python3 generate_models.py)
+          models=$(python list_test_models.py)
           echo "$models" | while read -r model_name model_path; do
-              echo "Processing model: $model_name at $model_path"
               optimum-cli export openvino --trust-remote-code --weight-format fp16 --model "$model_name" "$model_path"
           done
           python -m pytest test_generate_api.py
@@ -49,6 +56,7 @@ jobs:
       - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
       - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && cmake --build ./build/ --config Release -j
       - run: python -m pip install "numpy<1.27"
+      - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -m pip install ./thirdparty/openvino_tokenizers/[transformers]
       - run: set "PYTHONPATH=./src/python;" && call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -c "from openvino_genai import LLMPipeline"  # cmd evaluates variables in a different way. Setting PYTHONPATH before setupvars.bat instead of doing that after solves that.
-      - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -m pip install .
+      - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -m pip install .      
       - run: python -c "from openvino_genai import LLMPipeline"

tests/python_tests/list_test_models.py

@@ -1,10 +1,8 @@
-# generate_models.py
-
 def models_list():
     model_ids = [
         ("TinyLlama/TinyLlama-1.1B-Chat-v1.0", "TinyLlama-1.1B-Chat-v1.0"),
-        ("google/gemma-2b-it", "gemma-2b-it"),
-        ("google/gemma-7b-it", "gemma-7b-it"),
+        # ("google/gemma-2b-it", "gemma-2b-it"),
+        # ("google/gemma-7b-it", "gemma-7b-it"),
         # ("meta-llama/Llama-2-7b-chat-hf", "Llama-2-7b-chat-hf"),
         # ("meta-llama/Llama-2-13b-chat-hf", "Llama-2-13b-chat-hf"),
         # ("openlm-research/open_llama_3b", "open_llama_3b"),

tests/python_tests/requirements.txt

@@ -1,3 +1,4 @@
 pytest
 transformers
-torch
+torch
+optimum-intel[openvino] @ git+https://github.com/huggingface/optimum-intel.git@fb1b35bef23242d65b2fb057c4a7ac78a7cfd4c3