[PT FE] Update tests for timm models (openvinotoolkit#25696)

### Details:
 - *Update timm model list according to version 1.0.7*
 - *Update for torch==2.3.1*
 - *Update torchvision models list for export scenario*

### Tickets:
 - *ticket-id*

Author: mvafin

.github/workflows/job_pytorch_models_tests.yml

@@ -134,7 +134,7 @@ jobs:
         if: ${{ inputs.model_scope == 'precommit' || inputs.model_scope == 'nightly_scope1' }}
         run: |
           export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
-          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_timm_tv_tests.html --self-contained-html -v -n 4 -k "TestTimmConvertModel or TestTorchHubConvertModel"
+          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_timm_tv_${{ inputs.model_scope }}_tests.html --self-contained-html -v -n 2 -k "TestTimmConvertModel or TestTorchHubConvertModel"
         env:
           TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }}
           TEST_DEVICE: CPU
@@ -144,7 +144,7 @@ jobs:
         if: ${{ inputs.model_scope == 'precommit' || inputs.model_scope == 'nightly_scope2' }}
         run: |
           export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
-          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v -k "not (TestTimmConvertModel or TestTorchHubConvertModel)"
+          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_${{ inputs.model_scope }}_tests.html --self-contained-html -v -k "not (TestTimmConvertModel or TestTorchHubConvertModel)"
         env:
           TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }}
           TEST_DEVICE: CPU
@@ -155,7 +155,7 @@ jobs:
         if: ${{ inputs.model_scope == 'precommit' }}
         run: |
           export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
-          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_pa_transformation.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_pagedattention_tests.html --self-contained-html -v --tb=short -n 4
+          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_pa_transformation.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_pagedattention_tests.html --self-contained-html -v --tb=short -n 2
         env:
           TEST_DEVICE: CPU
           USE_SYSTEM_CACHE: False
@@ -183,7 +183,7 @@ jobs:
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: ${{ !cancelled() }}
         with:
-          name: test-results-torch-models
+          name: test-results-torch-models-${{ inputs.model_scope == 'precommit' }}
           path: |
             ${{ env.INSTALL_TEST_DIR }}/TEST-torch*
           if-no-files-found: 'error'

.github/workflows/linux.yml

@@ -590,7 +590,7 @@ jobs:
   #     - /mount:/mount
   PyTorch_Models_Tests:
     name: PyTorch Models tests
-    if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test
+    if: ${{ github.event_name != 'schedule' && fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test }}
     needs: [ Build, Smart_CI, Openvino_tokenizers ]
     uses: ./.github/workflows/job_pytorch_models_tests.yml
     with:

src/bindings/python/src/openvino/frontend/pytorch/fx_decoder.py

@@ -267,9 +267,7 @@ def get_named_input(self, name):
         raise RuntimeError("This input is not a Node")
 
     def get_subgraph_size(self):
-        if issubclass(type(self.pt_module), torch.fx.Node):
-            return 0
-        return len(self.get_subgraphs()) if hasattr(self.pt_module, 'blocks') else 1
+        return len(self.get_subgraphs())
 
     def decoder_type_name(self) -> str:
         return "fx"
@@ -287,9 +285,7 @@ def visit_subgraph(self, node_visitor):
             node_visitor(decoder)
 
     def get_subgraphs(self):
-        if issubclass(type(self.pt_module), torch.fx.Node):
-            return []
-        return list(self.pt_module.blocks())
+        return []
 
     def get_subgraph_decoder(self, index):
         decoder = TorchFXPythonDecoder(self.get_subgraphs()[index],

src/frontends/pytorch/src/translate_session.cpp

@@ -225,7 +225,8 @@ std::shared_ptr<Model> TranslateSession::convert_pytorch_model(
             }
         };
 
-        FRONT_END_GENERAL_CHECK(pytorch_model->get_subgraph_size() == 1, "Model should have exactly 1 subgraph.");
+        FRONT_END_GENERAL_CHECK(pytorch_model->decoder_type_name() != "ts" || pytorch_model->get_subgraph_size() == 1,
+                                "Model should have exactly 1 subgraph for TorchScript.");
         pytorch_model->visit_subgraph(node_visitor);
 
         ResultVector results;

tests/constraints.txt

@@ -27,6 +27,7 @@ jaxlib<=0.4.14
 kornia==0.7.0
 networkx<=3.3
 keras>=2.0.0,<3.0.0
+timm==1.0.7
 
 --extra-index-url https://download.pytorch.org/whl/cpu
 torch>=1.13,<2.4

tests/model_hub_tests/models_hub_common/utils.py

@@ -27,7 +27,6 @@ def get_models_list(file_name: str):
                 model_name, model_link = model_info.split(',')
             elif len(model_info.split(',')) == 4:
                 model_name, model_link, mark, reason = model_info.split(',')
-                assert mark in ["skip", "xfail"], "Incorrect failure mark for model info {}".format(model_info)
             models.append((model_name, model_link, mark, reason))
 
     return models

tests/model_hub_tests/pytorch/hf_transformers_models

@@ -4,7 +4,7 @@ abeja/gpt-neox-japanese-2.7b,gpt_neox_japanese
 acl-submission-anonym/EAM-spectral,examuse,skip,Load problem
 adalbertojunior/modular-test,modular,skip,Load problem
 adept/persimmon-8b-base,persimmon
-aerner/lm-v2,open-llama,xfail,Example input problem
+aerner/lm-v2,open-llama
 afonsosamarques/ardt-vanilla-combo_train_hopper_v2-2508_1336-33,decision_transformer,xfail,Tracing problem
 aihijo/gec-zh-gector-bert-large,gector,skip,Load problem
 albert-base-v2,albert
@@ -170,6 +170,7 @@ huggingface/time-series-transformer-tourism-monthly,time_series_transformer,skip
 HuggingFaceM4/tiny-random-idefics,idefics,xfail,Unsupported op aten::any aten::einsum prim::TupleConstruct prim::TupleUnpack
 HuggingFaceM4/tiny-random-vllama-clip,vllama,skip,Load problem
 HuggingFaceM4/tiny-random-vopt-clip,vopt,skip,Load problem
+HuggingFaceH4/zephyr-7b-beta,mistral
 HuiHuang/gpt3-damo-base-zh,gpt3,skip,Load problem
 hustvl/yolos-tiny,yolos
 iakarshu/tilt_base,tilt_base_configuration,skip,Load problem
@@ -184,7 +185,7 @@ jaketae/fastspeech2-ljspeech,fastspeech2,skip,Load problem
 jambran/depression-classification,DepressionDetection,skip,Load problem
 Jellywibble/dalio-reward-charlie-v1,reward-model,skip,Load problem
 JonasGeiping/crammed-bert-legacy,crammedBERT,skip,Load problem
-jonatasgrosman/wav2vec2-large-xlsr-53-english,wav2vec2,xfail,Unsupported op aten::index_put_ prim::TupleConstruct
+jonatasgrosman/wav2vec2-large-xlsr-53-english,wav2vec2
 Joqsan/test-my-fnet,my_fnet,skip,Load problem
 jozhang97/deta-swin-large,deta,skip,Load problem
 jploski/retnet-mini-shakespeare,retnet,skip,Load problem
@@ -257,7 +258,6 @@ microsoft/xclip-base-patch32,xclip
 microsoft/xprophetnet-large-wiki100-cased,xlm-prophetnet
 miguelvictor/python-fromzero-lstmlm,lstmlm,skip,Load problem
 mingzi151/test-hf-wav2vec2bert,wav2vec2bert,skip,Load problem
-mistralai/Mistral-7B-v0.1,mistral
 MIT/ast-finetuned-audioset-10-10-0.4593,audio-spectrogram-transformer
 Mizuiro-sakura/luke-japanese-large-sentiment-analysis-wrime,luke
 mlml-chip/thyme2_colon_e2e,cnlpt,skip,Load problem

tests/model_hub_tests/pytorch/test_timm.py

@@ -8,7 +8,7 @@
 import torch
 from models_hub_common.utils import get_models_list
 
-from torch_utils import TestTorchConvertModel, process_pytest_marks
+from torch_utils import TestTorchConvertModel
 
 
 def filter_timm(timm_list: list) -> list:
@@ -42,10 +42,6 @@ def filter_timm(timm_list: list) -> list:
     return sorted([v[1] for v in unique_models.values()])
 
 
-def get_all_models() -> list:
-    return process_pytest_marks(os.path.join(os.path.dirname(__file__), "timm_models"))
-
-
 # To make tests reproducible we seed the random generator
 torch.manual_seed(0)
 
@@ -82,10 +78,16 @@ def test_convert_model_precommit(self, name, ie_device):
         self.run(name, None, ie_device)
 
     @pytest.mark.nightly
-    @pytest.mark.parametrize("name", get_all_models())
+    @pytest.mark.parametrize("name,link,mark,reason", get_models_list(os.path.join(os.path.dirname(__file__), "timm_models")))
     @pytest.mark.parametrize("mode", ["trace", "export"])
-    def test_convert_model_all_models(self, mode, name, ie_device):
+    def test_convert_model_all_models(self, mode, name, link, mark, reason, ie_device):
         self.mode = mode
+        assert mark is None or mark in [
+            'skip', 'xfail', 'xfail_trace', 'xfail_export'], f"Incorrect test case for {name}"
+        if mark == 'skip':
+            pytest.skip(reason)
+        elif mark in ['xfail', f'xfail_{mode}']:
+            pytest.xfail(reason)
         self.run(name, None, ie_device)
 
     @pytest.mark.nightly

tests/model_hub_tests/pytorch/test_torchvision_models.py

@@ -7,8 +7,9 @@
 import pytest
 import torch
 import torchvision.transforms.functional as F
+from models_hub_common.utils import get_models_list
 
-from torch_utils import process_pytest_marks, TestTorchConvertModel
+from torch_utils import TestTorchConvertModel
 
 
 def get_all_models() -> list:
@@ -103,10 +104,15 @@ def test_convert_model_precommit_export(self, model_name, ie_device):
         self.mode = "export"
         self.run(model_name, None, ie_device)
 
-    @pytest.mark.parametrize("name",
-                             process_pytest_marks(os.path.join(os.path.dirname(__file__), "torchvision_models")))
+    @pytest.mark.parametrize("name,link,mark,reason", get_models_list(os.path.join(os.path.dirname(__file__), "torchvision_models")))
     @pytest.mark.parametrize("mode", ["trace", "export"])
     @pytest.mark.nightly
-    def test_convert_model_all_models(self, mode, name, ie_device):
+    def test_convert_model_all_models(self, mode, name, link, mark, reason,  ie_device):
         self.mode = mode
+        assert mark is None or mark in [
+            'skip', 'xfail', 'xfail_trace', 'xfail_export'], f"Incorrect test case for {name}"
+        if mark == 'skip':
+            pytest.skip(reason)
+        elif mark in ['xfail', f'xfail_{mode}']:
+            pytest.xfail(reason)
         self.run(name, None, ie_device)

tests/model_hub_tests/pytorch/timm_models

@@ -13,7 +13,7 @@ cait_s36_384.fb_dist_in1k,None
 cait_xs24_384.fb_dist_in1k,None
 cait_xxs24_224.fb_dist_in1k,None
 cait_xxs36_224.fb_dist_in1k,None
-coat_tiny.in1k,None
+coat_tiny.in1k,None,xfail_export,Requested None inlined input
 coatnet_bn_0_rw_224.sw_in1k,None
 coatnet_nano_rw_224.sw_in1k,None
 coatnet_rmlp_1_rw2_224.sw_in12k,None
@@ -23,7 +23,7 @@ convformer_b36.sail_in1k,None
 convformer_m36.sail_in1k,None
 convformer_s18.sail_in1k,None
 convformer_s36.sail_in1k,None
-convit_base.fb_in1k,None,xfail,Trace failed
+convit_base.fb_in1k,None,xfail_trace,Trace failed
 convmixer_1024_20_ks9_p14.in1k,None
 convmixer_1536_20.in1k,None
 convnext_atto_ols.a2_in1k,None
@@ -102,10 +102,10 @@ edgenext_xx_small.in1k,None
 efficientformer_l1.snap_dist_in1k,None
 efficientformer_l3.snap_dist_in1k,None
 efficientformer_l7.snap_dist_in1k,None
-efficientformerv2_l.snap_dist_in1k,None
-efficientformerv2_s0.snap_dist_in1k,None
-efficientformerv2_s1.snap_dist_in1k,None
-efficientformerv2_s2.snap_dist_in1k,None
+efficientformerv2_l.snap_dist_in1k,None,xfail_export,Requested None inlined input
+efficientformerv2_s0.snap_dist_in1k,None,xfail_export,Requested None inlined input
+efficientformerv2_s1.snap_dist_in1k,None,xfail_export,Requested None inlined input
+efficientformerv2_s2.snap_dist_in1k,None,xfail_export,Requested None inlined input
 efficientnet_b0.ra_in1k,None
 efficientnet_b1.ft_in1k,None
 efficientnet_b1_pruned.in1k,None
@@ -144,6 +144,9 @@ eva02_base_patch14_224.mim_in22k,None
 eva02_base_patch16_clip_224.merged2b,None
 eva02_large_patch14_clip_224.merged2b,None
 fastvit_ma36.apple_dist_in1k,None
+fastvit_mci0.apple_mclip,None
+fastvit_mci1.apple_mclip,None
+fastvit_mci2.apple_mclip,None,xfail_trace,Accuracy validation failed
 fastvit_s12.apple_dist_in1k,None
 fastvit_sa12.apple_dist_in1k,None
 fastvit_sa24.apple_dist_in1k,None
@@ -167,7 +170,7 @@ gcresnext50ts.ch_in1k,None
 gcvit_base.in1k,None
 gernet_s.idstcv_in1k,None
 ghostnet_100.in1k,None
-ghostnetv2_100.in1k,None
+ghostnetv2_100.in1k,None,xfail_export,Requested None inlined input
 gmixer_24_224.ra3_in1k,None
 gmlp_s16_224.ra3_in1k,None
 halo2botnet50ts_256.a1h_in1k,None
@@ -188,16 +191,18 @@ hgnetv2_b3.ssld_stage1_in22k_in1k,None
 hgnetv2_b4.ssld_stage1_in22k_in1k,None
 hgnetv2_b5.ssld_stage1_in22k_in1k,None
 hgnetv2_b6.ssld_stage1_in22k_in1k,None
-hrnet_w18_small.gluon_in1k,None
-hrnet_w18_small_v2.gluon_in1k,None
-hrnet_w18_ssld.paddle_in1k,None
-hrnet_w30.ms_in1k,None
-hrnet_w32.ms_in1k,None
-hrnet_w40.ms_in1k,None
-hrnet_w44.ms_in1k,None
-hrnet_w48.ms_in1k,None
-hrnet_w48_ssld.paddle_in1k,None
-hrnet_w64.ms_in1k,None
+hiera_base_224.mae,None
+hiera_base_plus_224.mae,None
+hrnet_w18_small.gluon_in1k,None,xfail_export,Requested None inlined input
+hrnet_w18_small_v2.gluon_in1k,None,xfail_export,Requested None inlined input
+hrnet_w18_ssld.paddle_in1k,None,xfail_export,Requested None inlined input
+hrnet_w30.ms_in1k,None,xfail_export,Requested None inlined input
+hrnet_w32.ms_in1k,None,xfail_export,Requested None inlined input
+hrnet_w40.ms_in1k,None,xfail_export,Requested None inlined input
+hrnet_w44.ms_in1k,None,xfail_export,Requested None inlined input
+hrnet_w48.ms_in1k,None,xfail_export,Requested None inlined input
+hrnet_w48_ssld.paddle_in1k,None,xfail_export,Requested None inlined input
+hrnet_w64.ms_in1k,None,xfail_export,Requested None inlined input
 inception_next_base.sail_in1k,None
 inception_resnet_v2.tf_ens_adv_in1k,None
 inception_v3.gluon_in1k,None
@@ -236,6 +241,9 @@ mobilenetv2_110d.ra_in1k,None
 mobilenetv2_120d.ra_in1k,None
 mobilenetv3_rw.rmsp_in1k,None
 mobilenetv3_small_050.lamb_in1k,None
+mobilenetv4_conv_blur_medium.e500_r224_in1k,None
+mobilenetv4_conv_small.e1200_r224_in1k,None
+mobilenetv4_hybrid_medium.e500_r224_in1k,None
 mobileone_s0.apple_in1k,None
 mobileone_s1.apple_in1k,None
 mobileone_s2.apple_in1k,None
@@ -324,6 +332,8 @@ resnest50d.in1k,None
 resnest50d_1s4x24d.in1k,None
 resnest50d_4s2x40d.in1k,None
 resnet101.a1_in1k,None
+resnet101_clip.openai,None
+resnet101_clip_gap.openai,None
 resnet101c.gluon_in1k,None
 resnet101d.gluon_in1k,None
 resnet101s.gluon_in1k,None
@@ -344,10 +354,18 @@ resnet33ts.ra2_in1k,None
 resnet34.a1_in1k,None
 resnet34d.ra2_in1k,None
 resnet50.a1_in1k,None
+resnet50_clip.openai,None
+resnet50_clip_gap.openai,None
 resnet50_gn.a1h_in1k,None
 resnet50c.gluon_in1k,None
 resnet50d.a1_in1k,None
 resnet50s.gluon_in1k,None
+resnet50x16_clip.openai,None
+resnet50x16_clip_gap.openai,None
+resnet50x4_clip.openai,None
+resnet50x4_clip_gap.openai,None
+resnet50x64_clip.openai,None
+resnet50x64_clip_gap.openai,None
 resnet51q.ra2_in1k,None
 resnet61q.ra2_in1k,None
 resnetaa101d.sw_in12k,None
@@ -388,7 +406,7 @@ selecsls60.in1k,None
 selecsls60b.in1k,None
 semnasnet_075.rmsp_in1k,None
 senet154.gluon_in1k,None
-sequencer2d_s.in1k,None
+sequencer2d_s.in1k,None,xfail_export,No conversion rule found for operations aten.mkldnn_rnn_layer.default
 seresnet152d.ra2_in1k,None
 seresnet33ts.ra2_in1k,None
 seresnet50.a1_in1k,None
@@ -453,7 +471,7 @@ tinynet_b.in1k,None
 tinynet_c.in1k,None
 tinynet_d.in1k,None
 tinynet_e.in1k,None
-tnt_s_patch16_224,None
+tnt_s_patch16_224,None,xfail_export,Requested None inlined input
 tresnet_m.miil_in1k,None
 tresnet_v2_l.miil_in21k,None
 twins_pcpvt_base.in1k,None
@@ -467,39 +485,58 @@ vgg16_bn.tv_in1k,None
 vgg19.tv_in1k,None
 vgg19_bn.tv_in1k,None
 visformer_tiny.in1k,None
+vit_base_mci_224.apple_mclip,None
 vit_base_patch14_dinov2.lvd142m,None
 vit_base_patch14_reg4_dinov2.lvd142m,None
 vit_base_patch16_224.augreg2_in21k_ft_in1k,None
 vit_base_patch16_224_miil.in21k,None
 vit_base_patch16_clip_224.datacompxl,None
 vit_base_patch16_clip_quickgelu_224.metaclip_2pt5b,None
+vit_base_patch16_rope_reg1_gap_256.sbb_in1k,None,xfail,Argument shapes are inconsistent
 vit_base_patch16_rpn_224.sw_in1k,None
 vit_base_patch16_siglip_224.webli,None
+vit_base_patch16_siglip_gap_224.webli,None
 vit_base_patch32_224.augreg_in1k,None
 vit_base_patch32_clip_224.datacompxl,None
 vit_base_patch32_clip_quickgelu_224.metaclip_2pt5b,None
 vit_base_patch8_224.augreg2_in21k_ft_in1k,None
 vit_base_r50_s16_224.orig_in21k,None
+vit_betwixt_patch16_reg1_gap_256.sbb_in1k,None
+vit_betwixt_patch16_reg4_gap_256.sbb_in12k,None
+vit_betwixt_patch16_rope_reg4_gap_256.sbb_in1k,None,xfail,Argument shapes are inconsistent
+vit_betwixt_patch32_clip_224.tinyclip_laion400m,None
 vit_huge_patch14_224.mae,None
 vit_huge_patch14_gap_224.in1k_ijepa,None
 vit_large_patch14_clip_224.datacompxl,None
 vit_large_patch14_clip_quickgelu_224.dfn2b,None
 vit_large_r50_s32_224.augreg_in21k,None
+vit_little_patch16_reg1_gap_256.sbb_in12k,None
+vit_little_patch16_reg4_gap_256.sbb_in1k,None
 vit_medium_patch16_gap_240.sw_in12k,None
+vit_medium_patch16_reg1_gap_256.sbb_in1k,None
+vit_medium_patch16_reg4_gap_256.sbb_in12k,None
+vit_mediumd_patch16_reg4_gap_256.sbb_in12k,None
+vit_mediumd_patch16_rope_reg1_gap_256.sbb_in1k,None,xfail,Argument shapes are inconsistent
+vit_pwee_patch16_reg1_gap_256.sbb_in1k,None
 vit_relpos_base_patch16_224.sw_in1k,None
 vit_relpos_base_patch16_clsgap_224.sw_in1k,None
 vit_relpos_base_patch32_plus_rpn_256.sw_in1k,None
 vit_relpos_medium_patch16_cls_224.sw_in1k,None
 vit_relpos_medium_patch16_rpn_224.sw_in1k,None
 vit_small_r26_s32_224.augreg_in21k,None
 vit_so400m_patch14_siglip_224.webli,None
+vit_so400m_patch14_siglip_gap_224.pali_mix,None,skip,Access to model google/paligemma-3b-mix-224-jax is restricted
 vit_srelpos_small_patch16_224.sw_in1k,None
 vit_tiny_r_s16_p8_224.augreg_in21k,None
-volo_d1_224.sail_in1k,None
-volo_d2_224.sail_in1k,None
-volo_d3_224.sail_in1k,None
-volo_d4_224.sail_in1k,None
-volo_d5_224.sail_in1k,None
+vit_wee_patch16_reg1_gap_256.sbb_in1k,None
+vit_xsmall_patch16_clip_224.tinyclip_yfcc15m,None
+vitamin_base_224.datacomp1b_clip,None,xfail,RuntimeError Error in loading state_dict for VisionTransformer
+vitamin_large2_224.datacomp1b_clip,None
+volo_d1_224.sail_in1k,None,xfail,Cannot get length of dynamic dimension
+volo_d2_224.sail_in1k,None,xfail,Cannot get length of dynamic dimension
+volo_d3_224.sail_in1k,None,xfail,Cannot get length of dynamic dimension
+volo_d4_224.sail_in1k,None,xfail,Cannot get length of dynamic dimension
+volo_d5_224.sail_in1k,None,xfail,Cannot get length of dynamic dimension
 wide_resnet101_2.tv2_in1k,None
 wide_resnet50_2.racm_in1k,None
 xception41.tf_in1k,None