Handling of skip category of tests (#2597)

Summary:
many test got skipped either due to metadat.yaml or any other reason but at the end showing result passed instead of skipped
so these change will help to distinguish genuine skipped vs real passed models

Pull Request resolved: #2597

Reviewed By: xuzhao9

Differential Revision: D73366712

Pulled By: atalman

fbshipit-source-id: b7b346cd067d8c63f83f864dd315bc2b278cc824

Author: arathi-hlab

test_bench.py

@@ -72,7 +72,9 @@ def test_train(self, model_path, device, benchmark):
                 extra_args=[],
                 metadata=get_metadata_from_yaml(model_path),
             ):
-                raise NotImplementedError("Test skipped by its metadata.")
+                pytest.skip(
+                    f"Test train on {device} is skipped by its metadata skipping..."
+                )
             # TODO: skipping quantized tests for now due to BC-breaking changes for prepare
             # api, enable after PyTorch 1.13 release
             if "quantized" in model_name:
@@ -90,8 +92,11 @@ def test_train(self, model_path, device, benchmark):
             )
             benchmark.extra_info["test"] = "train"
 
-        except NotImplementedError:
-            print(f"Test train on {device} is not implemented, skipping...")
+        except Exception as e:
+            if isinstance(e, NotImplementedError):
+                print(f"Test train on {device} is not implemented")
+            else:
+                pytest.fail(f"Test train failed on {device} due to this error: {e}")
 
     def test_eval(self, model_path, device, benchmark, pytestconfig):
         try:
@@ -102,7 +107,9 @@ def test_eval(self, model_path, device, benchmark, pytestconfig):
                 extra_args=[],
                 metadata=get_metadata_from_yaml(model_path),
             ):
-                raise NotImplementedError("Test skipped by its metadata.")
+                pytest.skip(
+                    f"Test eval on {device} is skipped by its metadata skipping..."
+                )
             # TODO: skipping quantized tests for now due to BC-breaking changes for prepare
             # api, enable after PyTorch 1.13 release
             if "quantized" in model_name:
@@ -121,8 +128,11 @@ def test_eval(self, model_path, device, benchmark, pytestconfig):
             )
             benchmark.extra_info["test"] = "eval"
 
-        except NotImplementedError:
-            print(f"Test eval on {device} is not implemented, skipping...")
+        except Exception as e:
+            if isinstance(e, NotImplementedError):
+                print(f"Test eval on {device} is not implemented")
+            else:
+                pytest.fail(f"Test eval failed on {device} due to this error: {e}")
 
 
 @pytest.mark.benchmark(