assert returned value against expected result

cle-b · cle-b · commit 01418c0c4a0d · 2024-10-06T17:24:28.000+02:00
diff --git a/README.md b/README.md
@@ -1,8 +1,8 @@
 # lymbo
 
-`lymbo` is a test runner designed for large test suites.
+* **lymbo** is a test runner designed for large test suites and small scripts.
 
-The key features of `lymbo` are:
+The key features of **lymbo** are:
 
   * Parallel execution by default
   * Simplicity
@@ -11,7 +11,7 @@ The key features of `lymbo` are:
 
 ## Concetps
 
-In `lymbo`, there are only two key concepts to understand: `test` and `resource`.
+In **lymbo**, there are only two key concepts to understand: `test` and `resource`.
 
 ### Test
 
@@ -100,6 +100,31 @@ We can group the tests to ensure they run sequentially on the same worker. In th
 ==== 5 tests in 2 groups
 ``` 
 
+For a very simple unit test, you can decorate the function you want to test and verify the returned value to assert the test outcome.
+
+```python
+@lymbo.test(args(a=4, b=2), expected(2))
+@lymbo.test(args(a=9, b=2), expected=expected(4.5))
+@lymbo.test(args(a=9, b=0), expected=expected(ZeroDivisionError))
+def division(a, b):
+    return a / b
+```
+
+```
+(venv) ~/dev/lymbo$ lymbo examples/readme.py --filter=division
+** lymbo 0.3.0 (python 3.9.18) (Linux-5.15.0-122-generic-x86_64-with-glibc2.35) **
+==== collecting tests
+==== 3 tests in 3 groups
+==== running tests
+.P.P.P
+==== tests executed in 0 second
+==== results
+- examples/readme.py::division(a=4,b=2)->(value=2) [PASSED]
+- examples/readme.py::division(a=9,b=2)->(value=4.5) [PASSED]
+- examples/readme.py::division(a=9,b=0)->(value=ZeroDivisionError) [PASSED]
+==== 3 passed 
+```
+
 ### Resource
 
 A resource is a standard context manager.
@@ -216,7 +241,7 @@ usage: lymbo [-h] [--version] [--collect] [--groupby {GroupBy.NONE,GroupBy.MODUL
              [--filter FILTER]
              [PATH ...]
 
-A test runner designed for large test suites.
+A test runner designed for large test suites and small scripts.
 
 positional arguments:
   PATH                  Path(s) for test collection
diff --git a/examples/readme.py b/examples/readme.py
@@ -3,7 +3,7 @@
 import time
 
 import lymbo
-from lymbo import args, expand
+from lymbo import args, expand, expected
 from lymbo import scope_global
 
 
@@ -12,6 +12,13 @@ def addition():
     assert 1 + 2 == 3, "Addition test failed: 1 + 2 did not equal 3"
 
 
+@lymbo.test(args(a=4, b=2), expected(2))
+@lymbo.test(args(a=9, b=2), expected=expected(4.5))
+@lymbo.test(args(a=9, b=0), expected=expected(ZeroDivisionError))
+def division(a, b):
+    return a / b
+
+
 @lymbo.test(args(n=expand(1, 4, 9, 116)))
 def is_perfect_square(n):
     assert (
diff --git a/lymbo/__init__.py b/lymbo/__init__.py
@@ -4,17 +4,19 @@
 
 from lymbo.cm import args
 from lymbo.cm import expand
+from lymbo.cm import expected
 from lymbo.cm import test
 from lymbo.resource_manager import scope_class
 from lymbo.resource_manager import scope_function
 from lymbo.resource_manager import scope_global
 from lymbo.resource_manager import scope_module
 
-__version__ = "0.2.2"
+__version__ = "0.3.0"
 
 __all__ = [
     "args",
     "expand",
+    "expected",
     "test",
     "scope_class",
     "scope_function",
diff --git a/lymbo/cm.py b/lymbo/cm.py
@@ -1,11 +1,17 @@
 import contextlib
+from dataclasses import dataclass
 import os
+import re
+from typing import Any
+from typing import Type
+from typing import Optional
+from typing import Union
 
 from lymbo.env import LYMBO_TEST_COLLECTION
 
 
 @contextlib.contextmanager
-def test(args=None):
+def test(args=None, expected=None):
     yield
 
 
@@ -49,6 +55,47 @@ def args(*args, **kwargs):
         return flattened_params
 
 
+@dataclass
+class ExpectedAssertion:
+    value: Union[Type[Any], Any] = None
+    match: Union[str, None] = None
+
+    def assert_(self, returned_value: Any) -> Optional[str]:
+        """
+        Compare the value returned by a function with its expected value.
+
+        Return a description of the failure if they don't match, or None if they matched.
+        """
+        failure: Optional[str] = None
+
+        if self.value:
+            if isinstance(self.value, type):
+                if type(returned_value) is not self.value:
+                    failure = f"Expected type {self.value.__name__}, but got type {type(returned_value).__name__}."
+            else:
+                if returned_value != self.value:
+                    failure = f"Expected value {self.value}, but got {returned_value}."
+
+        if not failure:
+            if self.match:
+                if not re.match(self.match, str(returned_value)):
+                    failure = f"Value '{returned_value}' does not match the expected pattern '{self.match}'."
+
+        return failure
+
+
+def expected(
+    value: Union[Type[Any], Any, None] = None, match: Union[str, None] = None
+) -> ExpectedAssertion:
+    """
+    Define what we expect the function to return.
+
+    - value: The expected type or object value. Can be None.
+    - match: A regular expression to match. Can be None.
+    """
+    return ExpectedAssertion(value, match)
+
+
 class ArgParams:
 
     def __init__(self, *args):
diff --git a/lymbo/collect.py b/lymbo/collect.py
@@ -133,13 +133,18 @@ def parse_body(
                             if getattr(decorator.func.value, "id", None) == "lymbo":
 
                                 args_call = None
+                                expected = None
 
                                 if decorator.args:
                                     args_call = decorator.args[0]
-                                else:
-                                    for kw in decorator.keywords:
-                                        if kw.arg == "args":
-                                            args_call = kw.value
+                                    if len(decorator.args) == 2:
+                                        expected = decorator.args[1]
+
+                                for kw in decorator.keywords:
+                                    if kw.arg == "args":
+                                        args_call = kw.value
+                                    if kw.arg == "expected":
+                                        expected = kw.value
 
                                 if args_call:
                                     flattened_args = eval_ast_call(
@@ -148,6 +153,12 @@ def parse_body(
                                 else:
                                     flattened_args = args()
 
+                                expected_assertion = None
+                                if expected:
+                                    expected_assertion = eval_ast_call(
+                                        expected, global_vars, local_vars
+                                    )
+
                                 tests = []
                                 for f_args in flattened_args:
                                     tests.append(
@@ -158,6 +169,7 @@ def parse_body(
                                                 item.name,
                                                 f_args,
                                                 classdef.name if classdef else None,
+                                                expected_assertion,
                                             ),
                                         ]
                                     )
diff --git a/lymbo/config.py b/lymbo/config.py
@@ -9,7 +9,7 @@
 def parse_args() -> argparse.Namespace:
 
     parser = argparse.ArgumentParser(
-        description="A test runner designed for large test suites."
+        description="A test runner designed for large test suites and small scripts."
     )
 
     parser.add_argument(
diff --git a/lymbo/item.py b/lymbo/item.py
@@ -16,6 +16,7 @@
 from typing import Union
 
 import lymbo
+from lymbo.cm import ExpectedAssertion
 from lymbo.env import LYMBO_REPORT_PATH
 from lymbo.env import LYMBO_TEST_SCOPE_CLASS
 from lymbo.env import LYMBO_TEST_SCOPE_FUNCTION
@@ -58,12 +59,14 @@ def __init__(
         fnc: str,
         parameters: tuple[tuple[Any], dict[str, Any]],
         cls: Optional[str],
+        expected: Optional[ExpectedAssertion],
     ):
         self.path = path
         self.asynchronous = asynchronous
         self.fnc = fnc
         self.parameters = parameters
         self.cls = cls
+        self.expected: ExpectedAssertion = expected if expected else ExpectedAssertion()
 
         md5 = hashlib.md5(str(self).encode()).hexdigest()
         timestamp = int(time.time() * 1000000)
@@ -101,7 +104,21 @@ def print_variable(variable):
             call.append(f"{k}={print_variable(v)}")
         s += ",".join(call)
         s += ")"
-
+        if self.expected.value or self.expected.match:
+            s += "->("
+            if self.expected.value:
+                s += "value="
+                if type(self.expected.value) is str:
+                    s += f'"{self.expected.value}"'
+                elif isinstance(self.expected.value, type):
+                    s += f"{self.expected.value.__name__}"
+                else:
+                    s += f"{self.expected.value}"
+            if self.expected.value and self.expected.match:
+                s += ", "
+            if self.expected.match:
+                s += f"match={self.expected.match}"
+            s += ")"
         return s
 
     def __repr__(self) -> str:
diff --git a/lymbo/run.py b/lymbo/run.py
@@ -161,14 +161,32 @@ def run_test(test_item: TestItem):
         try:
             test_item.start()
             if asynchronous:
-                asyncio.run(test_function(*args, **kwargs))
+                returned_value = asyncio.run(test_function(*args, **kwargs))
             else:
-                test_function(*args, **kwargs)
-            test_item.end()
-            print(f"{color.GREEN}P{color.RESET}", end="", flush=True)
+                returned_value = test_function(*args, **kwargs)
+            failure = test_item.expected.assert_(returned_value)
+            if failure:
+                ex = AssertionError(failure)
+                test_item.end(reason=ex)
+                print(f"{color.RED}F{color.RESET}", end="", flush=True)
+            else:
+                test_item.end()
+                print(f"{color.GREEN}P{color.RESET}", end="", flush=True)
         except AssertionError as ex:
             test_item.end(reason=ex)
             print(f"{color.RED}F{color.RESET}", end="", flush=True)
         except Exception as ex:
-            test_item.end(reason=ex)
-            print(f"{color.YELLOW}B{color.RESET}", end="", flush=True)
+            if isinstance(test_item.expected.value, type) and issubclass(
+                test_item.expected.value, BaseException
+            ):
+                failure = test_item.expected.assert_(ex)
+                if failure:
+                    ex = AssertionError(failure)
+                    test_item.end(reason=ex)
+                    print(f"{color.RED}F{color.RESET}", end="", flush=True)
+                else:
+                    test_item.end()
+                    print(f"{color.GREEN}P{color.RESET}", end="", flush=True)
+            else:
+                test_item.end(reason=ex)
+                print(f"{color.YELLOW}B{color.RESET}", end="", flush=True)
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ name = "lymbo"
 authors = [
     {name = "cle-b", email = "cle@tictac.pm"},
 ]
-description="A test runner designed for large test suites."
+description="A test runner designed for large test suites and small scripts."
 readme="README.md"
 requires-python = ">=3.9"
 license = {text = "Apache-2.0"}
diff --git a/tests/data_run/run_expected.py b/tests/data_run/run_expected.py
@@ -0,0 +1,58 @@
+import lymbo
+from lymbo import args
+from lymbo import expected
+
+
+@lymbo.test(args(a=4, b=2), expected(2))
+def value_passed(a, b):
+    return a / b
+
+
+@lymbo.test(args(a=4, b=2), expected(1))
+def value_failed(a, b):
+    return a / b
+
+
+@lymbo.test(args(a=4, b=2), expected(1))
+def value_broken(a, b):
+    raise NameError("boum")
+
+
+@lymbo.test(args(a=4, b=2), expected(float))
+def type_passed(a, b):
+    return a / b
+
+
+@lymbo.test(args(a=4, b=2), expected(str))
+def type_failed(a, b):
+    return a / b
+
+
+@lymbo.test(args(a=4, b=2), expected(int))
+def type_broken(a, b):
+    raise NameError("boum")
+
+
+@lymbo.test(args(a=4, b=0), expected(ZeroDivisionError))
+def exception_passed(a, b):
+    return a / b
+
+
+@lymbo.test(args(a=4, b=0), expected(NameError))
+def exception_failed(a, b):
+    return a / b
+
+
+@lymbo.test(args(name="cle"), expected(match="H.* cle.*"))
+def match_passed(name):
+    return "Hi cle!"
+
+
+@lymbo.test(args(name="cle"), expected(match="H.* cle.*"))
+def match_failed(name):
+    return "Bonjour cle!"
+
+
+@lymbo.test(args(name="cle"), expected(match="H.* cle.*"))
+def match_broken(name):
+    raise NameError("boum")
diff --git a/tests/test_run.py b/tests/test_run.py

Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@`
`9`	`9`	`def parse_args() -> argparse.Namespace:`
`10`	`10`
`11`	`11`	`parser = argparse.ArgumentParser(`
`12`		`- description="A test runner designed for large test suites."`
	`12`	`+ description="A test runner designed for large test suites and small scripts."`
`13`	`13`	`)`
`14`	`14`
`15`	`15`	`parser.add_argument(`
Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,7 @@ name = "lymbo"`
`7`	`7`	`authors = [`
`8`	`8`	`{name = "cle-b", email = "cle@tictac.pm"},`
`9`	`9`	`]`
`10`		`-description="A test runner designed for large test suites."`
	`10`	`+description="A test runner designed for large test suites and small scripts."`
`11`	`11`	`readme="README.md"`
`12`	`12`	`requires-python = ">=3.9"`
`13`	`13`	`license = {text = "Apache-2.0"}`