✨ Events (#99)

* ✨ Events

* Fix lint

* 📚 Add Event documentation

- Add sample output

Author: roniemartinez

.gitignore

@@ -138,3 +138,5 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
+# Draw.io
+*.drawio

README.md

@@ -65,6 +65,37 @@ You can run your scraper from terminal/shell/command-line by supplying URLs, the
 dude scrape --url "<url>" --output data.json path/to/script.py
 ```
 
+The output in `data.json` should contain the actual URL and the metadata prepended with underscore.
+
+```json5
+[
+  {
+    "_page_number": 1,
+    "_page_url": "https://dude.ron.sh/",
+    "_group_id": 4502003824,
+    "_group_index": 0,
+    "_element_index": 0,
+    "url": "/url-1.html"
+  },
+  {
+    "_page_number": 1,
+    "_page_url": "https://dude.ron.sh/",
+    "_group_id": 4502003824,
+    "_group_index": 0,
+    "_element_index": 1,
+    "url": "/url-2.html"
+  },
+  {
+    "_page_number": 1,
+    "_page_url": "https://dude.ron.sh/",
+    "_group_id": 4502003824,
+    "_group_index": 0,
+    "_element_index": 2,
+    "url": "/url-3.html"
+  }
+]
+```
+
 ## Features
 
 - Simple [Flask](https://github.com/pallets/flask)-inspired design - build a scraper with decorators.

docs/advanced/14_events.md

@@ -0,0 +1,69 @@
+# Events
+
+Functions can be registered to be called on specific events which makes it possible to run custom actions like setting 
+up databases or calling API for authentication, and performing additional actions on page objects
+(can be soup, driver, selector or tree objects) like taking screenshots.
+
+Here is a diagram when events are being executed.
+
+![Events](../diagrams/events.png)
+
+## Startup Event
+
+The Startup event is executed at the start of the process. 
+
+The `@startup()` decorator can be used to register a function for startup. 
+This can be used to setup databases or authenticate to APIs and other possible use-cases prior to actual web scraping.
+
+```python
+from pathlib import Path
+
+from dude import startup
+
+SAVE_DIR: Path
+
+
+@startup()
+def initialize_csv():
+    global SAVE_DIR
+    SAVE_DIR = Path(__file__).resolve().parent / "temp"
+    SAVE_DIR.mkdir(exist_ok=True)
+```
+
+## Pre-Setup Event
+
+The Pre-Setup event is executed after loading a page or getting an HTTP response.
+
+The `@pre_setup()` decorator can be used to register a function for pre-setup. 
+Note that the function should accept one argument which can either be a page, driver, soup, Parsel selector or LXML tree.
+
+```python
+import uuid
+from dude import pre_setup
+
+...
+
+@pre_setup()
+def screenshot(page):
+    unique_name = str(uuid.uuid4())
+    page.screenshot(path=SAVE_DIR / f"{unique_name}.png")
+```
+
+## Post-Setup Event
+
+The Post-Setup event is executed after running the [setup functions](https://roniemartinez.github.io/dude/advanced/01_setup.html).
+
+The `@post_setup()` decorator can be used to register a function for post-setup. 
+Note that the function should accept one argument which can either be a page, driver, soup, Parsel selector or LXML tree.
+
+```python
+import uuid
+from dude import post_setup
+
+...
+
+@post_setup()
+def print_pdf(page):
+    unique_name = str(uuid.uuid4())
+    page.pdf(path=SAVE_DIR / f"{unique_name}.pdf")
+```

docs/diagrams/events.png

@@ -65,6 +65,37 @@ To start scraping, use any of the following options. Click on the annotations (+
 
     1. You can also use **dude.run()** function and run **python path/to/script.py** from terminal.
 
+The output in `data.json` should contain the actual URL and the metadata prepended with underscore.
+
+```json5
+[
+  {
+    "_page_number": 1,
+    "_page_url": "https://dude.ron.sh/",
+    "_group_id": 4502003824,
+    "_group_index": 0,
+    "_element_index": 0,
+    "url": "/url-1.html"
+  },
+  {
+    "_page_number": 1,
+    "_page_url": "https://dude.ron.sh/",
+    "_group_id": 4502003824,
+    "_group_index": 0,
+    "_element_index": 1,
+    "url": "/url-2.html"
+  },
+  {
+    "_page_number": 1,
+    "_page_url": "https://dude.ron.sh/",
+    "_group_id": 4502003824,
+    "_group_index": 0,
+    "_element_index": 2,
+    "url": "/url-3.html"
+  }
+]
+```
+
 ## License
 
 This project is licensed under the terms of the GNU AGPLv3+ license.

docs/index.md

@@ -3,10 +3,10 @@
 from pathlib import Path
 from typing import Any
 
-from .context import group, run, save, select  # noqa: F401
+from .context import group, post_setup, pre_setup, run, save, select, startup  # noqa: F401
 from .scraper import Scraper  # noqa: F401
 
-__al__ = ["Scraper", "group", "run", "save", "select"]
+__al__ = ["Scraper", "group", "run", "save", "select", "startup", "pre_setup", "post_setup"]
 
 
 logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")

dude/__init__.py

@@ -10,6 +10,7 @@
     AsyncIterable,
     Callable,
     Coroutine,
+    DefaultDict,
     Deque,
     Dict,
     Iterable,
@@ -43,12 +44,14 @@ def __init__(
         rules: List[Rule] = None,
         groups: Dict[Callable, Selector] = None,
         save_rules: Dict[str, Any] = None,
+        events: Optional[DefaultDict] = None,
         has_async: bool = False,
         scraper: Optional["ScraperAbstract"] = None,
     ) -> None:
         self.rules: List[Rule] = rules or []
         self.groups: Dict[Callable, Selector] = groups or {}
         self.save_rules: Dict[str, Any] = save_rules or {"json": save_json}
+        self.events: DefaultDict = events or collections.defaultdict(list)
         self.has_async = has_async
         self.scraper = scraper
         self.adblock = Adblocker()
@@ -129,10 +132,8 @@ def wrapper(func: Callable) -> Union[Callable, Coroutine]:
                 navigate=navigate,
                 priority=priority,
             )
-            if not self.scraper:
-                self.rules.append(rule)
-            else:
-                self.scraper.rules.append(rule)
+            rules = self.scraper.rules if self.scraper else self.rules
+            rules.append(rule)
             return func
 
         return wrapper
@@ -198,10 +199,61 @@ def wrapper(func: Callable) -> Callable:
             if asyncio.iscoroutinefunction(func):
                 self.has_async = True
 
-            if not self.scraper:
-                self.save_rules[format] = func
-            else:
-                self.scraper.save_rules[format] = func
+            save_rules = self.scraper.save_rules if self.scraper else self.save_rules
+            save_rules[format] = func
+            return func
+
+        return wrapper
+
+    def startup(self) -> Callable:
+        """
+        Decorator to register a function to startup events.
+
+        Startup events are executed before any actual scraping happens to, for example, setup databases, etc.
+        """
+
+        def wrapper(func: Callable) -> Callable:
+            if asyncio.iscoroutinefunction(func):
+                self.has_async = True
+
+            events = self.scraper.events if self.scraper else self.events
+            events["startup"].append(func)
+            return func
+
+        return wrapper
+
+    def pre_setup(self) -> Callable:
+        """
+        Decorator to register a function to pre-setup events.
+
+        Pre-setup events are executed after a page is loaded (or HTTP response in case of HTTPX) and
+        before running the setup functions.
+        """
+
+        def wrapper(func: Callable) -> Callable:
+            if asyncio.iscoroutinefunction(func):
+                self.has_async = True
+
+            events = self.scraper.events if self.scraper else self.events
+            events["pre-setup"].append(func)
+            return func
+
+        return wrapper
+
+    def post_setup(self) -> Callable:
+        """
+        Decorator to register a function to post-setup events.
+
+        Post-setup events are executed after running the setup functions and before the actual web-scraping happens.
+        This is useful when "page clean-ups" are done in the setup functions.
+        """
+
+        def wrapper(func: Callable) -> Callable:
+            if asyncio.iscoroutinefunction(func):
+                self.has_async = True
+
+            events = self.scraper.events if self.scraper else self.events
+            events["post-setup"].append(func)
             return func
 
         return wrapper
@@ -220,16 +272,47 @@ def iter_urls(self) -> Iterable[str]:
         except IndexError:
             pass
 
+    def _update_rule_groups(self) -> Iterable[Rule]:
+        for rule in self.rules:
+            if rule.group:
+                yield rule
+            elif rule.handler in self.groups:
+                yield rule._replace(group=self.groups[rule.handler])
+            else:
+                yield rule._replace(group=Selector(selector=":root"))
+
+    def initialize_scraper(self, urls: Sequence[str]) -> None:
+        self.rules = [rule for rule in self._update_rule_groups()]
+        self.urls = collections.deque(urls)
+        self.allowed_domains = {urlparse(url).netloc for url in urls}
+        self.event_startup()
+
+    def event_startup(self) -> None:
+        """
+        Run all startup events
+        """
+        loop = None
+        if self.has_async:
+            loop = asyncio.get_event_loop()
+
+        for func in self.events["startup"]:
+            if asyncio.iscoroutinefunction(func):
+                assert loop is not None
+                loop.run_until_complete(func())
+            else:
+                func()
+
 
 class ScraperAbstract(ScraperBase):
     def __init__(
         self,
         rules: List[Rule] = None,
         groups: Dict[Callable, Selector] = None,
         save_rules: Dict[str, Any] = None,
+        events: Optional[DefaultDict] = None,
         has_async: bool = False,
     ) -> None:
-        super(ScraperAbstract, self).__init__(rules, groups, save_rules, has_async)
+        super(ScraperAbstract, self).__init__(rules, groups, save_rules, events, has_async)
         self.collected_data: List[ScrapedData] = []
 
     @abstractmethod
@@ -248,18 +331,6 @@ def navigate(self) -> bool:
     async def navigate_async(self) -> bool:
         raise NotImplementedError  # pragma: no cover
 
-    def update_rule_groups(self) -> None:
-        self.rules = [rule for rule in self._update_rule_groups()]
-
-    def _update_rule_groups(self) -> Iterable[Rule]:
-        for rule in self.rules:
-            if rule.group:
-                yield rule
-            elif rule.handler in self.groups:
-                yield rule._replace(group=self.groups[rule.handler])
-            else:
-                yield rule._replace(group=Selector(selector=":root"))
-
     @abstractmethod
     def collect_elements(self) -> Iterable[Tuple[str, int, int, int, Any, Callable]]:
         """
@@ -276,6 +347,34 @@ async def collect_elements_async(self) -> AsyncIterable[Tuple[str, int, int, int
         yield "", 0, 0, 0, 0, str  # HACK: mypy does not identify this as AsyncIterable  # pragma: no cover
         raise NotImplementedError  # pragma: no cover
 
+    def event_pre_setup(self, *args: Any) -> None:
+        """
+        Run all pre-setup events.
+        """
+        for func in self.events["pre-setup"]:
+            func(*args)
+
+    def event_post_setup(self, *args: Any) -> None:
+        """
+        Run all post-setup events.
+        """
+        for func in self.events["post-setup"]:
+            func(*args)
+
+    async def event_pre_setup_async(self, *args: Any) -> None:
+        """
+        Run all pre-setup events.
+        """
+        for func in self.events["pre-setup"]:
+            await func(*args)
+
+    async def event_post_setup_async(self, *args: Any) -> None:
+        """
+        Run all post-setup events.
+        """
+        for func in self.events["post-setup"]:
+            await func(*args)
+
     def extract_all(self, page_number: int, **kwargs: Any) -> Iterable[ScrapedData]:
         """
         Extracts all the data using the registered handler functions.

dude/base.py

@@ -10,6 +10,9 @@
 run = _scraper.run
 save = _scraper.save
 select = _scraper.select
+startup = _scraper.startup
+pre_setup = _scraper.pre_setup
+post_setup = _scraper.post_setup
 
 
 """