crawl_manager: default errback to None

Author: pawelmhm

docs/source/api.rst

@@ -97,14 +97,20 @@ callback
     - optional
 
     Must exist as method of scheduled spider, does not need to contain string "self".
-    If not passed or not found on spider default callback `parse`_ will be used.
+    If not passed default Scrapy callback `parse`_ will be used. If there is no spider method
+    with name specified by callback argument or callback is not callable API will return 400 HTTP error.
+
+    Example request with callback: ``/crawl.json?url=https://quotes.toscrape.com/&spider_name=toscrape-css&callback=parse_page``
 
 errback
     - type: string
     - optional
 
     Scrapy errback for request made from spider. It must exist as method of
-    scheduled spider, otherwise exception will be raised. String does not need to contain 'self'.
+    scheduled spider, otherwise API will return 400 HTTP error. String does not need to contain 'self'.
+    Defaults to None, can be adjusted with `DEFAULT_ERRBACK_NAME`_ setting.
+
+    Example request with errback: ``/crawl.json?url=https://quotes.toscrape.com/&spider_name=toscrape-css&errback=my_errback``
 
 max_requests
     - type: integer
@@ -520,11 +526,12 @@ Default: ``utf-8``.
 DEFAULT_ERRBACK_NAME
 ~~~~~~~~~~~~~~~~~~~~
 
-Default: ``"parse"``
+Default: ``None``
 
-The name of the default errback_.
+String with the name of the default errback_.
 
-Use an empty string or ``None`` to unset the errback altogether. 
+Use this setting to set default errback for scrapy spider requests made from ScrapyRT.
+Errback must exist as method of spider and must be callable, otherwise 400 HTTP error will be raised.
 
 .. _errback: https://docs.scrapy.org/en/latest/topics/request-response.htm#using-errbacks-to-catch-exceptions-in-request-processing
 

scrapyrt/conf/default_settings.py

@@ -33,4 +33,4 @@
 
 TWISTED_REACTOR = None
 
-DEFAULT_ERRBACK_NAME = 'parse'
+DEFAULT_ERRBACK_NAME = None

scrapyrt/core.py

@@ -177,22 +177,29 @@ def spider_idle(self, spider):
                 callback = getattr(self.crawler.spider, self.callback_name)
                 assert callable(callback), 'Invalid callback'
                 self.request = self.request.replace(callback=callback)
-
-
+            except (AssertionError, AttributeError):
+                msg = f"Invalid spider callback {self.callback_name}, callback not callable or not a method of a spider {self.spider_name}"
+                self.user_error = Error(400, message=msg)
+            try:
                 if self.errback_name:
                     errback = getattr(self.crawler.spider, self.errback_name)
                     assert callable(errback), 'Invalid errback'
                     self.request = self.request.replace(errback=errback)
-                modify_request = getattr(
-                    self.crawler.spider, "modify_realtime_request", None)
-                if callable(modify_request):
-                    self.request = modify_request(self.request)
-                spider.crawler.engine.crawl(self.request)
-                self._request_scheduled = True
-            except Exception as e:
-                self.user_error = Error(400, message=traceback.format_exc())
-            else:
-                raise DontCloseSpider
+            except (AssertionError, AttributeError):
+                msg = f"Invalid spider errback {self.errback_name}, errback not callable or not a method of a spider {self.spider_name}"
+                self.user_error = Error(400, message=msg)
+            if self.user_error:
+                log.msg(self.user_error.message, level=log.ERROR)
+                return
+
+            modify_request = getattr(
+                self.crawler.spider, "modify_realtime_request", None)
+            if callable(modify_request):
+                self.request = modify_request(self.request)
+
+            spider.crawler.engine.crawl(self.request)
+            self._request_scheduled = True
+            raise DontCloseSpider
 
     def handle_scheduling(self, request, spider):
         """Handler of request_scheduled signal.

tests/test_crawl_manager.py

@@ -113,7 +113,7 @@ def test_raise_error_if_not_callable(self):
         self.spider.parse_something = None
         self._call_spider_idle()
         self.assertIsNotNone(self.crawl_manager.user_error)
-        msg = "Invalid callback"
+        msg = "Invalid spider callback parse_something"
         assert re.search(msg, self.crawl_manager.user_error.message)
         self.assertFalse(self.crawler.engine.crawl.called)
 
@@ -153,7 +153,7 @@ def test_pass_wrong_spider_errback(self):
         assert mng.request.errback is None
 
         self.assertIsNotNone(mng.user_error)
-        msg = "has no attribute 'handle_error'"
+        msg = "Invalid spider errback"
         assert re.search(msg, mng.user_error.message)
 
     def test_pass_good_spider_errback(self):