Merge pull request #11584 from rednithin/nreddy/pr/sentinel-bug-fixes

Abnormal Security Data Connector Bug Fixes

Solutions/AbnormalSecurity/Data Connectors/SentinelFunctionsOrchestrator/soar_connector_async.py

@@ -48,7 +48,7 @@ def _get_header(self):
         return {
             "Authorization": f"Bearer {self.api_key}",
             "Soar-Integration-Origin": "AZURE SENTINEL",
-            "Azure-Sentinel-Version": "2024-11-29"
+            "Azure-Sentinel-Version": "2024-12-24"
         }
 
     def _get_filter_query(self, filter_param, gte_datetime=None, lte_datetime=None):

Solutions/AbnormalSecurity/Data Connectors/SentinelFunctionsOrchestrator/soar_connector_async_v2.py

@@ -36,7 +36,7 @@ def get_headers(ctx: Context) -> Dict[str, str]:
         "X-Abnormal-Trace-Id": str(ctx.TRACE_ID),
         "Authorization": f"Bearer {ctx.API_TOKEN}",
         "Soar-Integration-Origin": "AZURE SENTINEL",
-        "Azure-Sentinel-Version": "2024-11-29 V2",
+        "Azure-Sentinel-Version": "2024-12-24 V2",
     }
 
 
@@ -50,7 +50,7 @@ def compute_url(base_url: str, pathname: str, params: Dict[str, str]) -> str:
     return endpoint
 
 
-async def fetch_with_retries(url, retries=3, backoff=4, timeout=10, headers=None):
+async def fetch_with_retries(url, retries=3, backoff=8, timeout=60, headers=None):
     logging.info(f"Fetching url: {url}")
     async def fetch(session, url):
         async with session.get(url, headers=headers, timeout=timeout) as response:
@@ -68,23 +68,27 @@ async def fetch(session, url):
             logging.info(f"API Response Status for URL: `{url}` is `{response.status}`")
             return json.loads(text)
 
-    async with aiohttp.ClientSession() as session:
-        for attempt in range(1, retries + 1):
+    for attempt in range(1, retries + 1):
+        async with aiohttp.ClientSession() as session:
             try:
+                logging.info(f"Fetch Attempt `{attempt}` for url: `{url}`")
                 response = await fetch(session, url)
                 return response
             except aiohttp.ClientResponseError as e:
                 if 500 <= e.status < 600:
-                    logging.error("Attempt {attempt} failed with error", exc_info=e)
+                    logging.error(f"Attempt {attempt} for {url} failed with error", exc_info=e)
                     if attempt == retries:
                         raise
                     else:
                         await asyncio.sleep(backoff**attempt)
                 else:
                     raise
-            except aiohttp.ClientError as e:
-                logging.error("Request failed with non-retryable error", exc_info=e)
-                raise
+            except Exception as e:
+                logging.error(f"Attempt {attempt} for {url} failed with error", exc_info=e)
+                if attempt == retries:
+                    raise
+                else:
+                    await asyncio.sleep(backoff**attempt)
 
 
 async def call_threat_campaigns_endpoint(

Solutions/AbnormalSecurity/Data Connectors/SentinelFunctionsOrchestrator/utils.py

@@ -17,7 +17,7 @@ def try_str_to_datetime(time: str) -> datetime:
         return datetime.strptime(time, TIME_FORMAT)
     except Exception as _:
         pass
-    return datetime.strptime(time, TIME_FORMAT_WITHMS)
+    return datetime.strptime((time[:26] + 'Z') if len(time) > 26 else time, TIME_FORMAT_WITHMS)
 
 
 class TimeRange(BaseModel):

Solutions/AbnormalSecurity/Data Connectors/Tests/soar_connector_async_v2_test.py

@@ -111,7 +111,7 @@ def test_valid_headers(self):
             "X-Abnormal-Trace-Id": str(self.trace_id),
             "Authorization": f"Bearer {self.api_token}",
             "Soar-Integration-Origin": "AZURE SENTINEL",
-            "Azure-Sentinel-Version": "2024-11-29 V2",
+            "Azure-Sentinel-Version": "2024-12-24 V2",
         }
         self.maxDiff = None
         self.assertEqual(headers, expected_headers)

Solutions/AbnormalSecurity/Data Connectors/Tests/utils_test.py

@@ -11,6 +11,7 @@
 )
 from pydantic import ValidationError
 from uuid import uuid4
+import random 
 
 
 class TestTryStrToDateTime(unittest.TestCase):
@@ -27,6 +28,29 @@ def test_format_with_ms(self):
         expected = datetime.strptime(time_str, TIME_FORMAT_WITHMS)
         result = try_str_to_datetime(time_str)
         self.assertEqual(result, expected)
+
+    def test_format_with_ns(self):
+        # Test case for format with milliseconds
+        time_str_ns = "2024-10-01T12:34:56.123456789Z"
+        time_str_ms = "2024-10-01T12:34:56.123456Z"
+        expected = datetime.strptime(time_str_ms, TIME_FORMAT_WITHMS)
+        result = try_str_to_datetime(time_str_ns)
+        self.assertEqual(result, expected)
+
+    def test_format_with_ns_2(self):
+        # Test case for format with milliseconds
+        time_str_ns = "2024-10-01T12:34:56.12345678913Z"
+        result = try_str_to_datetime(time_str_ns)
+        self.assertIsNotNone(result)
+
+    def test_format_with_ns_3(self):
+        # Test case for format with milliseconds
+        f = ""
+        for i in range(100):
+            f +=  random.choice("1234567890")
+            time_str_ns = f"2024-10-01T12:34:56.{f}Z"
+            result = try_str_to_datetime(time_str_ns)
+            self.assertIsNotNone(result)
 
     def test_invalid_format(self):
         # Test case for invalid format