From 7a0916aa86bfa15b6c68c54996e1ae295a268983 Mon Sep 17 00:00:00 2001
From: juju4 <juju4@users.noreply.github.com>
Date: Sat, 30 Sep 2023 15:40:37 +0000
Subject: [PATCH 1/7] feat: add file transfer detection rules for sftp based on
 threshold to adapt to context

---
 .../sftp_file_transfer_above_threshold.yaml   | 58 ++++++++++++++++++
 ...file_transfer_folders_above_threshold.yaml | 60 +++++++++++++++++++
 2 files changed, 118 insertions(+)
 create mode 100644 Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml
 create mode 100644 Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml

diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml
new file mode 100644
index 00000000000..75ed9ef9f64
--- /dev/null
+++ b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml	
@@ -0,0 +1,58 @@
+id: bb6a74c8-889d-4c6e-8412-7d5efe33f4ed
+name: SFTP File transfer above threshold
+description: |
+  'Identifies SFTP File Transfers above certain threshold in a 15min time period. It requires SFTP VERBOSE loglevel to be enabled.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
+   You may want to add custom details with transfer count and files list.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: Syslog
+    dataTypes:
+      - Syslog
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 50;
+  Syslog
+  | where ProcessName has "sftp"
+      and SyslogMessage has "close "
+      and SyslogMessage has " bytes read "
+  | parse SyslogMessage with "close \"" filepath "\" bytes read " readbytes: int " written " writtenbytes: int
+  | join kind=leftouter (
+      Syslog
+      | where ProcessName has "sftp" and SyslogMessage has "session opened for"
+      | parse SyslogMessage with "session opened for local user " username: string " from [" src_ip "]"
+      | project username, src_ip, ProcessID
+      )
+      on ProcessID
+  | project-away ProcessID1
+  | summarize count_distinct(filepath), fileslist=make_set(filepath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
+  | where count_distinct_filepath >= threshold
+  | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: username
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: src_ip
+  - entityType: Host
+    fieldMappings:
+      - identifier: HostName
+        columnName: Computer
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: FileSample
+version: 1.0.0
+kind: Scheduled
diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml
new file mode 100644
index 00000000000..f4ad6821327
--- /dev/null
+++ b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml	
@@ -0,0 +1,60 @@
+id: 7355434e-09d5-4401-b56d-e03e9379dfb1
+name: SFTP File transfer folder count above threshold
+description: |
+  'Identifies SFTP File Transfers with distinct folder count above certain threshold in a 15min time period.
+   It requires SFTP VERBOSE loglevel to be enabled.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
+   You may want to add custom details with transfer count and files list.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: Syslog
+    dataTypes:
+      - Syslog
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 10;
+  Syslog
+  | where ProcessName has "sftp"
+      and SyslogMessage has "close "
+      and SyslogMessage has " bytes read "
+  | parse SyslogMessage with "close \"" filepath "\" bytes read " readbytes: int " written " writtenbytes: int
+  | parse kind=regex filepath with dirpath:string "/" filename:string
+  | join kind=leftouter (
+      Syslog
+      | where ProcessName has "sftp" and SyslogMessage has "session opened for"
+      | parse SyslogMessage with "session opened for local user " username: string " from [" src_ip "]"
+      | project username, src_ip, ProcessID
+      )
+      on ProcessID
+  | project-away ProcessID1
+  | summarize count_distinct(dirpath), dirlist=make_set(dirpath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
+  | where count_distinct_dirpath >= threshold
+  | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: username
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: src_ip
+  - entityType: Host
+    fieldMappings:
+      - identifier: HostName
+        columnName: Computer
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: DirSample
+version: 1.0.0
+kind: Scheduled

From 619e4333e76a5b134f6eaecef4afc6f3b879f496 Mon Sep 17 00:00:00 2001
From: juju4 <juju4@users.noreply.github.com>
Date: Sat, 30 Sep 2023 15:41:11 +0000
Subject: [PATCH 2/7] feat: add file transfer detection rules for sharepoint
 based on threshold to adapt to context

---
 ...repoint_file_transfer_above_threshold.yaml | 43 +++++++++++++++++++
 ...file_transfer_folders_above_threshold.yaml | 43 +++++++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml
 create mode 100644 Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml

diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml
new file mode 100644
index 00000000000..539a29175cc
--- /dev/null
+++ b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml	
@@ -0,0 +1,43 @@
+id: 8b4f03e7-3460-4401-824d-e65a8dd464f0
+name: Office365 Sharepoint File transfer above threshold
+description: |
+  'Identifies Office365 Sharepoint File Transfers above certain threshold in a 15min time period.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
+   You may want to add custom details with transfer count and files list.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: Office365
+    dataTypes:
+      - OfficeActivity
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 5000;
+  OfficeActivity
+  | where EventSource == "SharePoint" and OfficeWorkload has_any("SharePoint", "OneDrive") and Operation has_any ("FileDownloaded", "FileSyncDownloadedFull", "FileSyncUploadedFull", "FileUploaded")
+  | summarize count_distinct(OfficeObjectId), fileslist=make_set(OfficeObjectId) by UserId,ClientIP,bin(TimeGenerated, 15m)
+  | where count_distinct_OfficeObjectId >= threshold
+  | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: UserId
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: ClientIP
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: FileSample
+version: 1.0.0
+kind: Scheduled
diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml
new file mode 100644
index 00000000000..05e423d9970
--- /dev/null
+++ b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml	
@@ -0,0 +1,43 @@
+id: 8a547285-801c-4290-aa2e-5e7e20ca157d
+name: Office365 Sharepoint File transfer above threshold
+description: |
+  'Identifies Office365 Sharepoint File Transfers with distinct folder count above certain threshold in a 15min time period.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
+   You may want to add custom details with transfer count and files list.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: Office365
+    dataTypes:
+      - OfficeActivity
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 500;
+  OfficeActivity
+  | where EventSource == "SharePoint" and OfficeWorkload has_any("SharePoint", "OneDrive") and Operation has_any ("FileDownloaded", "FileSyncDownloadedFull", "FileSyncUploadedFull", "FileUploaded")
+  | summarize count_distinct(SourceRelativeUrl), dirlist=make_set(SourceRelativeUrl) by UserId,ClientIP,UserAgent,bin(TimeGenerated, 15m)
+  | where count_distinct_SourceRelativeUrl >= threshold
+  | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: UserId
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: ClientIP
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: DirSample
+version: 1.0.0
+kind: Scheduled

From 6bf9874f15b32cf79e97106683868b7855ef5bd7 Mon Sep 17 00:00:00 2001
From: juju4 <juju4@users.noreply.github.com>
Date: Sat, 30 Sep 2023 15:41:41 +0000
Subject: [PATCH 3/7] feat: add file transfer detection rules for progress
 moveit based on threshold to adapt to context

---
 .../moveit_file_transfer_above_threshold.yml  | 50 +++++++++++++++++++
 ..._file_transfer_folders_above_threshold.yml | 50 +++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml
 create mode 100644 Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml

diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml
new file mode 100644
index 00000000000..a6f6f3291c4
--- /dev/null
+++ b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml	
@@ -0,0 +1,50 @@
+id: 9bd18b63-f1ca-4375-95db-39fda00bfe20
+name: Progress MOVEIt File transfer above threshold
+description: |
+  'Identifies Progress MOVEIt File Transfers above certain threshold in a 15min time period.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
+   You may want to add custom details with transfer count and files list.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: WindowsForwardedEvents
+    dataTypes:
+      - WindowsEvent
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 500;
+  Event
+  | where EventID == 0 and Source == "MOVEit DMZ Audit" and EventLog == "Application"
+  | where RenderedDescription has_any ("Downloaded", "Uploaded")
+  | parse RenderedDescription with * "User '" userdisplayname "' (" *
+  | parse RenderedDescription with * "IPAddress: " ipaddress " FileID:" * " FileName: " filename " FolderID: " folderid " FolderPath: " folderpath " Username: " username " AgentBrand: " *
+  | summarize count(), fileslist=make_set(filename) by Computer,userdisplayname,ipaddress,bin(TimeGenerated, 1d)
+  | where count_ >= threshold
+  | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: userdisplayname
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: ipaddress
+  - entityType: Host
+    fieldMappings:
+      - identifier: HostName
+        columnName: Computer
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: FileSample
+version: 1.0.0
+kind: Scheduled
diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml
new file mode 100644
index 00000000000..789d421d6b2
--- /dev/null
+++ b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml	
@@ -0,0 +1,50 @@
+id: 26a993ca-0a96-45a0-8405-05a210fb98f8
+name: Progress MOVEIt File transfer folder count above threshold
+description: |
+  'Identifies Progress MOVEIt File Transfers  with distinct folder count above certain threshold in a 15min time period.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
+   You may want to add custom details with transfer count and files list.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: WindowsForwardedEvents
+    dataTypes:
+      - WindowsEvent
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 500;
+  Event
+  | where EventID == 0 and Source == "MOVEit DMZ Audit" and EventLog == "Application"
+  | where RenderedDescription has_any ("Downloaded", "Uploaded")
+  | parse RenderedDescription with * "User '" userdisplayname "' (" *
+  | parse RenderedDescription with * "IPAddress: " ipaddress " FileID:" * " FileName: " filename " FolderID: " folderid " FolderPath: " folderpath " Username: " username " AgentBrand: " *
+  | summarize count_distinct(folderpath), dirlist=make_set(folderpath) by Computer,userdisplayname,username,ipaddress,bin(TimeGenerated, 15m)
+  | where count_distinct_folderpath >= threshold
+  | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: userdisplayname
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: ipaddress
+  - entityType: Host
+    fieldMappings:
+      - identifier: HostName
+        columnName: Computer
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: DirSample
+version: 1.0.0
+kind: Scheduled

From 1a76e0338eacb541178a8689ae7d2a8e5ab5e4a9 Mon Sep 17 00:00:00 2001
From: juju4 <juju4@users.noreply.github.com>
Date: Sat, 7 Oct 2023 18:48:31 +0000
Subject: [PATCH 4/7] fix: explicitely defined count variable, set
 customDetails

---
 .../sharepoint_file_transfer_above_threshold.yaml        | 8 +++++---
 ...sharepoint_file_transfer_folders_above_threshold.yaml | 8 +++++---
 .../sftp_file_transfer_above_threshold.yaml              | 9 ++++++---
 .../sftp_file_transfer_folders_above_threshold.yaml      | 8 +++++---
 .../moveit_file_transfer_above_threshold.yml             | 8 +++++---
 .../moveit_file_transfer_folders_above_threshold.yml     | 8 +++++---
 6 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml
index 539a29175cc..0ab76438efe 100644
--- a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml	
+++ b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml	
@@ -4,8 +4,7 @@ description: |
   'Identifies Office365 Sharepoint File Transfers above certain threshold in a 15min time period.
    Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
    value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
-   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
-   You may want to add custom details with transfer count and files list.'
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
 severity: Medium
 requiredDataConnectors:
   - connectorId: Office365
@@ -23,7 +22,7 @@ query: |
   let threshold = 5000;
   OfficeActivity
   | where EventSource == "SharePoint" and OfficeWorkload has_any("SharePoint", "OneDrive") and Operation has_any ("FileDownloaded", "FileSyncDownloadedFull", "FileSyncUploadedFull", "FileUploaded")
-  | summarize count_distinct(OfficeObjectId), fileslist=make_set(OfficeObjectId) by UserId,ClientIP,bin(TimeGenerated, 15m)
+  | summarize count_distinct_OfficeObjectId=count_distinct(OfficeObjectId), fileslist=make_set(OfficeObjectId) by UserId,ClientIP,bin(TimeGenerated, 15m)
   | where count_distinct_OfficeObjectId >= threshold
   | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
 entityMappings:
@@ -39,5 +38,8 @@ entityMappings:
     fieldMappings:
       - identifier: Name
         columnName: FileSample
+customDetails:
+  TransferCount: count_distinct_OfficeObjectId
+  FilesList: fileslist
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml
index 05e423d9970..9a4ce46e51e 100644
--- a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml	
+++ b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml	
@@ -4,8 +4,7 @@ description: |
   'Identifies Office365 Sharepoint File Transfers with distinct folder count above certain threshold in a 15min time period.
    Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
    value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
-   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
-   You may want to add custom details with transfer count and files list.'
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
 severity: Medium
 requiredDataConnectors:
   - connectorId: Office365
@@ -23,7 +22,7 @@ query: |
   let threshold = 500;
   OfficeActivity
   | where EventSource == "SharePoint" and OfficeWorkload has_any("SharePoint", "OneDrive") and Operation has_any ("FileDownloaded", "FileSyncDownloadedFull", "FileSyncUploadedFull", "FileUploaded")
-  | summarize count_distinct(SourceRelativeUrl), dirlist=make_set(SourceRelativeUrl) by UserId,ClientIP,UserAgent,bin(TimeGenerated, 15m)
+  | summarize count_distinct_SourceRelativeUrl=count_distinct(SourceRelativeUrl), dirlist=make_set(SourceRelativeUrl) by UserId,ClientIP,UserAgent,bin(TimeGenerated, 15m)
   | where count_distinct_SourceRelativeUrl >= threshold
   | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
 entityMappings:
@@ -39,5 +38,8 @@ entityMappings:
     fieldMappings:
       - identifier: Name
         columnName: DirSample
+customDetails:
+  TransferCount: count_distinct_SourceRelativeUrl
+  FilesList: dirlist
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml
index 75ed9ef9f64..bed174370a3 100644
--- a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml	
+++ b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml	
@@ -4,8 +4,7 @@ description: |
   'Identifies SFTP File Transfers above certain threshold in a 15min time period. It requires SFTP VERBOSE loglevel to be enabled.
    Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
    value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
-   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
-   You may want to add custom details with transfer count and files list.'
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
 severity: Medium
 requiredDataConnectors:
   - connectorId: Syslog
@@ -34,7 +33,7 @@ query: |
       )
       on ProcessID
   | project-away ProcessID1
-  | summarize count_distinct(filepath), fileslist=make_set(filepath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
+  | summarize count_distinct_filepath=count_distinct(filepath), fileslist=make_set(filepath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
   | where count_distinct_filepath >= threshold
   | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
 entityMappings:
@@ -54,5 +53,9 @@ entityMappings:
     fieldMappings:
       - identifier: Name
         columnName: FileSample
+customDetails:
+  TransferCount: count_distinct_filepath
+  FilesList: fileslist
+alertDetailsOverride:
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml
index f4ad6821327..08c3b001ad6 100644
--- a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml	
+++ b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml	
@@ -5,8 +5,7 @@ description: |
    It requires SFTP VERBOSE loglevel to be enabled.
    Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
    value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
-   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
-   You may want to add custom details with transfer count and files list.'
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
 severity: Medium
 requiredDataConnectors:
   - connectorId: Syslog
@@ -36,7 +35,7 @@ query: |
       )
       on ProcessID
   | project-away ProcessID1
-  | summarize count_distinct(dirpath), dirlist=make_set(dirpath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
+  | summarize count_distinct_dirpath=count_distinct(dirpath), dirlist=make_set(dirpath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
   | where count_distinct_dirpath >= threshold
   | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
 entityMappings:
@@ -56,5 +55,8 @@ entityMappings:
     fieldMappings:
       - identifier: Name
         columnName: DirSample
+customDetails:
+  TransferCount: count_distinct_dirpath
+  FilesList: dirlist
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml
index a6f6f3291c4..4f1b5dd798d 100644
--- a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml	
+++ b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml	
@@ -4,8 +4,7 @@ description: |
   'Identifies Progress MOVEIt File Transfers above certain threshold in a 15min time period.
    Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
    value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
-   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
-   You may want to add custom details with transfer count and files list.'
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
 severity: Medium
 requiredDataConnectors:
   - connectorId: WindowsForwardedEvents
@@ -26,7 +25,7 @@ query: |
   | where RenderedDescription has_any ("Downloaded", "Uploaded")
   | parse RenderedDescription with * "User '" userdisplayname "' (" *
   | parse RenderedDescription with * "IPAddress: " ipaddress " FileID:" * " FileName: " filename " FolderID: " folderid " FolderPath: " folderpath " Username: " username " AgentBrand: " *
-  | summarize count(), fileslist=make_set(filename) by Computer,userdisplayname,ipaddress,bin(TimeGenerated, 1d)
+  | summarize count_=count(), fileslist=make_set(filename) by Computer,userdisplayname,ipaddress,bin(TimeGenerated, 1d)
   | where count_ >= threshold
   | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
 entityMappings:
@@ -46,5 +45,8 @@ entityMappings:
     fieldMappings:
       - identifier: Name
         columnName: FileSample
+customDetails:
+  TransferCount: count_
+  FilesList: fileslist
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml
index 789d421d6b2..4f16eb3ed68 100644
--- a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml	
+++ b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml	
@@ -4,8 +4,7 @@ description: |
   'Identifies Progress MOVEIt File Transfers  with distinct folder count above certain threshold in a 15min time period.
    Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
    value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
-   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.
-   You may want to add custom details with transfer count and files list.'
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
 severity: Medium
 requiredDataConnectors:
   - connectorId: WindowsForwardedEvents
@@ -26,7 +25,7 @@ query: |
   | where RenderedDescription has_any ("Downloaded", "Uploaded")
   | parse RenderedDescription with * "User '" userdisplayname "' (" *
   | parse RenderedDescription with * "IPAddress: " ipaddress " FileID:" * " FileName: " filename " FolderID: " folderid " FolderPath: " folderpath " Username: " username " AgentBrand: " *
-  | summarize count_distinct(folderpath), dirlist=make_set(folderpath) by Computer,userdisplayname,username,ipaddress,bin(TimeGenerated, 15m)
+  | summarize count_distinct_folderpath=count_distinct(folderpath), dirlist=make_set(folderpath) by Computer,userdisplayname,username,ipaddress,bin(TimeGenerated, 15m)
   | where count_distinct_folderpath >= threshold
   | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
 entityMappings:
@@ -46,5 +45,8 @@ entityMappings:
     fieldMappings:
       - identifier: Name
         columnName: DirSample
+customDetails:
+  TransferCount: count_distinct_folderpath
+  FilesList: dirlist
 version: 1.0.0
 kind: Scheduled

From 619f6cf77489da16384b94035573217e67f1534a Mon Sep 17 00:00:00 2001
From: juju4 <juju4@users.noreply.github.com>
Date: Sun, 15 Oct 2023 15:18:09 +0000
Subject: [PATCH 5/7] chore: s/yml/yaml/ file extension

---
 ...ve_threshold.yml => moveit_file_transfer_above_threshold.yaml} | 0
 ...hold.yml => moveit_file_transfer_folders_above_threshold.yaml} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename Solutions/Windows Forwarded Events/Analytic Rules/{moveit_file_transfer_above_threshold.yml => moveit_file_transfer_above_threshold.yaml} (100%)
 rename Solutions/Windows Forwarded Events/Analytic Rules/{moveit_file_transfer_folders_above_threshold.yml => moveit_file_transfer_folders_above_threshold.yaml} (100%)

diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yaml
similarity index 100%
rename from Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yml
rename to Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yaml
diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml
similarity index 100%
rename from Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yml
rename to Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml

From 7810110d28142ce97d46a2899917e7211b18ad51 Mon Sep 17 00:00:00 2001
From: juju4 <juju4@users.noreply.github.com>
Date: Sat, 28 Oct 2023 16:35:10 +0000
Subject: [PATCH 6/7] fix: s/count_distinct()/dcount()/

---
 .../sharepoint_file_transfer_above_threshold.yaml               | 2 +-
 .../sharepoint_file_transfer_folders_above_threshold.yaml       | 2 +-
 .../Analytic Rules/sftp_file_transfer_above_threshold.yaml      | 2 +-
 .../sftp_file_transfer_folders_above_threshold.yaml             | 2 +-
 .../moveit_file_transfer_folders_above_threshold.yaml           | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml
index 0ab76438efe..72693d0f375 100644
--- a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml	
+++ b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml	
@@ -22,7 +22,7 @@ query: |
   let threshold = 5000;
   OfficeActivity
   | where EventSource == "SharePoint" and OfficeWorkload has_any("SharePoint", "OneDrive") and Operation has_any ("FileDownloaded", "FileSyncDownloadedFull", "FileSyncUploadedFull", "FileUploaded")
-  | summarize count_distinct_OfficeObjectId=count_distinct(OfficeObjectId), fileslist=make_set(OfficeObjectId) by UserId,ClientIP,bin(TimeGenerated, 15m)
+  | summarize count_distinct_OfficeObjectId=dcount(OfficeObjectId), fileslist=make_set(OfficeObjectId) by UserId,ClientIP,bin(TimeGenerated, 15m)
   | where count_distinct_OfficeObjectId >= threshold
   | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
 entityMappings:
diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml
index 9a4ce46e51e..52233903e98 100644
--- a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml	
+++ b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml	
@@ -22,7 +22,7 @@ query: |
   let threshold = 500;
   OfficeActivity
   | where EventSource == "SharePoint" and OfficeWorkload has_any("SharePoint", "OneDrive") and Operation has_any ("FileDownloaded", "FileSyncDownloadedFull", "FileSyncUploadedFull", "FileUploaded")
-  | summarize count_distinct_SourceRelativeUrl=count_distinct(SourceRelativeUrl), dirlist=make_set(SourceRelativeUrl) by UserId,ClientIP,UserAgent,bin(TimeGenerated, 15m)
+  | summarize count_distinct_SourceRelativeUrl=dcount(SourceRelativeUrl), dirlist=make_set(SourceRelativeUrl) by UserId,ClientIP,UserAgent,bin(TimeGenerated, 15m)
   | where count_distinct_SourceRelativeUrl >= threshold
   | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
 entityMappings:
diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml
index bed174370a3..466cae6a120 100644
--- a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml	
+++ b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml	
@@ -33,7 +33,7 @@ query: |
       )
       on ProcessID
   | project-away ProcessID1
-  | summarize count_distinct_filepath=count_distinct(filepath), fileslist=make_set(filepath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
+  | summarize count_distinct_filepath=dcount(filepath), fileslist=make_set(filepath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
   | where count_distinct_filepath >= threshold
   | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
 entityMappings:
diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml
index 08c3b001ad6..96ca9717966 100644
--- a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml	
+++ b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml	
@@ -35,7 +35,7 @@ query: |
       )
       on ProcessID
   | project-away ProcessID1
-  | summarize count_distinct_dirpath=count_distinct(dirpath), dirlist=make_set(dirpath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
+  | summarize count_distinct_dirpath=dcount(dirpath), dirlist=make_set(dirpath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
   | where count_distinct_dirpath >= threshold
   | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
 entityMappings:
diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml
index 4f16eb3ed68..57d858a910c 100644
--- a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml	
+++ b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml	
@@ -25,7 +25,7 @@ query: |
   | where RenderedDescription has_any ("Downloaded", "Uploaded")
   | parse RenderedDescription with * "User '" userdisplayname "' (" *
   | parse RenderedDescription with * "IPAddress: " ipaddress " FileID:" * " FileName: " filename " FolderID: " folderid " FolderPath: " folderpath " Username: " username " AgentBrand: " *
-  | summarize count_distinct_folderpath=count_distinct(folderpath), dirlist=make_set(folderpath) by Computer,userdisplayname,username,ipaddress,bin(TimeGenerated, 15m)
+  | summarize count_distinct_folderpath=dcount(folderpath), dirlist=make_set(folderpath) by Computer,userdisplayname,username,ipaddress,bin(TimeGenerated, 15m)
   | where count_distinct_folderpath >= threshold
   | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
 entityMappings:

From 458ff008699fb71bb2af91fa227d156cfeac3a47 Mon Sep 17 00:00:00 2001
From: juju4 <juju4@users.noreply.github.com>
Date: Sat, 28 Oct 2023 16:38:49 +0000
Subject: [PATCH 7/7] feat: add groupByEntities

---
 .../sharepoint_file_transfer_above_threshold.yaml    | 11 +++++++++++
 ...epoint_file_transfer_folders_above_threshold.yaml | 11 +++++++++++
 .../sftp_file_transfer_above_threshold.yaml          | 12 ++++++++++++
 .../sftp_file_transfer_folders_above_threshold.yaml  | 12 ++++++++++++
 .../moveit_file_transfer_above_threshold.yaml        | 12 ++++++++++++
 ...moveit_file_transfer_folders_above_threshold.yaml | 12 ++++++++++++
 6 files changed, 70 insertions(+)

diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml
index 72693d0f375..0a7bbb5cfe3 100644
--- a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml	
+++ b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml	
@@ -41,5 +41,16 @@ entityMappings:
 customDetails:
   TransferCount: count_distinct_OfficeObjectId
   FilesList: fileslist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    groupByAlertDetails: []
+    groupByCustomDetails: []
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml
index 52233903e98..11e8d1b9bcf 100644
--- a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml	
+++ b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml	
@@ -41,5 +41,16 @@ entityMappings:
 customDetails:
   TransferCount: count_distinct_SourceRelativeUrl
   FilesList: dirlist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    groupByAlertDetails: []
+    groupByCustomDetails: []
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml
index 466cae6a120..36ee0c0d967 100644
--- a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml	
+++ b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml	
@@ -56,6 +56,18 @@ entityMappings:
 customDetails:
   TransferCount: count_distinct_filepath
   FilesList: fileslist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    - Host
+    groupByAlertDetails: []
+    groupByCustomDetails: []
 alertDetailsOverride:
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml
index 96ca9717966..5c555a421e5 100644
--- a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml	
+++ b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml	
@@ -58,5 +58,17 @@ entityMappings:
 customDetails:
   TransferCount: count_distinct_dirpath
   FilesList: dirlist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    - Host
+    groupByAlertDetails: []
+    groupByCustomDetails: []
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yaml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yaml
index 4f1b5dd798d..32e35a8fef8 100644
--- a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yaml	
+++ b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yaml	
@@ -48,5 +48,17 @@ entityMappings:
 customDetails:
   TransferCount: count_
   FilesList: fileslist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    - Host
+    groupByAlertDetails: []
+    groupByCustomDetails: []
 version: 1.0.0
 kind: Scheduled
diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml
index 57d858a910c..e662297f000 100644
--- a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml	
+++ b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml	
@@ -48,5 +48,17 @@ entityMappings:
 customDetails:
   TransferCount: count_distinct_folderpath
   FilesList: dirlist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    - Host
+    groupByAlertDetails: []
+    groupByCustomDetails: []
 version: 1.0.0
 kind: Scheduled