Merge pull request #9131 from juju4/devel-file_transfers

feat: file transfers analytics rules based on threshold
Azure · Nov 1, 2023 · fe78559 · fe78559
2 parents 72c9543 + 458ff00
commit fe78559
Show file tree

Hide file tree

Showing 6 changed files with 387 additions and 0 deletions.
diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_above_threshold.yaml
@@ -0,0 +1,56 @@
+id: 8b4f03e7-3460-4401-824d-e65a8dd464f0
+name: Office365 Sharepoint File transfer above threshold
+description: |
+  'Identifies Office365 Sharepoint File Transfers above certain threshold in a 15min time period.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: Office365
+    dataTypes:
+      - OfficeActivity
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 5000;
+  OfficeActivity
+  | where EventSource == "SharePoint" and OfficeWorkload has_any("SharePoint", "OneDrive") and Operation has_any ("FileDownloaded", "FileSyncDownloadedFull", "FileSyncUploadedFull", "FileUploaded")
+  | summarize count_distinct_OfficeObjectId=dcount(OfficeObjectId), fileslist=make_set(OfficeObjectId) by UserId,ClientIP,bin(TimeGenerated, 15m)
+  | where count_distinct_OfficeObjectId >= threshold
+  | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: UserId
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: ClientIP
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: FileSample
+customDetails:
+  TransferCount: count_distinct_OfficeObjectId
+  FilesList: fileslist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    groupByAlertDetails: []
+    groupByCustomDetails: []
+version: 1.0.0
+kind: Scheduled
diff --git a/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml b/Solutions/Microsoft 365/Analytic Rules/sharepoint_file_transfer_folders_above_threshold.yaml
@@ -0,0 +1,56 @@
+id: 8a547285-801c-4290-aa2e-5e7e20ca157d
+name: Office365 Sharepoint File transfer above threshold
+description: |
+  'Identifies Office365 Sharepoint File Transfers with distinct folder count above certain threshold in a 15min time period.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: Office365
+    dataTypes:
+      - OfficeActivity
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 500;
+  OfficeActivity
+  | where EventSource == "SharePoint" and OfficeWorkload has_any("SharePoint", "OneDrive") and Operation has_any ("FileDownloaded", "FileSyncDownloadedFull", "FileSyncUploadedFull", "FileUploaded")
+  | summarize count_distinct_SourceRelativeUrl=dcount(SourceRelativeUrl), dirlist=make_set(SourceRelativeUrl) by UserId,ClientIP,UserAgent,bin(TimeGenerated, 15m)
+  | where count_distinct_SourceRelativeUrl >= threshold
+  | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: UserId
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: ClientIP
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: DirSample
+customDetails:
+  TransferCount: count_distinct_SourceRelativeUrl
+  FilesList: dirlist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    groupByAlertDetails: []
+    groupByCustomDetails: []
+version: 1.0.0
+kind: Scheduled
diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml
@@ -0,0 +1,73 @@
+id: bb6a74c8-889d-4c6e-8412-7d5efe33f4ed
+name: SFTP File transfer above threshold
+description: |
+  'Identifies SFTP File Transfers above certain threshold in a 15min time period. It requires SFTP VERBOSE loglevel to be enabled.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: Syslog
+    dataTypes:
+      - Syslog
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 50;
+  Syslog
+  | where ProcessName has "sftp"
+      and SyslogMessage has "close "
+      and SyslogMessage has " bytes read "
+  | parse SyslogMessage with "close \"" filepath "\" bytes read " readbytes: int " written " writtenbytes: int
+  | join kind=leftouter (
+      Syslog
+      | where ProcessName has "sftp" and SyslogMessage has "session opened for"
+      | parse SyslogMessage with "session opened for local user " username: string " from [" src_ip "]"
+      | project username, src_ip, ProcessID
+      )
+      on ProcessID
+  | project-away ProcessID1
+  | summarize count_distinct_filepath=dcount(filepath), fileslist=make_set(filepath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
+  | where count_distinct_filepath >= threshold
+  | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: username
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: src_ip
+  - entityType: Host
+    fieldMappings:
+      - identifier: HostName
+        columnName: Computer
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: FileSample
+customDetails:
+  TransferCount: count_distinct_filepath
+  FilesList: fileslist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    - Host
+    groupByAlertDetails: []
+    groupByCustomDetails: []
+alertDetailsOverride:
+version: 1.0.0
+kind: Scheduled
diff --git a/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml b/Solutions/Syslog/Analytic Rules/sftp_file_transfer_folders_above_threshold.yaml
@@ -0,0 +1,74 @@
+id: 7355434e-09d5-4401-b56d-e03e9379dfb1
+name: SFTP File transfer folder count above threshold
+description: |
+  'Identifies SFTP File Transfers with distinct folder count above certain threshold in a 15min time period.
+   It requires SFTP VERBOSE loglevel to be enabled.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: Syslog
+    dataTypes:
+      - Syslog
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 10;
+  Syslog
+  | where ProcessName has "sftp"
+      and SyslogMessage has "close "
+      and SyslogMessage has " bytes read "
+  | parse SyslogMessage with "close \"" filepath "\" bytes read " readbytes: int " written " writtenbytes: int
+  | parse kind=regex filepath with dirpath:string "/" filename:string
+  | join kind=leftouter (
+      Syslog
+      | where ProcessName has "sftp" and SyslogMessage has "session opened for"
+      | parse SyslogMessage with "session opened for local user " username: string " from [" src_ip "]"
+      | project username, src_ip, ProcessID
+      )
+      on ProcessID
+  | project-away ProcessID1
+  | summarize count_distinct_dirpath=dcount(dirpath), dirlist=make_set(dirpath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
+  | where count_distinct_dirpath >= threshold
+  | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: username
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: src_ip
+  - entityType: Host
+    fieldMappings:
+      - identifier: HostName
+        columnName: Computer
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: DirSample
+customDetails:
+  TransferCount: count_distinct_dirpath
+  FilesList: dirlist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    - Host
+    groupByAlertDetails: []
+    groupByCustomDetails: []
+version: 1.0.0
+kind: Scheduled
diff --git a/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yaml b/Solutions/Windows Forwarded Events/Analytic Rules/moveit_file_transfer_above_threshold.yaml
@@ -0,0 +1,64 @@
+id: 9bd18b63-f1ca-4375-95db-39fda00bfe20
+name: Progress MOVEIt File transfer above threshold
+description: |
+  'Identifies Progress MOVEIt File Transfers above certain threshold in a 15min time period.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: WindowsForwardedEvents
+    dataTypes:
+      - WindowsEvent
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 500;
+  Event
+  | where EventID == 0 and Source == "MOVEit DMZ Audit" and EventLog == "Application"
+  | where RenderedDescription has_any ("Downloaded", "Uploaded")
+  | parse RenderedDescription with * "User '" userdisplayname "' (" *
+  | parse RenderedDescription with * "IPAddress: " ipaddress " FileID:" * " FileName: " filename " FolderID: " folderid " FolderPath: " folderpath " Username: " username " AgentBrand: " *
+  | summarize count_=count(), fileslist=make_set(filename) by Computer,userdisplayname,ipaddress,bin(TimeGenerated, 1d)
+  | where count_ >= threshold
+  | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: userdisplayname
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: ipaddress
+  - entityType: Host
+    fieldMappings:
+      - identifier: HostName
+        columnName: Computer
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: FileSample
+customDetails:
+  TransferCount: count_
+  FilesList: fileslist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    - Host
+    groupByAlertDetails: []
+    groupByCustomDetails: []
+version: 1.0.0
+kind: Scheduled
diff --git a/...Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml b/...Windows Forwarded Events/Analytic Rules/moveit_file_transfer_folders_above_threshold.yaml
@@ -0,0 +1,64 @@
+id: 26a993ca-0a96-45a0-8405-05a210fb98f8
+name: Progress MOVEIt File transfer folder count above threshold
+description: |
+  'Identifies Progress MOVEIt File Transfers  with distinct folder count above certain threshold in a 15min time period.
+   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that
+   value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally,
+   if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'
+severity: Medium
+requiredDataConnectors:
+  - connectorId: WindowsForwardedEvents
+    dataTypes:
+      - WindowsEvent
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Exfiltration
+relevantTechniques:
+  - T1020
+query: |
+  let threshold = 500;
+  Event
+  | where EventID == 0 and Source == "MOVEit DMZ Audit" and EventLog == "Application"
+  | where RenderedDescription has_any ("Downloaded", "Uploaded")
+  | parse RenderedDescription with * "User '" userdisplayname "' (" *
+  | parse RenderedDescription with * "IPAddress: " ipaddress " FileID:" * " FileName: " filename " FolderID: " folderid " FolderPath: " folderpath " Username: " username " AgentBrand: " *
+  | summarize count_distinct_folderpath=dcount(folderpath), dirlist=make_set(folderpath) by Computer,userdisplayname,username,ipaddress,bin(TimeGenerated, 15m)
+  | where count_distinct_folderpath >= threshold
+  | extend DirSample = iff(array_length(dirlist) == 1, tostring(dirlist[0]), strcat("SeeDirListField","_", tostring(hash(tostring(dirlist)))))
+entityMappings:
+  - entityType: Account
+    fieldMappings:
+      - identifier: Name
+        columnName: userdisplayname
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: ipaddress
+  - entityType: Host
+    fieldMappings:
+      - identifier: HostName
+        columnName: Computer
+  - entityType: File
+    fieldMappings:
+      - identifier: Name
+        columnName: DirSample
+customDetails:
+  TransferCount: count_distinct_folderpath
+  FilesList: dirlist
+incidentConfiguration:
+  createIncident: true
+  groupingConfiguration:
+    enabled: true
+    reopenClosedIncident: false
+    lookbackDuration: 5h
+    matchingMethod: Selected
+    groupByEntities:
+    - Account
+    - Host
+    groupByAlertDetails: []
+    groupByCustomDetails: []
+version: 1.0.0
+kind: Scheduled