From 25d91d44a0a9ca85d19014153860ae8195614993 Mon Sep 17 00:00:00 2001 From: v-amolpatil Date: Thu, 2 Jan 2025 14:53:21 +0530 Subject: [PATCH 1/4] code changes in poll and gcp solution --- .../data_connector_definition.json | 97 +++++++++++++++++++ .../data_connector_poller.json | 24 +++++ .../Data Connectors/GCPAuditLogs_ccp/dcr.json | 25 +++++ .../common/createCCPConnector.ps1 | 90 +++++++++++++++-- 4 files changed, 230 insertions(+), 6 deletions(-) create mode 100644 Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/data_connector_definition.json create mode 100644 Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/data_connector_poller.json create mode 100644 Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/dcr.json diff --git a/Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/data_connector_definition.json b/Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/data_connector_definition.json new file mode 100644 index 00000000000..dba40025722 --- /dev/null +++ b/Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/data_connector_definition.json @@ -0,0 +1,97 @@ +{ + "type": "Microsoft.SecurityInsights/dataConnectorDefinitions", + "apiVersion": "2022-09-01-preview", + "name": "{{workspace}}/Microsoft.SecurityInsights/GCPAuditLogsDefinition", + "kind": "Customizable", + "properties": { + "connectorUiConfig": { + "id": "GCPAuditLogsDefinition", + "title": "GCP Pub/Sub Audit Logs", + "publisher": "Microsoft", + "descriptionMarkdown": "The Google Cloud Platform (GCP) audit logs, ingested from Sentinel's connector, enable you to capture three types of audit logs: admin activity logs, data access logs, and access transparency logs. Google cloud audit logs record a trail that practitioners can use to monitor access and detect potential threats across Google Cloud Platform (GCP) resources.", + "graphQueriesTableName": "GCPAuditLogs", + "graphQueries": [ + { + "metricName": "Total events received", + "legend": "GCP Audit Logs", + "baseQuery": "{{graphQueriesTableName}}" + } + ], + "sampleQueries": [ + { + "description": "Get Sample of GCP Audit Logs", + "query": "{{graphQueriesTableName}}\n | take 10" + } + ], + "dataTypes": [ + { + "name": "{{graphQueriesTableName}}", + "lastDataReceivedQuery": "{{graphQueriesTableName}}\n | where TimeGenerated > ago(12h) | summarize Time = max(TimeGenerated)\n | where isnotempty(Time)" + } + ], + "connectivityCriteria": [ + { + "type": "HasDataConnectors" + } + ], + "availability": { + "status": 1, + "isPreview": false + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "Read and Write permissions are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "read": true, + "write": true, + "delete": true, + "action": false + } + } + ] + }, + "instructionSteps": [ + { + "instructions": [ + { + "type": "MarkdownControlEnvBased", + "parameters": { + "prodScript": + "#### 1. Set up your GCP environment \n You must have the following GCP resources defined and configured: topic, subscription for the topic, workload identity pool, workload identity provider and service account with permissions to get and consume from subscription. \n Terraform provides API for the IAM that creates the resources. [Link to Terraform scripts](https://github.com/Azure/Azure-Sentinel/tree/master/DataConnectors/GCP/Terraform/sentinel_resources_creation).", + "govScript": + "#### 1. Set up your GCP environment \n You must have the following GCP resources defined and configured: topic, subscription for the topic, workload identity pool, workload identity provider and service account with permissions to get and consume from subscription. \n Terraform provides API for the IAM that creates the resources. [Link to Gov Terraform scripts](https://github.com/Azure/Azure-Sentinel/tree/master/DataConnectors/GCP/Terraform/sentinel_resources_creation_gov)." + } + }, + { + "type": "CopyableLabel", + "parameters": { + "label": "Tenant ID: A unique identifier that is used as an input in the Terraform configuration within a GCP environment.", + "fillWith": ["TenantId"], + "name": "PoolId", + "disabled": true + } + }, + { + "type": "Markdown", + "parameters": { + "content": "#### 2. Connect new collectors \n To enable GCP Audit Logs for Microsoft Sentinel, click the Add new collector button, fill the required information in the context pane and click on Connect." + } + }, + { + "type": "GCPGrid", + "parameters":{} + }, + { + "type": "GCPContextPane", + "parameters":{} + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/data_connector_poller.json b/Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/data_connector_poller.json new file mode 100644 index 00000000000..75a7340e57f --- /dev/null +++ b/Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/data_connector_poller.json @@ -0,0 +1,24 @@ +[{ + "type": "Microsoft.SecurityInsights/dataConnectors", + "apiVersion": "2022-10-01-preview", + "name": "{{workspace}}/Microsoft.SecurityInsights/GCPAuditLogs", + "kind": "GCP", + "properties": { + "connectorDefinitionName": "GCPAuditLogsDefinition", + "dataType": "GCPAuditLogs", + "dcrConfig": { + "streamName": "SENTINEL_GCP_AUDIT_LOGS" + }, + "auth": { + "serviceAccountEmail": "{{GCPServiceAccountEmail}}", + "projectNumber": "{{GCPProjectNumber}}", + "workloadIdentityProviderId": "{{GCPWorkloadIdentityProviderId}}" + }, + "request": { + "projectId": "{{GCPProjectId}}", + "subscriptionNames": [ + "{{GCPSubscriptionName}}" + ] + } + } +}] \ No newline at end of file diff --git a/Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/dcr.json b/Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/dcr.json new file mode 100644 index 00000000000..77cfddb68b0 --- /dev/null +++ b/Solutions/Google Cloud Platform Audit Logs/Data Connectors/GCPAuditLogs_ccp/dcr.json @@ -0,0 +1,25 @@ +[{ + "name": "GCPAuditLogs", + "apiVersion": "2021-09-01-preview", + "type": "Microsoft.Insights/dataCollectionRules", + "location": "{{location}}", + "properties": { + "destinations": { + "logAnalytics": [ + { + "name": "clv2ws1" + } + ] + }, + "dataFlows": [ + { + "streams": [ + "Microsoft-GCPAuditLogs" + ], + "destinations": [ + "clv2ws1" + ] + } + ] + } +}] \ No newline at end of file diff --git a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 index 974428ad55f..4588413ad76 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 @@ -228,19 +228,91 @@ function Get-ArmResource($name, $type, $kind, $properties){ } } -function addNewParameter($templateResourceObj, $parameterName, $isSecret = $false) { +function addNewParameter($templateResourceObj, $parameterName, $isSecret = $false, $minLength = 1) { $hasParameter = [bool]($templateResourceObj.parameters.PSobject.Properties.name -match "$parameterName") if (!$hasParameter) { $templateResourceObj.parameters | Add-Member -NotePropertyName "$parameterName" -NotePropertyValue ([PSCustomObject] @{ defaultValue = $isSecret ? "-NA-" : "Enter $parameterName value"; type = $isSecret ? "securestring" : "string"; - minLength = 1; + minLength = $minLength; }) } return $templateResourceObj; } +function Create-GCP($armResource, $templateContentConnections) { + $hasServiceAccountEmail = [bool]($armResource.properties.auth.PSobject.Properties.name -match "serviceAccountEmail") + if ($hasServiceAccountEmail) { + $serviceAccountEmailProperty = $armResource.properties.auth.serviceAccountEmail + $placeHoldersMatched = $serviceAccountEmailProperty | Select-String $placeHolderPatternMatches -AllMatches + + if ($placeHoldersMatched.Matches.Value.Count -gt 0) { + $armResource.properties.auth.serviceAccountEmail = "[[parameters('GCPServiceAccountEmail')]" + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'serviceAccountEmail' -isSecret $true + } + } else { + Write-Host "Parameter 'serviceAccountEmail' missing from GCP data connector 'auth' section." -BackgroundColor Red + exit 1; + } + + $hasProjectNumber = [bool]($armResource.properties.auth.PSobject.Properties.name -match "projectNumber") + if ($hasProjectNumber) { + $projectNumberProperty = $armResource.properties.auth.projectNumber + $placeHoldersMatched = $projectNumberProperty | Select-String $placeHolderPatternMatches -AllMatches + + if ($placeHoldersMatched.Matches.Value.Count -gt 0) { + $armResource.properties.auth.projectNumber = "[[parameters('GCPProjectNumber')]" + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'projectNumber' -isSecret $true + } + } else { + Write-Host "Parameter 'projectNumber' missing from GCP data connector 'auth' section." -BackgroundColor Red + exit 1; + } + + $hasWorkloadIdentityProviderId = [bool]($armResource.properties.auth.PSobject.Properties.name -match "workloadIdentityProviderId") + if ($hasWorkloadIdentityProviderId) { + $workloadIdentityProviderIdProperty = $armResource.properties.auth.workloadIdentityProviderId + $placeHoldersMatched = $workloadIdentityProviderIdProperty | Select-String $placeHolderPatternMatches -AllMatches + + if ($placeHoldersMatched.Matches.Value.Count -gt 0) { + $armResource.properties.auth.workloadIdentityProviderId = "[[parameters('GCPWorkloadIdentityProviderId')]" + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'workloadIdentityProviderId' -isSecret $true + } + } else { + Write-Host "Parameter 'workloadIdentityProviderId' missing from GCP data connector 'auth' section." -BackgroundColor Red + exit 1; + } + + # Request section projectId property + $hasProjectId = [bool]($armResource.properties.request.PSobject.Properties.name -match "projectId") + if ($hasProjectId) { + $projectIdValue = $armResource.properties.request.projectId + $placeHoldersMatched = $projectIdValue | Select-String $placeHolderPatternMatches -AllMatches + if ($placeHoldersMatched.Matches.Value.Count -gt 0) { + $armResource.properties.request.project = "[[parameters('GCPProjectId')]" + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'ProjectId' -isSecret $false -minLength 4 + } + } else { + Write-Host "Parameter 'ProjectId' missing from GCP data connector 'request' section." -BackgroundColor Red + exit 1; + } + + # Request section subscriptionNames property + $hasSubscriptionNames = [bool]($armResource.properties.request.PSobject.Properties.name -match "subscriptionNames") + if ($hasSubscriptionNames) { + $subscriptionNamesValue = $armResource.properties.request.subscriptionNames + $placeHoldersMatched = $subscriptionNamesValue | Select-String $placeHolderPatternMatches -AllMatches + if ($placeHoldersMatched.Matches.Value.Count -gt 0) { + $armResource.properties.request.project = "[[parameters('GCPSubscriptionName')]" + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'SubscriptionNames' -isSecret $false -minLength 3 + } + } else { + Write-Host "Parameter 'subscriptionNames' missing from GCP data connector 'request' section." -BackgroundColor Red + exit 1; + } +} + # THIS IS THE STARTUP FUNCTION FOR CCP RESOURCE CREATOR function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, $solutionFileMetadata, $dcFolderName, $ccpDict, $solutionBasePath, $solutionName, $ccpTables, $ccpTablesCounter) { Write-Host "Inside of CCP Connector Code!" @@ -401,8 +473,13 @@ function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, # if dataCollectionRuleImmutableId property not present then add it $armResource.properties.dcrConfig | Add-Member -MemberType NoteProperty -Name "dataCollectionRuleImmutableId" -Value "[[parameters('dcrConfig').dataCollectionRuleImmutableId]" } - - if($armResource.properties.auth.type.ToLower() -eq 'oauth2') + + # Start: GCP CODE + if ($armResource.kind.ToLower() -eq 'gcp'){ + Create-GCP -armResource $armResource -templateContentConnections $templateContentConnections + } + # End: GCP CODE + elseif($armResource.properties.auth.type.ToLower() -eq 'oauth2') { # clientid $hasClientId = [bool]($armResource.properties.auth.PSobject.Properties.name -match "ClientId") @@ -504,9 +581,10 @@ function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, } } - if ($armResource.properties.request.apiEndPoint.contains("{{")) { + if ($armResource.properties.request.PSObject.Properties["apiEndPoint"] -and + $armResource.properties.request.apiEndPoint.contains("{{")) { # identify any placeholders in apiEndpoint - $endPointUrl = $armResource.properties.request.apiEndPoint + $endPointUrl = $armResource.properties.request.apiEndPoint $placeHoldersMatched = $endPointUrl | Select-String $placeHolderPatternMatches -AllMatches if ($placeHoldersMatched.Matches.Value.Count -gt 0) { From c34fe9610fc842e1c75b4f8c8604aa60e0ac40d1 Mon Sep 17 00:00:00 2001 From: v-amolpatil Date: Mon, 13 Jan 2025 19:38:48 +0530 Subject: [PATCH 2/4] added code gcp and aws --- .../V3/createSolutionV3.ps1 | 2 +- .../common/commonFunctions.ps1 | 3 + .../common/createCCPConnector.ps1 | 562 +++++++++++------- .../common/get-ccp-details.ps1 | 63 +- .../common/standardLogStreams.ps1 | 25 + 5 files changed, 427 insertions(+), 228 deletions(-) create mode 100644 Tools/Create-Azure-Sentinel-Solution/common/standardLogStreams.ps1 diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 index 41509bac463..65e09fdae6c 100644 --- a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 @@ -320,5 +320,5 @@ try { } } catch { - Write-Host "Error occured in catch of createSolutionV3 file Error details are $_" + Write-Host "Error occurred in catch of createSolutionV3 file Error details are $_" } \ No newline at end of file diff --git a/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 index a2cdcf8dc50..ff5a9cce98f 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 @@ -3112,6 +3112,9 @@ function PrepareSolutionMetadata($solutionMetadataRawContent, $contentResourceDe if ($null -ne $global:baseCreateUiDefinition.parameters.steps -and $($global:baseCreateUiDefinition.parameters.steps).GetType() -ne [System.Object[]]) { $global:baseCreateUiDefinition.parameters.steps = @($global:baseCreateUiDefinition.parameters.steps) + } elseif ($null -eq $global:baseCreateUiDefinition.parameters.steps) { + # when there is no content then create ui fails as step is null + $global:baseCreateUiDefinition.parameters.steps = @(@{}) # [{}] } $global:baseCreateUiDefinition | ConvertTo-Json -Depth $jsonConversionDepth | Out-File $createUiDefinitionOutputPath -Encoding utf8 } diff --git a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 index 4588413ad76..5c782d8a62d 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 @@ -241,62 +241,182 @@ function addNewParameter($templateResourceObj, $parameterName, $isSecret = $fals return $templateResourceObj; } -function Create-GCP($armResource, $templateContentConnections) { - $hasServiceAccountEmail = [bool]($armResource.properties.auth.PSobject.Properties.name -match "serviceAccountEmail") - if ($hasServiceAccountEmail) { - $serviceAccountEmailProperty = $armResource.properties.auth.serviceAccountEmail - $placeHoldersMatched = $serviceAccountEmailProperty | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.serviceAccountEmail = "[[parameters('GCPServiceAccountEmail')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'serviceAccountEmail' -isSecret $true - } - } else { - Write-Host "Parameter 'serviceAccountEmail' missing from GCP data connector 'auth' section." -BackgroundColor Red - exit 1; +function addWorkspaceParameter($templateResourceObj, $parameterName) { + $hasParameter = [bool]($templateResourceObj.parameters.PSobject.Properties.name -match "$parameterName") + if (!$hasParameter) { + $templateResourceObj.parameters | Add-Member -NotePropertyName "$parameterName" -NotePropertyValue ([PSCustomObject] @{ + defaultValue = "[parameters('workspace')]"; + type = "string"; + }) } - $hasProjectNumber = [bool]($armResource.properties.auth.PSobject.Properties.name -match "projectNumber") - if ($hasProjectNumber) { - $projectNumberProperty = $armResource.properties.auth.projectNumber - $placeHoldersMatched = $projectNumberProperty | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.projectNumber = "[[parameters('GCPProjectNumber')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'projectNumber' -isSecret $true + return $templateResourceObj; +} + +function CreateRestApiPollerResourceProperties() { + $kindType = 'RestApiPoller' + if($armResource.properties.auth.type.ToLower() -eq 'oauth2') + { + # clientid + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'ClientId' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $true -isRequired $true -minLength 4 + + # client secret + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'ClientSecret' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $true -isRequired $true -minLength 4 + + # authorization code + if($armResource.properties.auth.grantType -eq 'authorization_code') + { + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'AuthorizationCode' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $true -isRequired $false -minLength 4 } - } else { - Write-Host "Parameter 'projectNumber' missing from GCP data connector 'auth' section." -BackgroundColor Red + + # AuthorizationEndpoint placeholder + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'AuthorizationEndpoint' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $true -isRequired $false -minLength 4 + + # TokenEndpoint placeholder + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'TokenEndpoint' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $false -isRequired $false -minLength 4 + } + elseif ($armResource.properties.auth.type.ToLower() -eq 'basic') + { + # username + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'username' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 + + # password + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'password' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $true -isRequired $true -minLength 4 + } + elseif ($armResource.properties.auth.type.ToLower() -eq 'apikey') + { + # ApiKey + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'apikey' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $true -isRequired $true -minLength 4 + } + elseif($armResource.properties.auth.type.ToLower() -eq 'JwtToken') + { + # userName object check + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth -propertyName 'userName' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 + + # userName object should have key + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth.userName -propertyName 'key' -isInnerObject $true -innerObjectName 'userName' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 + + # userName object should have value + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth.userName -propertyName 'value' -isInnerObject $true -innerObjectName 'userName' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 + + # password object check + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth -propertyName 'password' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $true -isRequired $true -minLength 4 + + # password object should have key + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth.password -propertyName 'key' -isInnerObject $true -innerObjectName 'password' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 + + # password object should have value + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth.password -propertyName 'value' -isInnerObject $true -innerObjectName 'password' -kindType $kindType -isSecret $true -isRequired $true -minLength 4 + } + else + { + Write-Host "Error: For kind RestApiPoller, Data Connector Poller file should have 'auth' object with 'type' attribute having value either 'Basic', 'OAuth2', 'APIKey' or 'JwtToken'" -BackgroundColor Red exit 1; } - $hasWorkloadIdentityProviderId = [bool]($armResource.properties.auth.PSobject.Properties.name -match "workloadIdentityProviderId") - if ($hasWorkloadIdentityProviderId) { - $workloadIdentityProviderIdProperty = $armResource.properties.auth.workloadIdentityProviderId - $placeHoldersMatched = $workloadIdentityProviderIdProperty | Select-String $placeHolderPatternMatches -AllMatches + if ($armResource.properties.request.PSObject.Properties["apiEndPoint"] -and + $armResource.properties.request.apiEndPoint.contains("{{")) + { + # identify any placeholders in apiEndpoint + $endPointUrl = $armResource.properties.request.apiEndPoint + $placeHoldersMatched = $endPointUrl | Select-String $placeHolderPatternMatches -AllMatches - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.workloadIdentityProviderId = "[[parameters('GCPWorkloadIdentityProviderId')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'workloadIdentityProviderId' -isSecret $true + if ($placeHoldersMatched.Matches.Value.Count -gt 0) + { + # has some placeholders + $finalizedEndpointUrl = "" + $finalizedEndpointUrl = "[[concat(" + $closureBrackets = ")]" + + foreach ($currentPlaceHolder in $placeHoldersMatched.Matches.Value) { + $placeHolderName = $currentPlaceHolder.replace("{{", "").replace("}}", "") + $splitEndpoint = $endPointUrl -split "($currentPlaceHolder)" + $commaCount = 0 + foreach($splitItem in $splitEndpoint) + { + if ($splitItem -eq $currentPlaceHolder) + { + if ($finalizedEndpointUrl -eq '') + { + $finalizedEndpointUrl += "parameters('" + $placeHolderName + "')" + } + else + { + $finalizedEndpointUrl += ", parameters('" + $placeHolderName + "')" + } + + if ($placeHolderName.Contains("secret") -or $placeHolderName.Contains("password")) + { + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName "$placeHolderName" -isSecret $true + } + else + { + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName "$placeHolderName" -isSecret $false + } + } + else + { + if ($commaCount -eq 0) + { + $finalizedEndpointUrl += "'"+ $splitItem + "'" + $commaCount += 1 + } + else + { + $finalizedEndpointUrl += ", '" + $splitItem + "'" + } + } + } } - } else { - Write-Host "Parameter 'workloadIdentityProviderId' missing from GCP data connector 'auth' section." -BackgroundColor Red - exit 1; + + $armResource.properties.request.apiEndPoint = $finalizedEndpointUrl + $closureBrackets + } } - # Request section projectId property - $hasProjectId = [bool]($armResource.properties.request.PSobject.Properties.name -match "projectId") - if ($hasProjectId) { - $projectIdValue = $armResource.properties.request.projectId - $placeHoldersMatched = $projectIdValue | Select-String $placeHolderPatternMatches -AllMatches - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.request.project = "[[parameters('GCPProjectId')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'ProjectId' -isSecret $false -minLength 4 + # headers placeholder + $hasHeaders = [bool]($armResource.properties.request.PSobject.Properties.name -match "headers") + if ($hasHeaders) + { + foreach ($headerProps in $armResource.properties.request.headers.PsObject.Properties) + { + $headerPropName = $headerProps.Name + $headerPropValue = $headerProps.Value + + if ($headerPropValue.contains("{{")) + { + $placeHoldersMatched = $headerPropValue | Select-String $placeHolderPatternMatches -AllMatches + if ($placeHoldersMatched.Matches.Value.Count -gt 0) + { + $placeHolderName = $placeHoldersMatched.Matches.Value.replace("{{", "").replace("}}", "") + $armResource.properties.request.headers."$headerPropName" = "[[parameters('$($placeHolderName)')]" + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName "$placeHolderName" -isSecret $false + } + } } - } else { - Write-Host "Parameter 'ProjectId' missing from GCP data connector 'request' section." -BackgroundColor Red - exit 1; } +} + +function CreateGCPResourceProperties($armResource, $templateContentConnections) { + $kindType = 'GCP' + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties -propertyName 'connectorDefinitionName' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties -propertyName 'dataType' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties -propertyName 'dcrConfig' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.dcrConfig -propertyName 'streamName' -isInnerObject $true -innerObjectName 'dcrConfig' -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties -propertyName 'auth' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'serviceAccountEmail' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'projectNumber' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $false -isRequired $true -minLength 1 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'workloadIdentityProviderId' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties -propertyName 'request' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.request -propertyName 'projectId' -isInnerObject $true -innerObjectName 'request' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 # Request section subscriptionNames property $hasSubscriptionNames = [bool]($armResource.properties.request.PSobject.Properties.name -match "subscriptionNames") @@ -304,15 +424,88 @@ function Create-GCP($armResource, $templateContentConnections) { $subscriptionNamesValue = $armResource.properties.request.subscriptionNames $placeHoldersMatched = $subscriptionNamesValue | Select-String $placeHolderPatternMatches -AllMatches if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.request.project = "[[parameters('GCPSubscriptionName')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'SubscriptionNames' -isSecret $false -minLength 3 + $placeHolderName = $placeHoldersMatched.Matches.Value.replace("{{", "").replace("}}", "") + $armResource.properties.request.subscriptionNames = @("[[parameters('$($placeHolderName)')]") + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName $placeHolderName -isSecret $false -minLength 3 } } else { - Write-Host "Parameter 'subscriptionNames' missing from GCP data connector 'request' section." -BackgroundColor Red + Write-Host "Error: Attribute 'subscriptionNames' missing from GCP data connector poller 'request' section." -BackgroundColor Red exit 1; } } +function CreateAwsResourceProperties($armResource, $templateContentConnections) { + $kindType = 'AmazonWebServicesS3' + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties -propertyName 'dataTypes' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.dataTypes -propertyName 'logs' -isInnerObject $true -innerObjectName 'dataTypes' -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.dataTypes.logs -propertyName 'state' -isInnerObject $true -innerObjectName 'logs' -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties -propertyName 'dcrConfig' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.dcrConfig -propertyName 'streamName' -isInnerObject $true -innerObjectName 'dcrConfig' -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties -propertyName 'destinationTable' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties -propertyName 'roleArn' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 + + ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties -propertyName 'sqsUrls' -isInnerObject $false -innerObjectName $null -kindType $kindType -isSecret $false -isRequired $true -minLength 3 +} + +function ProcessPropertyPlaceholders($armResource, $templateContentConnections, + $isOnlyObjectCheck, # check if object is present or not example: auth: {'seviceaccountemail'...}. Here check if auth is provided. + $propertyObject, # $armResource.properties + $propertyName, # sqsUrls + $isInnerObject, # false. This propertyName is not inside of any other object or array other than properties. if yes then specify that name below + $innerObjectName, # $null + $kindType, # AWS. This can we GCP or AWS or RestApiPoller + $isSecret, # false + $isRequired, # true, if true then it checks if the field is present or not else give error and exit + $minLength = 1 # default 1 + ) +{ + $placeHolderPatternMatches = '\{{[a-zA-Z0-9]+\}}' + $hasProperty = [bool]($propertyObject.PSobject.Properties.name -match "$($propertyName)") + + if ($hasProperty) + { + if (!$isOnlyObjectCheck) + { + $placeHoldersMatched = $propertyObject.$($propertyName) | Select-String $placeHolderPatternMatches -AllMatches + if ($null -ne $placeHoldersMatched -and $placeHoldersMatched.Matches.Value.Count -gt 0) + { + $placeHolderName = $placeHoldersMatched.Matches.Value.replace("{{", "").replace("}}", "") + $propertyObject.$($propertyName) = "[[parameters('$($placeHolderName)')]" + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName $placeHolderName -isSecret $isSecret -minLength $minLength + } + } + } + else + { + if ($isRequired -and $isInnerObject) + { + Write-Host "Error: Attribute '$($propertyName)' missing from '$($innerObjectName)' Object from $($kindType) data connector poller 'properties' section." -BackgroundColor Red + exit 1; + } + elseif ($isRequired -and $isInnerObject -eq $false) + { + Write-Host "Error: Attribute '$($propertyName)' missing from '$($kindType)' data connector poller 'properties' section." -BackgroundColor Red + exit 1; + } + } +} + +function ProcessPropertyPlaceholders1($armResource, $templateContentConnections, $propertyName, $isSecret, $minLength = 1) +{ + $placeHoldersMatched = $armResource.properties.$($propertyName) | Select-String $placeHolderPatternMatches -AllMatches + if ($placeHoldersMatched.Matches.Value.Count -gt 0) { + $placeHolderName = $placeHoldersMatched.Matches.Value.replace("{{", "").replace("}}", "") + $armResource.properties.dcrConfig.streamName = "[[parameters('$($placeHolderName)')]" + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName $placeHolderName -isSecret $isSecret -minLength $minLength + } +} + # THIS IS THE STARTUP FUNCTION FOR CCP RESOURCE CREATOR function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, $solutionFileMetadata, $dcFolderName, $ccpDict, $solutionBasePath, $solutionName, $ccpTables, $ccpTablesCounter) { Write-Host "Inside of CCP Connector Code!" @@ -435,13 +628,69 @@ function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, exit 1; } + function GetDataConnectorPollerResourceName ($dataConnectorName) { + $splitNamesBySlash = $dataConnectorName -split '/' + $concatenateParts = @() + $outputString = '' + + foreach ($currentName in $splitNamesBySlash) { + if ($currentName.Contains('{{')) { + $placeHolderFieldName = $currentName -replace '{{', '' -replace '}}', '' + $placeHolderMatched = [regex]::Matches($currentName, $placeHolderPatternMatches) + if ($placeHolderMatched.Length -eq $currentName.Length) { + $concatenateParts += "parameters('$($placeHolderFieldName)')" + + if ($placeHolderFieldName -like '*workspace*') { + $templateContentConnections.properties.mainTemplate = addWorkspaceParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName $($placeHolderFieldName) + } else { + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName $($placeHolderFieldName) -isSecret $false + } + } else { + $text = $currentName -replace $placeHolderMatched.Value, '' + $concatenateParts += "'/$($text)'" + $parameterNameValue = $placeHolderMatched.Value -replace '{{', '' -replace '}}', '' + $concatenateParts += "parameters('$($parameterNameValue)')" + + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName $($parameterNameValue) -isSecret $false + } + } else { + if ($currentName.Count -eq 1 -and $currentName -like '{{') { + $concatenateParts = "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/', '$($currentName)')]" + } else { + if ($concatenateParts.Count -ge 1) { + # if we have multiple parts in name {{innerWorkspace}}/Microsoft.SecurityInsights/OktaDCV1_{{domainname}} + $concatenateParts += "'/$($currentName)'" + } else { + # if name is abcwork + $concatenateParts += "$($currentName)" + } + } + } + } + + if ($concatenateParts.Count -gt 1 -and $concatenateParts -notmatch 'concat') { + $outputString = "[[concat($($concatenateParts -join ', '))]" + } elseif ($concatenateParts.Count -eq 1 -and $concatenateParts[0] -match 'parameters') { + # if we just have parameters('abcwork') + $outputString = "[[concat(parameters('workspace'),'/Microsoft.SecurityInsights/', $($concatenateParts[0]))]" + } else { + # if we just have 'abcwork' + $outputString = "[[concat(parameters('workspace'),'/Microsoft.SecurityInsights/', '$($concatenateParts[0])')]" + } + + if ($outputString -like "*parameters('workspace')*") { + $outputString = $outputString.Replace("[[", "[") + } + + return $outputString + } + function CCPDataConnectorsResource($fileContent) { if($fileContent.type -eq "Microsoft.SecurityInsights/dataConnectors") { Write-Host "Processing for CCP Poller file path: $ccpPollerFilePath" - $dataConnectorPollerName = $null -eq $fileContent.Name -or $fileContent.Name -eq '' ? $fileContent.properties.connectorDefinitionName : $fileContent.Name; - $resourceName = "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/', '$dataConnectorPollerName')]" - #$resourceName = "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/', '$templateName')]" + $resourceName = GetDataConnectorPollerResourceName -dataConnectorName $fileContent.name + $armResource = Get-ArmResource $resourceName $fileContent.type $fileContent.kind $fileContent.properties $armResource.type = "Microsoft.OperationalInsights/workspaces/providers/dataConnectors" $armResource.kind = $ccpItem.PollerKind; @@ -474,156 +723,24 @@ function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, $armResource.properties.dcrConfig | Add-Member -MemberType NoteProperty -Name "dataCollectionRuleImmutableId" -Value "[[parameters('dcrConfig').dataCollectionRuleImmutableId]" } - # Start: GCP CODE - if ($armResource.kind.ToLower() -eq 'gcp'){ - Create-GCP -armResource $armResource -templateContentConnections $templateContentConnections - } - # End: GCP CODE - elseif($armResource.properties.auth.type.ToLower() -eq 'oauth2') + if ($armResource.kind.ToLower() -eq 'gcp') { - # clientid - $hasClientId = [bool]($armResource.properties.auth.PSobject.Properties.name -match "ClientId") - if ($hasClientId) { - $clientIdProperty = $armResource.properties.auth.ClientId - $placeHoldersMatched = $clientIdProperty | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.ClientId = "[[parameters('ClientId')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'ClientId' -isSecret $true - } - } - - # client secret - $hasClientSecretId = [bool]($armResource.properties.auth.PSobject.Properties.name -match "ClientSecret") - if ($hasClientSecretId) { - $clientSecretIdProperty = $armResource.properties.auth.ClientSecret - $placeHoldersMatched = $clientSecretIdProperty | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.ClientSecret = "[[parameters('ClientSecret')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'ClientSecret' -isSecret $true - } - } - - # authorization code - if($armResource.properties.auth.grantType -eq 'authorization_code') { - $hasAuthorizationCode = [bool]($armResource.properties.auth.PSobject.Properties.name -match "AuthorizationCode") - if ($hasAuthorizationCode) { - $authorizationCodeProperty = $armResource.properties.auth.AuthorizationCode - $placeHoldersMatched = $authorizationCodeProperty | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.AuthorizationCode = "[[parameters('AuthorizationCode')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'AuthorizationCode' -isSecret $true - } - } - } - - # AuthorizationEndpoint placeholder - if ($null -ne $armResource.properties.auth.AuthorizationEndpoint -and $armResource.properties.request.auth.AuthorizationEndpoint.contains("{{")) { - $authorizationEndpointValue = $armResource.properties.auth.AuthorizationEndpoint - $placeHoldersMatched = $authorizationEndpointValue | Select-String $placeHolderPatternMatches -AllMatches - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.request.AuthorizationEndpoint = "[[parameters('AuthorizationEndpoint')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'AuthorizationEndpoint' -isSecret $false - } - } - - # TokenEndpoint placeholder - if ($null -ne $armResource.properties.auth.TokenEndpoint -and $armResource.properties.auth.TokenEndpoint.contains("{{")) { - $tokenEndpointValue = $armResource.properties.auth.TokenEndpoint - $placeHoldersMatched = $tokenEndpointValue | Select-String $placeHolderPatternMatches -AllMatches - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.TokenEndpoint = "[[parameters('TokenEndpoint')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'TokenEndpoint' -isSecret $false - } - } + CreateGCPResourceProperties -armResource $armResource -templateContentConnections $templateContentConnections } - elseif ($armResource.properties.auth.type.ToLower() -eq 'basic') { - # username - $hasUsername = [bool]($armResource.properties.auth.PSobject.Properties.name -match "username") - if ($hasUsername) { - $usernameProperty = $armResource.properties.auth.username - $placeHoldersMatched = $usernameProperty | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.username = "[[parameters('username')]" - - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'username' -isSecret $false - } - } - - # password - $hasPassword = [bool]($armResource.properties.auth.PSobject.Properties.name -match "password") - if ($hasPassword) { - $passwordProperty = $armResource.properties.auth.password - $placeHoldersMatched = $passwordProperty | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.password = "[[parameters('password')]" - - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'password' -isSecret $true - } - } + elseif ($armResource.kind.ToLower() -eq 'restapipoller') + { + CreateRestApiPollerResourceProperties -armResource $armResource -templateContentConnections $templateContentConnections } - elseif ($armResource.properties.auth.type.ToLower() -eq 'apikey') { - # ApiKey - $hasApiKey = [bool]($armResource.properties.auth.PSobject.Properties.name -match "ApiKey") - if ($hasApiKey) { - $apiKeyProperty = $armResource.properties.auth.ApiKey - $placeHoldersMatched = $apiKeyProperty | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $armResource.properties.auth.ApiKey = "[[parameters('apikey')]" - - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName 'apikey' -isSecret $true - } - } + elseif ($armResource.kind.ToLower() -eq 'amazonwebservicess3') + { + CreateAwsResourceProperties -armResource $armResource -templateContentConnections $templateContentConnections } - - if ($armResource.properties.request.PSObject.Properties["apiEndPoint"] -and - $armResource.properties.request.apiEndPoint.contains("{{")) { - # identify any placeholders in apiEndpoint - $endPointUrl = $armResource.properties.request.apiEndPoint - $placeHoldersMatched = $endPointUrl | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - # has some placeholders - $finalizedEndpointUrl = "" - $finalizedEndpointUrl = "[[concat(" - $closureBrackets = ")]" - - foreach ($currentPlaceHolder in $placeHoldersMatched.Matches.Value) { - $placeHolderName = $currentPlaceHolder.replace("{{", "").replace("}}", "") - $splitEndpoint = $endPointUrl -split "($currentPlaceHolder)" - $commaCount = 0 - foreach($splitItem in $splitEndpoint) { - if ($splitItem -eq $currentPlaceHolder) { - if ($finalizedEndpointUrl -eq '') { - $finalizedEndpointUrl += "parameters('" + $placeHolderName + "')" - } else { - $finalizedEndpointUrl += ", parameters('" + $placeHolderName + "')" - } - - if ($placeHolderName.Contains("secret") -or $placeHolderName.Contains("password")) { - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName "$placeHolderName" -isSecret $true - } else { - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName "$placeHolderName" -isSecret $false - } - } else { - if ($commaCount -eq 0) { - $finalizedEndpointUrl += "'"+ $splitItem + "'" - $commaCount += 1 - } else { - $finalizedEndpointUrl += ", '" + $splitItem + "'" - } - } - } - } - - $armResource.properties.request.apiEndPoint = $finalizedEndpointUrl + $closureBrackets - } + else + { + Write-Host "Error: For kind RestApiPoller, Data Connector Poller file should have 'kind' attribute with value either 'RestApiPoller', 'GCP' or 'AmazonWebServicesS3'" -BackgroundColor Red + exit 1; } + $templateContentConnections.properties.mainTemplate.resources += $armResource } } @@ -651,9 +768,23 @@ function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, if($fileContent.type -eq "Microsoft.Insights/dataCollectionRules") { Write-Host "Processing for CCP DCR file path: $ccpDCRFilePath" - foreach ($logAnalyticDestination in $fileContent.properties.destinations.logAnalytics) - { - $logAnalyticDestination.workspaceResourceId = "[variables('workspaceResourceId')]" + if (-not $fileContent.properties.destinations.PSObject.Properties.Name -contains "logAnalytics") { + # if logAnalytics array is not specified + $logAnalyticsObject = @{ + "name" = "clv2ws1" + "workspaceResourceId" = "[variables('workspaceResourceId')]" + } + $fileContent.properties.destinations | Add-Member -MemberType NoteProperty -Name "logAnalytics" -Value $logAnalyticsObject + } else { + if (-not $fileContent.properties.destinations.logAnalytics[0].PSObject.Properties.Name -contains "name") { + $fileContent.properties.destinations.logAnalytics[0] | Add-Member -MemberType NoteProperty -Name "name" -Value "clv2ws1" + } + + if ($fileContent.properties.destinations.logAnalytics[0].PSObject.Properties.Name -contains "workspaceResourceId") { + $fileContent.properties.destinations.logAnalytics[0].workspaceResourceId = "[variables('workspaceResourceId')]" + } else { + $fileContent.properties.destinations.logAnalytics[0] | Add-Member -MemberType NoteProperty -Name "workspaceResourceId" -Value "[variables('workspaceResourceId')]" + } } $dcrPlaceHolderMatched = $fileContent.name | Select-String $placeHolderPatternMatches -AllMatches @@ -746,24 +877,25 @@ function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, exit 1; } - if($fileContent.type -eq "Microsoft.OperationalInsights/workspaces/tables") - { - $resourceName = $fileContent.name - $armResource = Get-ArmResource $resourceName $fileContent.type $fileContent.kind $fileContent.properties - - $hasLocationProperty = [bool]($armResource.PSobject.Properties.name -match "location") - if ($hasLocationProperty) { - $locationProperty = $armResource.location - $placeHoldersMatched = $locationProperty | Select-String $placeHolderPatternMatches -AllMatches - - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $placeHolderName = $placeHoldersMatched.Matches.Value.replace("{{", "").replace("}}", "") - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName $placeHolderName -isSecret $false + foreach ($tableContent in $fileContent) { + if($fileContent.type -eq "Microsoft.OperationalInsights/workspaces/tables") { + $resourceName = $tableContent.name + $armResource = Get-ArmResource $resourceName $tableContent.type $tableContent.kind $tableContent.properties + + $hasLocationProperty = [bool]($armResource.PSobject.Properties.name -match "location") + if ($hasLocationProperty) { + $locationProperty = $armResource.location + $placeHoldersMatched = $locationProperty | Select-String $placeHolderPatternMatches -AllMatches + + if ($placeHoldersMatched.Matches.Value.Count -gt 0) { + $placeHolderName = $placeHoldersMatched.Matches.Value.replace("{{", "").replace("}}", "") + $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName $placeHolderName -isSecret $false + } } + + $templateContentConnectorDefinition.properties.mainTemplate.resources += $armResource + $tableCounter ++; } - - $templateContentConnectorDefinition.properties.mainTemplate.resources += $armResource - $tableCounter ++; } } @@ -862,6 +994,6 @@ function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, } } catch { - Write-Host "Error occured in createCCPConnector file. Error Details $_" + Write-Host "Error occurred in createCCPConnector file. Error Details $_" } } \ No newline at end of file diff --git a/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 index b056a8c7dc7..6a2b0b7c130 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 @@ -198,7 +198,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } } catch { - Write-Host "Error occured while identifying relation between definition and poller File. Identified error in " + $inputFile.Name + ". Error Details : $_" + Write-Host "Error occurred while identifying relation between definition and poller File. Identified error in " + $inputFile.Name + ". Error Details : $_" } } } @@ -220,19 +220,57 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } try { - if($fileContent.type -eq "Microsoft.Insights/dataCollectionRules" -and $fileContent.properties.dataFlows.streams -is [System.Object[]]) { + if($fileContent.type -eq "Microsoft.Insights/dataCollectionRules" -and $fileContent.properties.dataFlows.streams -is [System.String]) { $dataFlowStreams = $fileContent.properties.dataFlows; foreach ($dcrDataFlowStream in $dataFlowStreams) { - if ($dcrDataFlowStream.streams[0] -eq $ccpPollerFile.DCPollerStreamName) { - $ccpPollerFile.DCRFilePath = $inputFile.FullName - if ($fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { - $ccpPollerFile.TableOutputStream = $dcrDataFlowStream.outputStream.Replace('Custom-', '') - $ccpPollerFile.DCROutputStream = $dcrDataFlowStream.outputStream + $dcrStreamName = $dcrDataFlowStream.streams[0] + if ($dcrStreamName.contains('Microsoft-')) { + # this is a standard table + . "$PSScriptRoot/standardLogStreams.ps1" # load standard data connector poller and dcr mapper + $dcPollerStreamNameValue = GetKeyValue -key $ccpPollerFile.DCPollerStreamName + if ($null -eq $dcPollerStreamNameValue) { + # not found in mapping + Write-Host "Error For given CCP Data Connector poller, mapping not found in StandardLogStreams. Please make sure that StreamName in Data Connector poller file and that in DCR Stream name are correct. Refer StandardLogStreams.ps1 file for Standard mapping." -BackgroundColor Red + exit 1; + } else { + if ($dcrDataFlowStream.streams[0] -ne $dcPollerStreamNameValue) { + # when dc poller streamname expected mapping value is not eqault to dcr file stream name + Write-Host "Error For given CCP, DCR file 'stream' value is not correct. Please make sure that StreamName in Data Connector poller file and that in DCR Stream name are correct. Refer StandardLogStreams.ps1 file for Standard mapping." -BackgroundColor Red + exit 1; + } else { + $ccpPollerFile.DCRFilePath = $inputFile.FullName + } + } + } else { + # this is a custom table + if ($dcrDataFlowStream.streams[0] -eq $ccpPollerFile.DCPollerStreamName) { + $ccpPollerFile.DCRFilePath = $inputFile.FullName + if ($fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { + $ccpPollerFile.TableOutputStream = $dcrDataFlowStream.outputStream.Replace('Custom-', '') + $ccpPollerFile.DCROutputStream = $dcrDataFlowStream.outputStream + } } } } } else { if($fileContent.type -eq "Microsoft.Insights/dataCollectionRules") { + if ($fileContent.properties.dataFlows.Count -gt 1) { + foreach ($dataFlowItem in $fileContent.properties.dataFlows) { + $dataFlowStreamName = $dataFlowItem.streams[0] + $dataFlowOutputStreamName = $dataFlowItem.outputStream + + if ($dataFlowStreamName -eq $ccpPollerFile.DCPollerStreamName) { + $ccpPollerFile.DCRFilePath = $inputFile.FullName + + if ($dataFlowOutputStreamName.contains('Custom-')) { + $ccpPollerFile.TableOutputStream = $dataFlowStreamName.Replace('Custom-', '') + $ccpPollerFile.DCROutputStream = $dataFlowOutputStreamName + } + + break + } + } + } if ($fileContent.properties.dataFlows[0].streams[0] -eq $ccpPollerFile.DCPollerStreamName) { $ccpPollerFile.DCRFilePath = $inputFile.FullName if ($fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { @@ -244,7 +282,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } } catch { - Write-Host "Error occured while identifying relation between Poller and DCR File. Identified error in " + $inputFile.Name + ". Error Details : $_" + Write-Host "Error occurred while identifying relation between Poller and DCR File. Identified error in " + $inputFile.Name + ". Error Details : $_" } } } @@ -283,7 +321,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } } catch { - Write-Host "Error occured while identifying relation between DCR and Table File. Identified error in " + $inputFile.Name + ". Error Details : $_" + Write-Host "Error occurred while identifying relation between DCR and Table File. Identified error in " + $inputFile.Name + ". Error Details : $_" } } } @@ -293,10 +331,11 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold # THROW ERROR IF THERE IS NO RELATION BETWEEN DEFINITION->POLLER AND/OR POLLER->DCR if ($ccpDict.Count -gt 0) { foreach($localCCPDist in $ccpDict) { - if ($localCCPDist.DCDefinitionId -eq "" -or $localCCPDist.DCDefinitionFilePath -eq "" -or - $localCCPDist.DCPollerFilePath -eq "" -or $localCCPDist.DCPollerStreamName -eq "" -or $localCCPDist.DCRFilePath -eq "") + if (($localCCPDist.DCDefinitionId -eq "" -or $localCCPDist.DCDefinitionFilePath -eq "" -or + $localCCPDist.DCPollerFilePath -eq "" -or $localCCPDist.DCPollerStreamName -eq "" -or $localCCPDist.DCRFilePath -eq "") -and + $localCCPDist.PollerKind -eq 'RestApiPoller') { - Write-Host "Please verify if there is a mapping between ConnectorDefiniton with Poller file and/or Poller file with DCR file! If mapping is correct then check type property for ccp files!" + Write-Host "Please verify if there is a mapping between ConnectorDefinition with Poller file and/or Poller file with DCR file! If mapping is correct then check type property for ccp files!" exit 1 } } diff --git a/Tools/Create-Azure-Sentinel-Solution/common/standardLogStreams.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/standardLogStreams.ps1 new file mode 100644 index 00000000000..174f47035db --- /dev/null +++ b/Tools/Create-Azure-Sentinel-Solution/common/standardLogStreams.ps1 @@ -0,0 +1,25 @@ +# Standard Tables in Microsoft Sentinel mapping for Scuba and Streams. +$standardStreamMapping = @() + + # key is the Data connector poller StreamName and value is the DCR file streamName + # Example: For GCP audit, data connector poller file, 'StreamName' should be 'SENTINEL_GCP_AUDIT_LOGS' and in dcr file 'stream' should be 'Microsoft-GCPAuditLogs'. Here SENTINEL_GCP_AUDIT_LOGS is used for Scuba and for table Microsoft-GCPAuditLogs is used. + +$standardStreamMapping += @{ Key = 'SENTINEL_GCP_FIREWALL_LOGS'; Value = 'Microsoft-GCPFirewallLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_GCP_AUDIT_LOGS'; Value = 'Microsoft-GCPAuditLogs'} + +# Function to check if a key exists in the array of hashtables +function GetKeyValue { + param ( + [string]$key + ) + + # Iterate through each hashtable in the array + foreach ($pair in $standardStreamMapping) { + # Explicitly check if the 'Key' property matches the key you're looking for + if ($pair["Key"] -eq $key) { + return $pair["Value"] # Key found + } + } + + return $null # Key not found +} \ No newline at end of file From 4325973858d9bc4ae9b3e1e22d00ebe9f678d636 Mon Sep 17 00:00:00 2001 From: v-amolpatil Date: Mon, 13 Jan 2025 21:38:46 +0530 Subject: [PATCH 3/4] code customization --- .../common/createCCPConnector.ps1 | 32 +------------------ 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 index 5c782d8a62d..e5248a81918 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 @@ -288,29 +288,9 @@ function CreateRestApiPollerResourceProperties() { # ApiKey ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $false -propertyObject $armResource.properties.auth -propertyName 'apikey' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $true -isRequired $true -minLength 4 } - elseif($armResource.properties.auth.type.ToLower() -eq 'JwtToken') - { - # userName object check - ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth -propertyName 'userName' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 - - # userName object should have key - ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth.userName -propertyName 'key' -isInnerObject $true -innerObjectName 'userName' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 - - # userName object should have value - ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth.userName -propertyName 'value' -isInnerObject $true -innerObjectName 'userName' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 - - # password object check - ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth -propertyName 'password' -isInnerObject $true -innerObjectName 'auth' -kindType $kindType -isSecret $true -isRequired $true -minLength 4 - - # password object should have key - ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth.password -propertyName 'key' -isInnerObject $true -innerObjectName 'password' -kindType $kindType -isSecret $false -isRequired $true -minLength 4 - - # password object should have value - ProcessPropertyPlaceholders -armResource $armResource -templateContentConnections $templateContentConnections -isOnlyObjectCheck $true -propertyObject $armResource.properties.auth.password -propertyName 'value' -isInnerObject $true -innerObjectName 'password' -kindType $kindType -isSecret $true -isRequired $true -minLength 4 - } else { - Write-Host "Error: For kind RestApiPoller, Data Connector Poller file should have 'auth' object with 'type' attribute having value either 'Basic', 'OAuth2', 'APIKey' or 'JwtToken'" -BackgroundColor Red + Write-Host "Error: For kind RestApiPoller, Data Connector Poller file should have 'auth' object with 'type' attribute having value either 'Basic', 'OAuth2' or 'APIKey'." -BackgroundColor Red exit 1; } @@ -496,16 +476,6 @@ function ProcessPropertyPlaceholders($armResource, $templateContentConnections, } } -function ProcessPropertyPlaceholders1($armResource, $templateContentConnections, $propertyName, $isSecret, $minLength = 1) -{ - $placeHoldersMatched = $armResource.properties.$($propertyName) | Select-String $placeHolderPatternMatches -AllMatches - if ($placeHoldersMatched.Matches.Value.Count -gt 0) { - $placeHolderName = $placeHoldersMatched.Matches.Value.replace("{{", "").replace("}}", "") - $armResource.properties.dcrConfig.streamName = "[[parameters('$($placeHolderName)')]" - $templateContentConnections.properties.mainTemplate = addNewParameter -templateResourceObj $templateContentConnections.properties.mainTemplate -parameterName $placeHolderName -isSecret $isSecret -minLength $minLength - } -} - # THIS IS THE STARTUP FUNCTION FOR CCP RESOURCE CREATOR function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, $solutionFileMetadata, $dcFolderName, $ccpDict, $solutionBasePath, $solutionName, $ccpTables, $ccpTablesCounter) { Write-Host "Inside of CCP Connector Code!" From f58e203825019d5991d46897e8bacf92c5985da3 Mon Sep 17 00:00:00 2001 From: v-amolpatil Date: Wed, 15 Jan 2025 19:31:53 +0530 Subject: [PATCH 4/4] updated code to add standard table mapping. --- .../common/get-ccp-details.ps1 | 22 ++--- .../common/standardLogStreams.ps1 | 91 ++++++++++++++++++- 2 files changed, 100 insertions(+), 13 deletions(-) diff --git a/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 index 6a2b0b7c130..398509ed36a 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 @@ -224,24 +224,24 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold $dataFlowStreams = $fileContent.properties.dataFlows; foreach ($dcrDataFlowStream in $dataFlowStreams) { $dcrStreamName = $dcrDataFlowStream.streams[0] - if ($dcrStreamName.contains('Microsoft-')) { - # this is a standard table + + if (!$dcrStreamName.Contains('Custom-')) { . "$PSScriptRoot/standardLogStreams.ps1" # load standard data connector poller and dcr mapper $dcPollerStreamNameValue = GetKeyValue -key $ccpPollerFile.DCPollerStreamName + if ($null -eq $dcPollerStreamNameValue) { # not found in mapping - Write-Host "Error For given CCP Data Connector poller, mapping not found in StandardLogStreams. Please make sure that StreamName in Data Connector poller file and that in DCR Stream name are correct. Refer StandardLogStreams.ps1 file for Standard mapping." -BackgroundColor Red + Write-Host "Error For given CCP Data Connector poller, mapping not found in StandardLogStreams. Please make sure that StreamName in Data Connector poller file and that in DCR Stream name are correct. Refer StandardLogStreams.ps1 file for Standard mapping. Refer Tools/Create-Azure-Sentinel-Solution/common/StandardLogStreams.ps1 file for Standard mapping." -BackgroundColor Red + exit 1; + } elseif ($dcrDataFlowStream.streams[0] -ne $dcPollerStreamNameValue) { + # when dc poller streamname expected mapping value is not equal to dcr file stream name + Write-Host "Error For given CCP, DCR file 'stream' value is not correct. Please make sure that StreamName in Data Connector poller file and that in DCR Stream name are correct. Refer Tools/Create-Azure-Sentinel-Solution/common/StandardLogStreams.ps1 file for Standard mapping." -BackgroundColor Red exit 1; } else { - if ($dcrDataFlowStream.streams[0] -ne $dcPollerStreamNameValue) { - # when dc poller streamname expected mapping value is not eqault to dcr file stream name - Write-Host "Error For given CCP, DCR file 'stream' value is not correct. Please make sure that StreamName in Data Connector poller file and that in DCR Stream name are correct. Refer StandardLogStreams.ps1 file for Standard mapping." -BackgroundColor Red - exit 1; - } else { - $ccpPollerFile.DCRFilePath = $inputFile.FullName - } + $ccpPollerFile.DCRFilePath = $inputFile.FullName } - } else { + } + else { # this is a custom table if ($dcrDataFlowStream.streams[0] -eq $ccpPollerFile.DCPollerStreamName) { $ccpPollerFile.DCRFilePath = $inputFile.FullName diff --git a/Tools/Create-Azure-Sentinel-Solution/common/standardLogStreams.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/standardLogStreams.ps1 index 174f47035db..997ff34e692 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/standardLogStreams.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/standardLogStreams.ps1 @@ -2,10 +2,97 @@ $standardStreamMapping = @() # key is the Data connector poller StreamName and value is the DCR file streamName - # Example: For GCP audit, data connector poller file, 'StreamName' should be 'SENTINEL_GCP_AUDIT_LOGS' and in dcr file 'stream' should be 'Microsoft-GCPAuditLogs'. Here SENTINEL_GCP_AUDIT_LOGS is used for Scuba and for table Microsoft-GCPAuditLogs is used. + # Example: For GCP audit, data connector poller file, 'StreamName' should be 'SENTINEL_GCP_AUDIT_LOGS' and in dcr file 'stream' should be 'Microsoft-GCPAuditLogs' which is your standard table name. Here SENTINEL_GCP_AUDIT_LOGS is used for Scuba. -$standardStreamMapping += @{ Key = 'SENTINEL_GCP_FIREWALL_LOGS'; Value = 'Microsoft-GCPFirewallLogs'} +$standardStreamMapping += @{ Key = 'SAP_ABAPAUDITLOG'; Value = 'Microsoft-ABAPAuditLog'} +$standardStreamMapping += @{ Key = 'SECURITY_ALERT_DATA'; Value = 'Microsoft-SecurityAlert'} +$standardStreamMapping += @{ Key = 'SENTINEL_SECURITY_ALERT_DATA'; Value = 'Microsoft-SecurityAlert'} +$standardStreamMapping += @{ Key = 'LINUX_NAGIOSALERTS_BLOB'; Value = 'Microsoft-Alert'} +$standardStreamMapping += @{ Key = 'OMSALERTS_BLOB'; Value = 'Microsoft-Alert'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGALERTEVIDENCE'; Value = 'Microsoft-AlertEvidence'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGALERTINFO'; Value = 'Microsoft-AlertInfo'} +$standardStreamMapping += @{ Key = 'SENTINEL_ANOMALIES'; Value = 'Microsoft-Anomalies'} +$standardStreamMapping += @{ Key = 'APP_CENTER_ERROR_DATA'; Value = 'Microsoft-AppCenterError'} +$standardStreamMapping += @{ Key = 'SENTINEL_AUDIT_EVENTS'; Value = 'Microsoft-ASimAuditEventLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_AUTHENTICATION_EVENTS'; Value = 'Microsoft-ASimAuthenticationEventLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_DHCP_EVENTS'; Value = 'Microsoft-ASimDhcpEventLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_WINDOWS_DNS_EVENTS'; Value = 'Microsoft-ASimDnsActivityLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_FILE_EVENT_LOGS'; Value = 'Microsoft-ASimFileEventLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_NETWORK_SESSION_NORMALIZED'; Value = 'Microsoft-ASimNetworkSessionLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_NETWORK_SESSION_WINDOWS_FIREWALL_AMA'; Value = 'Microsoft-ASimNetworkSessionLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_PROCESS_EVENTS'; Value = 'Microsoft-ASimProcessEventLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_REGISTRY_EVENTS'; Value = 'Microsoft-ASimRegistryEventLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_USER_MANAGEMENT_ACTIVITY_LOGS'; Value = 'Microsoft-ASimUserManagementActivityLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_WEB_SESSION_LOGS'; Value = 'Microsoft-ASimWebSessionLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_AWSCLOUDTRAIL'; Value = 'Microsoft-AWSCloudTrail'} +$standardStreamMapping += @{ Key = 'SENTINEL_AWSCLOUDWATCH'; Value = 'Microsoft-AWSCloudWatch'} +$standardStreamMapping += @{ Key = 'SENTINEL_AWSGUARDDUTY'; Value = 'Microsoft-AWSGuardDuty'} +$standardStreamMapping += @{ Key = 'SENTINEL_AWSVPCFLOW'; Value = 'Microsoft-AWSVPCFlow'} +$standardStreamMapping += @{ Key = 'SENTINEL_AWSWAF'; Value = 'Microsoft-AWSWAF'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGCLOUDAPPEVENTS'; Value = 'Microsoft-CloudAppEvents'} +$standardStreamMapping += @{ Key = 'SECURITY_CISCO_ASA_BLOB'; Value = 'Microsoft-CommonSecurityLog'} +$standardStreamMapping += @{ Key = 'SECURITY_CEF_BLOB'; Value = 'Microsoft-CommonSecurityLog'} +$standardStreamMapping += @{ Key = 'COMPUTER_GROUP_BLOB'; Value = 'Microsoft-ComputerGroup'} +$standardStreamMapping += @{ Key = 'SENTINEL_WATCHLIST'; Value = 'Microsoft-Watchlist'} +$standardStreamMapping += @{ Key = 'OFFICEDATAVERSE_RESTAPI'; Value = 'Microsoft-DataverseActivity'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICEEVENTS'; Value = 'Microsoft-DeviceEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICEFILECERTIFICATEINFO'; Value = 'Microsoft-DeviceFileCertificateInfo'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICEFILEEVENTS'; Value = 'Microsoft-DeviceFileEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICEIMAGELOADEVENTS'; Value = 'Microsoft-DeviceImageLoadEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICEINFO'; Value = 'Microsoft-DeviceInfo'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICELOGONEVENTS'; Value = 'Microsoft-DeviceLogonEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICENETWORKEVENTS'; Value = 'Microsoft-DeviceNetworkEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICENETWORKINFO'; Value = 'Microsoft-DeviceNetworkInfo'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICEPROCESSEVENTS'; Value = 'Microsoft-DeviceProcessEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGDEVICEREGISTRYEVENTS'; Value = 'Microsoft-DeviceRegistryEvents'} +$standardStreamMapping += @{ Key = 'TVM_DEVICETVMSECURECONFIGURATIONASSESSMENT'; Value = 'Microsoft-DeviceTvmSecureConfigurationAssessment'} +$standardStreamMapping += @{ Key = 'TVM_DEVICETVMSECURECONFIGURATIONASSESSMENT_KB'; Value = 'Microsoft-DeviceTvmSecureConfigurationAssessmentKB'} +$standardStreamMapping += @{ Key = 'TVM_DEVICETVMSOFTWAREINVENTORY'; Value = 'Microsoft-DeviceTvmSoftwareInventory'} +$standardStreamMapping += @{ Key = 'TVM_DEVICETVMSOFTWAREVULNERABILITIES'; Value = 'Microsoft-DeviceTvmSoftwareVulnerabilities'} +$standardStreamMapping += @{ Key = 'TVM_DEVICETVMSOFTWAREVULNERABILITIES_KB'; Value = 'Microsoft-DeviceTvmSoftwareVulnerabilitiesKB'} +$standardStreamMapping += @{ Key = 'SENTINEL_WINDOWS_DNS_AUDIT_EVENTS'; Value = 'Microsoft-DnsAuditEvents'} +$standardStreamMapping += @{ Key = 'DNS_ANALYTICS_BLOB'; Value = 'Microsoft-DnsEvents'} +$standardStreamMapping += @{ Key = 'DNS_AUDIT_BLOB'; Value = 'Microsoft-DnsEvents'} +$standardStreamMapping += @{ Key = 'DNS_DYNAMIC_BLOB'; Value = 'Microsoft-DnsEvents'} +$standardStreamMapping += @{ Key = 'DNS_INVENTORY_BLOB'; Value = 'Microsoft-DnsInventory'} +$standardStreamMapping += @{ Key = 'DYNAMICS365_ACTIVITY_RESTAPI'; Value = 'Microsoft-Dynamics365Activity'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGEMAILATTACHMENTINFO'; Value = 'Microsoft-EmailAttachmentInfo'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGEMAILEVENTS'; Value = 'Microsoft-EmailEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGEMAILPOSTDELIVERYEVENTS'; Value = 'Microsoft-EmailPostDeliveryEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGEMAILURLINFO'; Value = 'Microsoft-EmailUrlInfo'} $standardStreamMapping += @{ Key = 'SENTINEL_GCP_AUDIT_LOGS'; Value = 'Microsoft-GCPAuditLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_GCP_FIREWALL_LOGS'; Value = 'Microsoft-GCPFirewallLogs'} +$standardStreamMapping += @{ Key = 'SENTINEL_GOOGLE_CLOUD_SCC'; Value = 'Microsoft-GoogleCloudSCC'} +$standardStreamMapping += @{ Key = 'HUNTING_BOOKMARKS_LOGANALYTICS'; Value = 'Microsoft-HuntingBookmark'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGIDENTITYDIRECTORYEVENTS'; Value = 'Microsoft-IdentityDirectoryEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGIDENTITYLOGONEVENTS'; Value = 'Microsoft-IdentityLogonEvents'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGIDENTITYQUERYEVENTS'; Value = 'Microsoft-IdentityQueryEvents'} +$standardStreamMapping += @{ Key = 'INSIGHTS_METRICS_BLOB'; Value = 'Microsoft-InsightsMetrics'} +$standardStreamMapping += @{ Key = 'LINUX_AUDITD_BLOB'; Value = 'Microsoft-LinuxAuditLog'} +$standardStreamMapping += @{ Key = 'McasShadowItReportingId'; Value = 'Microsoft-McasShadowItReporting'} +$standardStreamMapping += @{ Key = 'MICROSOFTPURVIEWINFORMATIONPROTECTION_RESTAPI'; Value = 'Microsoft-MicrosoftPurviewInformationProtection'} +$standardStreamMapping += @{ Key = 'OFFICEACTIVITY_RESTAPI'; Value = 'Microsoft-OfficeActivity'} +$standardStreamMapping += @{ Key = 'OPERATION_JSON'; Value = 'Microsoft-Operation'} +$standardStreamMapping += @{ Key = 'OPERATION_BLOB'; Value = 'Microsoft-Operation'} +$standardStreamMapping += @{ Key = 'OFFICEPOWERAPPS_RESTAPI'; Value = 'Microsoft-PowerAppsActivity'} +$standardStreamMapping += @{ Key = 'OFFICEPOWERAUTOMATE_RESTAPI'; Value = 'Microsoft-PowerAutomateActivity'} +$standardStreamMapping += @{ Key = 'OFFICEPOWERBI_RESTAPI'; Value = 'Microsoft-PowerBIActivity'} +$standardStreamMapping += @{ Key = 'OFFICEPOWERPLATFORMADMIN_RESTAPI'; Value = 'Microsoft-PowerPlatformAdminActivity'} +$standardStreamMapping += @{ Key = 'OFFICEPOWERAPPSRESOURCE_RESTAPI'; Value = 'Microsoft-PowerPlatformConnectorActivity'} +$standardStreamMapping += @{ Key = 'OFFICEPOWERPLATFORMADMINDLP_RESTAPI'; Value = 'Microsoft-PowerPlatformDlpActivity'} +$standardStreamMapping += @{ Key = 'OFFICEPROJECT_RESTAPI'; Value = 'Microsoft-ProjectActivity'} +$standardStreamMapping += @{ Key = 'SECURITY_ALERT_DATA'; Value = 'Microsoft-SecurityAlert'} +$standardStreamMapping += @{ Key = 'SENTINEL_SECURITY_ALERT_DATA'; Value = 'Microsoft-SecurityAlert'} +$standardStreamMapping += @{ Key = 'SECURITY_EVENT_BLOB'; Value = 'Microsoft-SecurityEvent'} +$standardStreamMapping += @{ Key = 'SECURITY_INCIDENT_DATA'; Value = 'Microsoft-SecurityIncident'} +$standardStreamMapping += @{ Key = 'SENTINEL_HEALTH'; Value = 'Microsoft-SentinelHealth'} +$standardStreamMapping += @{ Key = 'THREAT_INTELLIGENCE_INDICATOR_DATA'; Value = 'Microsoft-ThreatIntelligenceIndicator'} +$standardStreamMapping += @{ Key = 'TENANTMICROSOFTWINDOWSDEFENDERATP_ADVANCEDHUNTINGURLCLICKEVENTS'; Value = 'Microsoft-UrlClickEvents'} +$standardStreamMapping += @{ Key = 'USAGE_METERING'; Value = 'Microsoft-Usage'} +$standardStreamMapping += @{ Key = 'SENTINEL_WATCHLIST'; Value = 'Microsoft-Watchlist'} +$standardStreamMapping += @{ Key = 'SECURITY_WEF_EVENT_BLOB'; Value = 'Microsoft-WindowsEvent'} +$standardStreamMapping += @{ Key = 'SECURITY_WEF_EVENT_BLOB_OBO'; Value = 'Microsoft-WindowsEvent'} + # Function to check if a key exists in the array of hashtables function GetKeyValue {