Merge branch 'master' into anknar/SyslogWorkbook

.github/workflows/runAsimSchemaAndDataTesters.yaml

@@ -1,6 +1,6 @@
 # Each pull request that updates ASIM parsers triggers the script.
 # The script runs ASIM Schema and Data testers on the "eco-connector-test" workspace.
-name: Run ASIM testers on "ASIM-SchemaDataTester-GithubShared" workspace
+name: Run ASIM tests on "ASIM-SchemaDataTester-GithubShared" workspace
 on:
   pull_request_target:
     types: [opened, edited, reopened, synchronize, labeled]
@@ -26,7 +26,8 @@ permissions:
   contents: read
 
 jobs: 
-  run-asim-testers:
+  Run-ASim-Schema-Data-tests:
+    name: Run ASim Schema and Data tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout pull request branch
@@ -36,6 +37,14 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           persist-credentials: false
           fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
+
+      - name: Login to Azure Public Cloud with AzPowershell
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_ASIM_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          enable-AzPSSession: true
+          allow-no-subscriptions: true
 
       - name: Setup git config
         run: |
@@ -51,20 +60,121 @@ jobs:
             git merge --abort
             exit 1
           fi
-
-      - name: Login to Azure Public Cloud with AzPowershell
-        uses: azure/login@v2
-        with:
-          client-id: ${{ secrets.AZURE_ASIM_CLIENT_ID }}
-          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-          enable-AzPSSession: true
-          allow-no-subscriptions: true
-
-      - name: Run ASIM testers
+      - name: Run ASIM Schema and Data tests PowerShell script
         uses: azure/powershell@v2
         with:
           inlineScript: |
-            & ".script/tests/asimParsersTest/runAsimTesters.ps1"
+            $filePath = ".script/tests/asimParsersTest/runAsimTesters.ps1"
+            $url = "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/runAsimTesters.ps1"
+            # Check if file exists and delete if it does
+            if (Test-Path $filePath) {
+              Remove-Item $filePath -Force
+            }
+            # Download the file
+            Write-Host "Downloading script from the master: $url"
+            Invoke-WebRequest -Uri $url -OutFile $filePath
+            # Execute the script
+            & $filePath
           azPSVersion: "latest"
           errorActionPreference: continue
-          failOnStandardError: false
+          failOnStandardError: false
+  Run-ASim-TemplateValidation:
+    name: Run ASim Template Validation tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout pull request branch
+        uses: actions/checkout@v3
+        with:
+            ref: ${{github.event.pull_request.head.ref}}
+            repository: ${{github.event.pull_request.head.repo.full_name}}
+            persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token.
+            fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
+      - name: Setup git config
+        run: |
+              git config --local user.name "github-actions[bot]"
+              git config --local user.email "<>"
+      - name: Merge master into pull request branch
+        run: |
+              git merge origin/master
+              Conflicts=$(git ls-files -u | wc -l)
+              if [ "$Conflicts" -gt 0 ] ; then
+                echo "There is a merge conflict. Aborting"
+                git merge --abort
+                exit 1
+              fi
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+              python -m pip install --upgrade pip
+              pip install requests
+              pip install PyYAML
+              pip install tabulate
+      - name: Run ASim parsers template validations python script
+        run: |
+              filePath=".script/tests/asimParsersTest/VerifyASimParserTemplate.py"
+              url="https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/VerifyASimParserTemplate.py" 
+              # Check if file exists and delete if it does
+              if [ -f "$filePath" ]; then
+                rm -f "$filePath"
+              fi
+              # Download the file
+              echo "Downloading script from the master: $url"
+              curl -o "$filePath" "$url"
+              # Execute the script
+              python "$filePath"
+  Run-ASim-Parser-Filtering-Tests:
+    name: Run ASim Parser Filtering tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout pull request branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token.
+          fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
+      - name: Setup git config
+        run: |
+              git config --local user.name "github-actions[bot]"
+              git config --local user.email "<>"
+      - name: Merge master into pull request branch
+        run: |
+          git merge origin/master
+          Conflicts=$(git ls-files -u | wc -l)
+          if [ "$Conflicts" -gt 0 ] ; then
+            echo "There is a merge conflict. Aborting"
+            git merge --abort
+            exit 1
+          fi
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+              python -m pip install --upgrade pip
+              pip install PyYAML
+              pip install azure-identity
+              pip install azure-monitor-query
+      - name: Login to Azure Public Cloud
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_ASIM_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          allow-no-subscriptions: true
+      - name: Run ASim parsers filtering tests python script
+        run: |
+          filePath=".script/tests/asimParsersTest/ASimFilteringTest.py"
+          url="https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/ASimFilteringTest.py"
+          # Check if file exists and delete if it does
+          if [ -f "$filePath" ]; then
+            rm -f "$filePath"
+          fi
+          # Download the file
+          echo "Downloading script from the master: $url"
+          curl -o "$filePath" "$url"
+          # Execute the script
+          python "$filePath"

.script/dataConnectorValidator.ts

@@ -148,6 +148,10 @@ function getConnectorCategory(dataTypes : any, instructionSteps:[])
   {
     return ConnectorCategory.CrowdStrikeFalconIOC;
   }
+  else if (dataTypes[0].name.includes("WizIssues"))
+  {
+    return ConnectorCategory.Wiz;
+  }
   else if (dataTypes[0].name.includes("vectra_isession"))
   {
     return ConnectorCategory.VectraStreamAma;

.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Company_CL.json

@@ -0,0 +1,45 @@
+{
+    "Name": "Ipinfo_Company_CL",
+    "Properties": [
+        {
+            "Name": "TimeGenerated",
+            "Type": "datetime"
+        },
+        {
+            "Name": "as_domain",
+            "Type": "String"
+        },
+        {
+            "Name": "as_name",
+            "Type": "String"
+        },
+        {
+            "Name": "as_type",
+            "Type": "String"
+        },
+        {
+            "Name": "asn",
+            "Type": "String"
+        },
+        {
+            "Name": "country",
+            "Type": "String"
+        },
+        {
+            "Name": "company_domain",
+            "Type": "String"
+        },
+        {
+            "Name": "company_name",
+            "Type": "String"
+        },
+        {
+            "Name": "company_type",
+            "Type": "String"
+        },
+        {
+            "Name": "range",
+            "Type": "String"
+        }
+    ]
+}

.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Location_CL.json

@@ -0,0 +1,49 @@
+{
+    "Name": "Ipinfo_Location_CL",
+    "Properties": [
+        {
+            "Name": "TimeGenerated",
+            "Type": "datetime"
+        },
+        {
+            "Name": "city",
+            "Type": "String"
+        },
+        {
+            "Name": "country",
+            "Type": "String"
+        },
+        {
+            "Name": "geoname_id",
+            "Type": "String"
+        },
+        {
+            "Name": "lat",
+            "Type": "String"
+        },
+        {
+            "Name": "lng",
+            "Type": "String"
+        },
+        {
+            "Name": "postal_code",
+            "Type": "String"
+        },
+        {
+            "Name": "region",
+            "Type": "String"
+        },
+        {
+            "Name": "region_code",
+            "Type": "String"
+        },
+        {
+            "Name": "range",
+            "Type": "String"
+        },
+        {
+            "Name": "timezone",
+            "Type": "String"
+        }
+    ]
+}

.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Privacy_CL.json

@@ -0,0 +1,37 @@
+{
+    "Name": "Ipinfo_Privacy_CL",
+    "Properties": [
+        {
+            "Name": "TimeGenerated",
+            "Type": "datetime"
+        },
+        {
+            "Name": "hosting",
+            "Type": "String"
+        },
+        {
+            "Name": "proxy",
+            "Type": "String"
+        },
+        {
+            "Name": "relay",
+            "Type": "String"
+        },
+        {
+            "Name": "service",
+            "Type": "String"
+        },
+        {
+            "Name": "tor",
+            "Type": "String"
+        },
+        {
+            "Name": "vpn",
+            "Type": "String"
+        },
+        {
+            "Name": "range",
+            "Type": "String"
+        }
+    ]
+}

.script/tests/KqlvalidationsTests/CustomTables/TransmitSecurityAdminActivity_CL.json

@@ -0,0 +1,37 @@
+{
+    "Name": "TransmitSecurityAdminActivity_CL",
+    "Properties": [
+      {
+        "Name": "activity",
+        "Type": "string"
+      },
+      {
+        "Name": "actor_id",
+        "Type": "string"
+      },
+      {
+        "Name": "actor_type",
+        "Type": "string"
+      },
+      {
+        "Name": "ip",
+        "Type": "string"
+      },
+      {
+        "Name": "target_resource_id",
+        "Type": "string"
+      },
+      {
+        "Name": "target_resource_type",
+        "Type": "string"
+      },
+      {
+        "Name": "timestamp",
+        "Type": "datetime"
+      },
+      {
+        "Name": "user_agent",
+        "Type": "string"
+      }
+    ]
+  }

.script/tests/KqlvalidationsTests/CustomTables/TransmitSecurityUserActivity_CL.json

@@ -0,0 +1,29 @@
+{
+  "Name": "TransmitSecurityUserActivity_CL",
+  "Properties": [
+    {
+      "Name": "activity",
+      "Type": "string"
+    },
+    {
+      "Name": "app_id",
+      "Type": "string"
+    },
+    {
+      "Name": "ip",
+      "Type": "string"
+    },
+    {
+      "Name": "timestamp",
+      "Type": "datetime"
+    },
+    {
+      "Name": "user_agent",
+      "Type": "string"
+    },
+    {
+      "Name": "user_id",
+      "Type": "string"
+    }
+  ]
+}