AIK-4442 Report blocked requests per IPList/Botlist

agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java

@@ -30,10 +30,18 @@ public ReportingApi(int timeoutInSec) {
 
     public record APIListsResponse(
             List<ListsResponseEntry> blockedIPAddresses,
+            List<ListsResponseEntry> monitoredIPAddresses,
             List<ListsResponseEntry> allowedIPAddresses,
-            String blockedUserAgents
+            String blockedUserAgents,
+            String monitoredUserAgents,
+            List<UserAgentDetail> userAgentDetails
     ) {}
-    public record ListsResponseEntry(String source, String description, List<String> ips) {}
+
+    public record ListsResponseEntry(String key, String source, String description, List<String> ips) {
+    }
+
+    public record UserAgentDetail(String key, String pattern) {
+    }
     /**
      * Fetch blocked lists using a separate API call, these can include :
      * -> blocked IP Addresses (e.g. geo restrictions)

agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java

@@ -3,6 +3,7 @@
 import dev.aikido.agent_api.background.Endpoint;
 import dev.aikido.agent_api.context.Context;
 import dev.aikido.agent_api.context.ContextObject;
+import dev.aikido.agent_api.context.RouteMetadata;
 import dev.aikido.agent_api.storage.ServiceConfiguration;
 import dev.aikido.agent_api.storage.statistics.StatisticsStore;
 
@@ -38,31 +39,47 @@
         // Increment total hits :
         StatisticsStore.incrementHits();
 
-        // Per-route IP allowlists :
-        List<Endpoint> matchedEndpoints = matchEndpoints(newContext.getRouteMetadata(), config.getEndpoints());
-        if (!ipAllowedToAccessRoute(newContext.getRemoteAddress(), matchedEndpoints)) {
+        Res endpointAllowlistRes = checkEndpointAllowlist(newContext.getRouteMetadata(), newContext.getRemoteAddress(), config);
+        Res blockedIpsRes = checkBlockedIps(newContext.getRemoteAddress(), config);
+        Res blockedUserAgentsRes = checkBlockedUserAgents(newContext.getHeader("user-agent"), config);
+
+        // make sure to follow a certain order when giving error messages.
+        if (endpointAllowlistRes != null)
+            return endpointAllowlistRes;
+        else if (blockedIpsRes != null)
+            return blockedIpsRes;
+        else return blockedUserAgentsRes;
+    }
+
+    private static Res checkEndpointAllowlist(RouteMetadata routeMetadata, String remoteAddress, ServiceConfiguration config) {
+        List<Endpoint> matchedEndpoints = matchEndpoints(routeMetadata, config.getEndpoints());
+        if (!ipAllowedToAccessRoute(remoteAddress, matchedEndpoints)) {
             String msg = "Your IP address is not allowed to access this resource.";
-            msg += " (Your IP: " + newContext.getRemoteAddress() + ")";
+            msg += " (Your IP: " + remoteAddress + ")";
             return new Res(msg, 403);
         }
+        return null; // not blocked
+    }
 
-        // Blocked IP lists (e.g. Geo restrictions)
-        ServiceConfiguration.BlockedResult ipBlocked = config.isIpBlocked(newContext.getRemoteAddress());
+    private static Res checkBlockedIps(String remoteAddress, ServiceConfiguration config) {
+        ServiceConfiguration.BlockedResult ipBlocked = config.isIpBlocked(remoteAddress);
         if (ipBlocked.blocked()) {
             String msg = "Your IP address is blocked. Reason: " + ipBlocked.description();
-            msg += " (Your IP: " + newContext.getRemoteAddress() + ")";
+            msg += " (Your IP: " + remoteAddress + ")";
             return new Res(msg, 403);
         }
+        return null; // not blocked
+    }
 
-        // User-Agent blocking (e.g. blocking bots)
-        String userAgent = newContext.getHeader("user-agent");
-        if (userAgent != null && !userAgent.isEmpty()) {
-            if (config.isBlockedUserAgent(userAgent)) {
-                String msg = "You are not allowed to access this resource because you have been identified as a bot.";
-                return new Res(msg, 403);
-            }
+    private static Res checkBlockedUserAgents(String userAgent, ServiceConfiguration config) {
+        if (userAgent == null || userAgent.isEmpty()) {
+            return null; // not blocked
+        }
+        if (config.isBlockedUserAgent(userAgent)) {
+            String msg = "You are not allowed to access this resource because you have been identified as a bot.";
+            return new Res(msg, 403);
         }
-        return null;
+        return null; // not blocked
     }
 
     public record Res(String msg, Integer status) {

agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java

@@ -4,11 +4,12 @@
 import dev.aikido.agent_api.background.cloud.api.APIResponse;
 import dev.aikido.agent_api.background.cloud.api.ReportingApi;
 import dev.aikido.agent_api.helpers.net.IPList;
+import dev.aikido.agent_api.storage.service_configuration.ParsedFirewallLists;
+import dev.aikido.agent_api.storage.statistics.StatisticsStore;
 
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
-import java.util.regex.Pattern;
 
 import static dev.aikido.agent_api.helpers.IPListBuilder.createIPList;
 import static dev.aikido.agent_api.vulnerabilities.ssrf.IsPrivateIP.isPrivateIp;
@@ -18,16 +19,13 @@
  * It is essential for e.g. rate limiting
  */
 public class ServiceConfiguration {
-    private final List<IPListEntry> blockedIps = new ArrayList<>();
-    private final List<IPListEntry> allowedIps = new ArrayList<>();
+    private final ParsedFirewallLists firewallLists = new ParsedFirewallLists();
     private boolean blockingEnabled;
     private boolean receivedAnyStats;
     private boolean middlewareInstalled;
     private IPList bypassedIPs = new IPList();
     private HashSet<String> blockedUserIDs = new HashSet<>();
     private List<Endpoint> endpoints = new ArrayList<>();
-    // User-Agent Blocking (e.g. bot blocking) :
-    private Pattern blockedUserAgentRegex;
 
     public ServiceConfiguration() {
         this.receivedAnyStats = true; // true by default, waiting for the startup event
@@ -87,68 +85,39 @@ public boolean isIpBypassed(String ip) {
      * Check if the IP is blocked (e.g. Geo IP Restrictions)
      */
     public BlockedResult isIpBlocked(String ip) {
+        BlockedResult blockedResult = new BlockedResult(false, null);
+
         // Check for allowed ip addresses (i.e. only one country is allowed to visit the site)
         // Always allow access from private IP addresses (those include local IP addresses)
-        if (!allowedIps.isEmpty() && !isPrivateIp(ip)) {
-            boolean ipAllowed = false;
-            for (IPListEntry entry : allowedIps) {
-                if (entry.ipList.matches(ip)) {
-                    ipAllowed = true; // We allow IP addresses as long as they match with one of the lists.
-                    break;
-                }
-            }
-            if (!ipAllowed) {
-                return new BlockedResult(true, "not in allowlist");
-            }
+        if (!isPrivateIp(ip) && !firewallLists.matchesAllowedIps(ip)) {
+            blockedResult = new BlockedResult(true, "not in allowlist");
         }
 
         // Check for blocked ip addresses
-        for (IPListEntry entry : blockedIps) {
-            if (entry.ipList.matches(ip)) {
-                return new BlockedResult(true, entry.description);
+        for (ParsedFirewallLists.Match match : firewallLists.matchBlockedIps(ip)) {
+            StatisticsStore.incrementIpHits(match.key());
+            // when a blocking match is found, set blocked result if it hasn't been set already.
+            if (match.block() && !blockedResult.blocked()) {
+                blockedResult = new BlockedResult(true, match.description());
             }
         }
-        return new BlockedResult(false, null);
+
+        return blockedResult;
     }
 
     public void updateBlockedLists(ReportingApi.APIListsResponse res) {
-        // Update blocked IP addresses (e.g. for geo restrictions) :
-        blockedIps.clear();
-        if (res.blockedIPAddresses() != null) {
-            for (ReportingApi.ListsResponseEntry entry : res.blockedIPAddresses()) {
-                IPList ipList = createIPList(entry.ips());
-                blockedIps.add(new IPListEntry(ipList, entry.description()));
-            }
-        }
-
-        // Update allowed IP addresses (e.g. for geo restrictions) :
-        allowedIps.clear();
-        if (res.allowedIPAddresses() != null) {
-            for (ReportingApi.ListsResponseEntry entry : res.allowedIPAddresses()) {
-                IPList ipList = createIPList(entry.ips());
-                this.allowedIps.add(new IPListEntry(ipList, entry.description()));
-            }
-        }
-
-        // Update Blocked User-Agents regex
-        blockedUserAgentRegex = null;
-        if (res.blockedUserAgents() != null && !res.blockedUserAgents().isEmpty()) {
-            this.blockedUserAgentRegex = Pattern.compile(res.blockedUserAgents(), Pattern.CASE_INSENSITIVE);
-        }
+        this.firewallLists.update(res);
     }
 
     /**
      * Check if a given User-Agent is blocked or not :
      */
     public boolean isBlockedUserAgent(String userAgent) {
-        if (blockedUserAgentRegex != null) {
-            return blockedUserAgentRegex.matcher(userAgent).find();
+        ParsedFirewallLists.UABlockedResult result = this.firewallLists.matchBlockedUserAgents(userAgent);
+        for (String matchedKey : result.matchedKeys()) {
+            StatisticsStore.incrementUAHits(matchedKey);
         }
-        return false;
-    }
-
-    // IP restrictions (e.g. Geo-IP Restrictions) :
-    public record IPListEntry(IPList ipList, String description) {
+        return result.block();
     }
 
     public record BlockedResult(boolean blocked, String description) {

agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java

@@ -0,0 +1,141 @@
+package dev.aikido.agent_api.storage.service_configuration;
+
+import dev.aikido.agent_api.background.cloud.api.ReportingApi;
+import dev.aikido.agent_api.helpers.net.IPList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static dev.aikido.agent_api.helpers.IPListBuilder.createIPList;
+
+public class ParsedFirewallLists {
+    private final List<IPEntry> blockedIps = new ArrayList<>();
+    private final List<IPEntry> allowedIps = new ArrayList<>();
+    private final List<UADetailsEntry> uaDetails = new ArrayList<>();
+    private Pattern blockedUserAgents = null;
+    private Pattern monitoredUserAgents = null;
+
+    public ParsedFirewallLists() {
+
+    }
+
+    public List<Match> matchBlockedIps(String ip) {
+        List<Match> matches = new ArrayList<>();
+        for (IPEntry entry : this.blockedIps) {
+            if (entry.ips().matches(ip)) {
+                matches.add(new Match(entry.key(), !entry.monitor(), entry.description()));
+            }
+        }
+        return matches;
+    }
+
+    // returns true if one or more matches has been found with allowlist.
+    public boolean matchesAllowedIps(String ip) {
+        if (this.allowedIps.isEmpty()) {
+            return true; // Empty allowed is means all ips match
+        }
+        for (IPEntry entry : this.allowedIps) {
+            if (entry.ips().matches(ip)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public UABlockedResult matchBlockedUserAgents(String userAgent) {
+        boolean isBlocked = false;
+        if (blockedUserAgents != null)
+            isBlocked = blockedUserAgents.matcher(userAgent).find();
+
+        boolean isMonitored = false;
+        if (monitoredUserAgents != null)
+            isMonitored = monitoredUserAgents.matcher(userAgent).find();
+
+        if (!isMonitored && !isBlocked)
+            // only run the more detailed matches if it's an actual attack/monitored.
+            return new UABlockedResult(false, List.of());
+
+        List<String> matchedUAKeys = new ArrayList<>();
+        for (UADetailsEntry entry : this.uaDetails) {
+            if (entry.pattern().matcher(userAgent).find()) {
+                matchedUAKeys.add(entry.key());
+            }
+        }
+        return new UABlockedResult(isBlocked, matchedUAKeys);
+    }
+
+    public void update(ReportingApi.APIListsResponse response) {
+        updateBlockedIps(response.blockedIPAddresses());
+        updateMonitoredIps(response.monitoredIPAddresses());
+        updateAllowedIps(response.allowedIPAddresses());
+
+        updateBlockedAndMonitoredUAs(response.blockedUserAgents(), response.monitoredUserAgents());
+        updateUADetails(response.userAgentDetails());
+    }
+
+    public void updateBlockedIps(List<ReportingApi.ListsResponseEntry> blockedIpsList) {
+        blockedIps.clear();
+        if (blockedIpsList == null)
+            return;
+        for (ReportingApi.ListsResponseEntry entry : blockedIpsList) {
+            IPList ipList = createIPList(entry.ips());
+            blockedIps.add(new IPEntry(/* monitor */ false, entry.key(), entry.source(), entry.description(), ipList));
+        }
+    }
+
+    public void updateMonitoredIps(List<ReportingApi.ListsResponseEntry> monitoredIpsList) {
+        if (monitoredIpsList == null)
+            return;
+        for (ReportingApi.ListsResponseEntry entry : monitoredIpsList) {
+            IPList ipList = createIPList(entry.ips());
+            blockedIps.add(new IPEntry(/* monitor */ true, entry.key(), entry.source(), entry.description(), ipList));
+        }
+    }
+
+    public void updateAllowedIps(List<ReportingApi.ListsResponseEntry> allowedIpsList) {
+        allowedIps.clear();
+        if (allowedIpsList == null)
+            return;
+        for (ReportingApi.ListsResponseEntry entry : allowedIpsList) {
+            IPList ipList = createIPList(entry.ips());
+            boolean shouldMonitor = false; // we don't monitor allowed ips
+            allowedIps.add(new IPEntry(shouldMonitor, entry.key(), entry.source(), entry.description(), ipList));
+        }
+    }
+
+    public void updateUADetails(List<ReportingApi.UserAgentDetail> userAgentDetails) {
+        this.uaDetails.clear();
+        if (userAgentDetails == null)
+            return;
+        for (ReportingApi.UserAgentDetail entry : userAgentDetails) {
+            Pattern pattern = Pattern.compile(entry.pattern(), Pattern.CASE_INSENSITIVE);
+            this.uaDetails.add(new UADetailsEntry(entry.key(), pattern));
+        }
+    }
+
+    public void updateBlockedAndMonitoredUAs(String blockedUAs, String monitoredUAs) {
+        this.blockedUserAgents = null;
+        if (blockedUAs != null && !blockedUAs.isEmpty()) {
+            this.blockedUserAgents = Pattern.compile(blockedUAs, Pattern.CASE_INSENSITIVE);
+        }
+
+        this.monitoredUserAgents = null;
+        if (monitoredUAs != null && !monitoredUAs.isEmpty()) {
+            this.monitoredUserAgents = Pattern.compile(monitoredUAs, Pattern.CASE_INSENSITIVE);
+        }
+    }
+
+
+    public record Match(String key, boolean block, String description) {
+    }
+
+    public record UABlockedResult(boolean block, List<String> matchedKeys) {
+    }
+
+    private record IPEntry(boolean monitor, String key, String source, String description, IPList ips) {
+    }
+
+    private record UADetailsEntry(String key, Pattern pattern) {
+    }
+}