From 9a6293dd3a0ec579964009cf831b0e5f60c428ba Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 16 Apr 2025 11:05:44 +0200 Subject: [PATCH 01/45] Create new FirewallListsRecord to store blocked and total --- .../statistics/FirewallListsRecord.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java new file mode 100644 index 00000000..edb2b7d5 --- /dev/null +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java @@ -0,0 +1,25 @@ +package dev.aikido.agent_api.storage.statistics; + +public class FirewallListsRecord { + private int total = 0; + private int blocked = 0; + + public FirewallListsRecord() { + } + + public int getTotal() { + return this.total; + } + + public void incrementTotal() { + this.total += 1; + } + + public int getBlocked() { + return this.blocked; + } + + public void incrementBlocked() { + this.blocked += 1; + } +} From 7094ac5834485fca7256040c33978832b93c2b34 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 16 Apr 2025 11:06:22 +0200 Subject: [PATCH 02/45] Reformat the Statistics object, adding ipAddresses and userAgents --- .../storage/statistics/Statistics.java | 90 ++++++++++++++++--- 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java index f465e6f4..301eab05 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java @@ -6,33 +6,47 @@ import java.util.Map; public class Statistics { + private final Map operations = new HashMap<>(); + private final Map ipAddresses = new HashMap<>(); + private final Map userAgents = new HashMap<>(); private int totalHits; private int attacksDetected; private int attacksBlocked; private long startedAt; - private final Map operations = new HashMap<>(); + public Statistics(int totalHits, int attacksDetected, int attacksBlocked) { this.totalHits = totalHits; this.attacksDetected = attacksDetected; this.attacksBlocked = attacksBlocked; this.startedAt = UnixTimeMS.getUnixTimeMS(); } + public Statistics() { this(0, 0, 0); } + + // hits public void incrementTotalHits(int count) { totalHits += count; } - public int getTotalHits() { return totalHits; } + public int getTotalHits() { + return totalHits; + } + + + // attack stats public void incrementAttacksDetected(String operation) { this.attacksDetected += 1; if (this.operations.containsKey(operation)) { this.operations.get(operation).incrementAttacksDetected(); } } - public int getAttacksDetected() { return attacksDetected; } + + public int getAttacksDetected() { + return attacksDetected; + } public void incrementAttacksBlocked(String operation) { this.attacksBlocked += 1; @@ -40,8 +54,13 @@ public void incrementAttacksBlocked(String operation) { this.operations.get(operation).incrementAttacksBlocked(); } } - public int getAttacksBlocked() { return attacksBlocked; } + public int getAttacksBlocked() { + return attacksBlocked; + } + + + // operations public void registerCall(String operation, OperationKind kind) { if (!this.operations.containsKey(operation)) { this.operations.put(operation, new OperationRecord(kind)); @@ -49,27 +68,60 @@ public void registerCall(String operation, OperationKind kind) { // increase total count by 1 this.operations.get(operation).incrementTotal(); } + public Map getOperations() { return new HashMap<>(this.operations); } - // Stats records for sending out the heartbeat : - public record StatsRequestsRecord(long total, long aborted, Map attacksDetected) {}; - public record StatsRecord(long startedAt, long endedAt, StatsRequestsRecord requests, - Map operations) { + // firewall lists + public void incrementIpTotal(String key) { + if (!this.ipAddresses.containsKey(key)) { + this.ipAddresses.put(key, new FirewallListsRecord()); + } + this.ipAddresses.get(key).incrementTotal(); } + public void incrementIpBlocked(String key) { + if (!this.ipAddresses.containsKey(key)) { + this.ipAddresses.put(key, new FirewallListsRecord()); + } + this.ipAddresses.get(key).incrementBlocked(); + } + + public Map getIpAddresses() { + return this.ipAddresses; + } + + public void incrementUATotal(String key) { + if (!this.ipAddresses.containsKey(key)) { + this.ipAddresses.put(key, new FirewallListsRecord()); + } + this.ipAddresses.get(key).incrementTotal(); + } + + public void incrementUABlocked(String key) { + if (!this.ipAddresses.containsKey(key)) { + this.ipAddresses.put(key, new FirewallListsRecord()); + } + this.ipAddresses.get(key).incrementBlocked(); + } + + public Map getUserAgents() { + return this.ipAddresses; + } + + public StatsRecord getRecord() { long endedAt = UnixTimeMS.getUnixTimeMS(); return new StatsRecord(this.startedAt, endedAt, new StatsRequestsRecord( - /* total */ totalHits, - /* aborted */ 0, // Unknown statistic, default to 0, - /* attacksDetected */ Map.of( - "total", attacksDetected, - "blocked", attacksBlocked + /* total */ totalHits, + /* aborted */ 0, // Unknown statistic, default to 0, + /* attacksDetected */ Map.of( + "total", attacksDetected, + "blocked", attacksBlocked )), - /* operations */ getOperations() + getOperations(), getIpAddresses(), getUserAgents() ); } @@ -80,4 +132,14 @@ public void clear() { this.startedAt = UnixTimeMS.getUnixTimeMS(); this.operations.clear(); } + + // Stats records for sending out the heartbeat : + public record StatsRequestsRecord(long total, long aborted, Map attacksDetected) { + } + + public record StatsRecord(long startedAt, long endedAt, StatsRequestsRecord requests, + Map operations, + Map ipAddresses, + Map userAgents) { + } } From ba1dcd1404fb1475d9b200384b56770588e1033b Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 16 Apr 2025 11:14:19 +0200 Subject: [PATCH 03/45] stats: merge total/blocked funcs for iplists/ua's --- .../storage/statistics/Statistics.java | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java index 301eab05..6640cedb 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java @@ -75,36 +75,29 @@ public Map getOperations() { // firewall lists - public void incrementIpTotal(String key) { + public void incrementIpHits(String key, boolean blocked) { if (!this.ipAddresses.containsKey(key)) { this.ipAddresses.put(key, new FirewallListsRecord()); } - this.ipAddresses.get(key).incrementTotal(); - } - public void incrementIpBlocked(String key) { - if (!this.ipAddresses.containsKey(key)) { - this.ipAddresses.put(key, new FirewallListsRecord()); + this.ipAddresses.get(key).incrementTotal(); + if (blocked) { + this.ipAddresses.get(key).incrementBlocked(); } - this.ipAddresses.get(key).incrementBlocked(); } public Map getIpAddresses() { return this.ipAddresses; } - public void incrementUATotal(String key) { + public void incrementUAHits(String key, boolean blocked) { if (!this.ipAddresses.containsKey(key)) { this.ipAddresses.put(key, new FirewallListsRecord()); } this.ipAddresses.get(key).incrementTotal(); - } - - public void incrementUABlocked(String key) { - if (!this.ipAddresses.containsKey(key)) { - this.ipAddresses.put(key, new FirewallListsRecord()); + if (blocked) { + this.ipAddresses.get(key).incrementBlocked(); } - this.ipAddresses.get(key).incrementBlocked(); } public Map getUserAgents() { From 0dd1b843a4d465184faf6fa1bc3d8f6132b510f4 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 16 Apr 2025 11:14:29 +0200 Subject: [PATCH 04/45] Add ip lists/ua's to StatisticsStore --- .../storage/statistics/StatisticsStore.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java index 3c9a88a2..056f3f99 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java @@ -62,6 +62,24 @@ public static void registerCall(String sink, OperationKind kind) { } } + public static void incrementIpHits(String key, boolean blocked) { + mutex.lock(); + try { + stats.incrementIpHits(key, blocked); + } finally { + mutex.unlock(); + } + } + + public static void incrementUAHits(String key, boolean blocked) { + mutex.lock(); + try { + stats.incrementUAHits(key, blocked); + } finally { + mutex.unlock(); + } + } + public static void clear() { mutex.lock(); try { From ffecb7bf7c1c40d36f126b0d65422b3873066628 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 16 Apr 2025 11:19:38 +0200 Subject: [PATCH 05/45] Add new StatisticsStore test cases & fix broken heartbeat test --- .../cloud/api/HeartbeatEventTest.java | 2 +- .../java/storage/StatisticsStoreTest.java | 94 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 agent_api/src/test/java/storage/StatisticsStoreTest.java diff --git a/agent_api/src/test/java/background/cloud/api/HeartbeatEventTest.java b/agent_api/src/test/java/background/cloud/api/HeartbeatEventTest.java index 1a1966bd..d2d92743 100644 --- a/agent_api/src/test/java/background/cloud/api/HeartbeatEventTest.java +++ b/agent_api/src/test/java/background/cloud/api/HeartbeatEventTest.java @@ -45,7 +45,7 @@ public void setUp() { @Test public void testGetHeartbeatEvent() { // Arrange - Statistics.StatsRecord stats = new Statistics.StatsRecord(0, 1, null, null); + Statistics.StatsRecord stats = new Statistics.StatsRecord(0, 1, null, null, null, null); var hostnames = new Hostnames(5000); hostnames.add("aikido.dev", 8080); RouteEntry[] routes = new RouteEntry[0]; // Replace with actual RouteEntry array if needed diff --git a/agent_api/src/test/java/storage/StatisticsStoreTest.java b/agent_api/src/test/java/storage/StatisticsStoreTest.java new file mode 100644 index 00000000..27758847 --- /dev/null +++ b/agent_api/src/test/java/storage/StatisticsStoreTest.java @@ -0,0 +1,94 @@ +package storage; + +import dev.aikido.agent_api.storage.statistics.OperationKind; +import dev.aikido.agent_api.storage.statistics.Statistics; +import dev.aikido.agent_api.storage.statistics.StatisticsStore; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class StatisticsStoreTest { + + @BeforeEach + public void setUp() { + // Clear the statistics before each test + StatisticsStore.clear(); + } + + @Test + public void testIncrementHits() { + StatisticsStore.incrementHits(); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(1, record.requests().total()); + } + + @Test + public void testIncrementAttacksDetected() { + String operation = "testOperation"; + StatisticsStore.incrementAttacksDetected(operation); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(1, record.requests().attacksDetected().get("total")); + } + + @Test + public void testIncrementAttacksBlocked() { + String operation = "testOperation"; + StatisticsStore.incrementAttacksBlocked(operation); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(1, record.requests().attacksDetected().get("blocked")); + } + + @Test + public void testRegisterCall() { + String operation = "testOperation"; + StatisticsStore.registerCall(operation, OperationKind.FS_OP); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertTrue(record.operations().containsKey(operation)); + assertEquals(1, record.operations().get(operation).total()); + } + + @Test + public void testIncrementIpHits() { + String ip = "192.168.1.1"; + StatisticsStore.incrementIpHits(ip, false); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertTrue(record.ipAddresses().containsKey(ip)); + assertEquals(1, record.ipAddresses().get(ip).getTotal()); + } + + @Test + public void testIncrementUAHits() { + String userAgent = "Mozilla/5.0"; + StatisticsStore.incrementUAHits(userAgent, true); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertTrue(record.userAgents().containsKey(userAgent)); + assertEquals(1, record.userAgents().get(userAgent).getTotal()); + assertEquals(1, record.userAgents().get(userAgent).getBlocked()); + } + + @Test + public void testClear() { + StatisticsStore.incrementHits(); + StatisticsStore.incrementIpHits("ip", false); + StatisticsStore.incrementUAHits("ip", false); + String operation = "testOperation"; + StatisticsStore.registerCall(operation, OperationKind.FS_OP); + + StatisticsStore.clear(); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + + assertNotNull(record); + assertEquals(0, record.requests().total()); + assertEquals(0, record.operations().size()); + assertEquals(0, record.ipAddresses().size()); + assertEquals(0, record.userAgents().size()); + + } +} From 67ca96c949cd2617332c2dc21d70175833c2147f Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 16 Apr 2025 11:19:53 +0200 Subject: [PATCH 06/45] Add ipAddreses and userAgents stats to clear() in Statistics.java --- .../dev/aikido/agent_api/storage/statistics/Statistics.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java index 6640cedb..3b7fcbef 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java @@ -124,6 +124,8 @@ public void clear() { this.attacksDetected = 0; this.startedAt = UnixTimeMS.getUnixTimeMS(); this.operations.clear(); + this.ipAddresses.clear(); + this.userAgents.clear(); } // Stats records for sending out the heartbeat : From ce63c4de13929bfac18046c7d546da2991b68c32 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Thu, 17 Apr 2025 12:35:47 +0200 Subject: [PATCH 07/45] Change FirewallListsRecord to use new format --- .../statistics/FirewallListsRecord.java | 37 +++++++++++++++++-- .../storage/statistics/Statistics.java | 36 ++++-------------- .../storage/statistics/StatisticsStore.java | 28 +++++++++++++- 3 files changed, 66 insertions(+), 35 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java index edb2b7d5..36b0ca2c 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java @@ -1,25 +1,54 @@ package dev.aikido.agent_api.storage.statistics; +import java.util.HashMap; +import java.util.Map; + public class FirewallListsRecord { + private final Map breakdown = new HashMap<>(); private int total = 0; private int blocked = 0; public FirewallListsRecord() { } - public int getTotal() { - return this.total; + public void increment(String key, boolean blocked) { + breakdown.computeIfAbsent(key, k -> new BreakdownEntry(0, 0)); + + int newTotal = breakdown.get(key).total + 1; + int newBlocked = breakdown.get(key).blocked; + if (blocked) { + newBlocked += 1; + } + + breakdown.put(key, new BreakdownEntry(newTotal, newBlocked)); + } + + public BreakdownEntry get(String key) { + return this.breakdown.get(key); } public void incrementTotal() { this.total += 1; } - public int getBlocked() { - return this.blocked; + public int getTotal() { + return this.total; } public void incrementBlocked() { this.blocked += 1; } + + public int getBlocked() { + return this.blocked; + } + + public void clear() { + this.total = 0; + this.blocked = 0; + this.breakdown.clear(); + } + + record BreakdownEntry(int total, int blocked) { + } } diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java index 3b7fcbef..e76d0346 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java @@ -7,8 +7,8 @@ public class Statistics { private final Map operations = new HashMap<>(); - private final Map ipAddresses = new HashMap<>(); - private final Map userAgents = new HashMap<>(); + private final FirewallListsRecord ipAddresses = new FirewallListsRecord(); + private final FirewallListsRecord userAgents = new FirewallListsRecord(); private int totalHits; private int attacksDetected; private int attacksBlocked; @@ -75,33 +75,11 @@ public Map getOperations() { // firewall lists - public void incrementIpHits(String key, boolean blocked) { - if (!this.ipAddresses.containsKey(key)) { - this.ipAddresses.put(key, new FirewallListsRecord()); - } - - this.ipAddresses.get(key).incrementTotal(); - if (blocked) { - this.ipAddresses.get(key).incrementBlocked(); - } - } - - public Map getIpAddresses() { + public FirewallListsRecord getIpAddresses() { return this.ipAddresses; } - - public void incrementUAHits(String key, boolean blocked) { - if (!this.ipAddresses.containsKey(key)) { - this.ipAddresses.put(key, new FirewallListsRecord()); - } - this.ipAddresses.get(key).incrementTotal(); - if (blocked) { - this.ipAddresses.get(key).incrementBlocked(); - } - } - - public Map getUserAgents() { - return this.ipAddresses; + public FirewallListsRecord getUserAgents() { + return this.userAgents; } @@ -134,7 +112,7 @@ public record StatsRequestsRecord(long total, long aborted, Map public record StatsRecord(long startedAt, long endedAt, StatsRequestsRecord requests, Map operations, - Map ipAddresses, - Map userAgents) { + FirewallListsRecord ipAddresses, + FirewallListsRecord userAgents) { } } diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java index 056f3f99..eb4fb0d5 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java @@ -65,7 +65,19 @@ public static void registerCall(String sink, OperationKind kind) { public static void incrementIpHits(String key, boolean blocked) { mutex.lock(); try { - stats.incrementIpHits(key, blocked); + stats.getIpAddresses().increment(key, blocked); + } finally { + mutex.unlock(); + } + } + + public static void incrementTotalIpHits(boolean blocked) { + mutex.lock(); + try { + stats.getIpAddresses().incrementTotal(); + if (blocked) { + stats.getIpAddresses().incrementBlocked(); + } } finally { mutex.unlock(); } @@ -74,7 +86,19 @@ public static void incrementIpHits(String key, boolean blocked) { public static void incrementUAHits(String key, boolean blocked) { mutex.lock(); try { - stats.incrementUAHits(key, blocked); + stats.getUserAgents().increment(key, blocked); + } finally { + mutex.unlock(); + } + } + + public static void incrementTotalUAHits(boolean blocked) { + mutex.lock(); + try { + stats.getUserAgents().incrementTotal(); + if (blocked) { + stats.getUserAgents().incrementBlocked(); + } } finally { mutex.unlock(); } From da5f113d3a0b5f1f430c2802aa3b5d38e95d120d Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Thu, 17 Apr 2025 12:47:31 +0200 Subject: [PATCH 08/45] Update test cases with new tests --- .../statistics/FirewallListsRecord.java | 6 +- .../java/storage/StatisticsStoreTest.java | 227 +++++++++++++++++- 2 files changed, 227 insertions(+), 6 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java index 36b0ca2c..02240a22 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java @@ -49,6 +49,10 @@ public void clear() { this.breakdown.clear(); } - record BreakdownEntry(int total, int blocked) { + public int size() { + return this.breakdown.size(); + } + + public record BreakdownEntry(int total, int blocked) { } } diff --git a/agent_api/src/test/java/storage/StatisticsStoreTest.java b/agent_api/src/test/java/storage/StatisticsStoreTest.java index 27758847..df710fbb 100644 --- a/agent_api/src/test/java/storage/StatisticsStoreTest.java +++ b/agent_api/src/test/java/storage/StatisticsStoreTest.java @@ -58,8 +58,8 @@ public void testIncrementIpHits() { StatisticsStore.incrementIpHits(ip, false); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); assertNotNull(record); - assertTrue(record.ipAddresses().containsKey(ip)); - assertEquals(1, record.ipAddresses().get(ip).getTotal()); + assertNotNull(record.ipAddresses().get(ip)); + assertEquals(1, record.ipAddresses().get(ip).total()); } @Test @@ -68,9 +68,9 @@ public void testIncrementUAHits() { StatisticsStore.incrementUAHits(userAgent, true); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); assertNotNull(record); - assertTrue(record.userAgents().containsKey(userAgent)); - assertEquals(1, record.userAgents().get(userAgent).getTotal()); - assertEquals(1, record.userAgents().get(userAgent).getBlocked()); + assertNotNull(record.userAgents().get(userAgent)); + assertEquals(1, record.userAgents().get(userAgent).total()); + assertEquals(1, record.userAgents().get(userAgent).blocked()); } @Test @@ -91,4 +91,221 @@ public void testClear() { assertEquals(0, record.userAgents().size()); } + + @Test + public void testIncrementMultipleHits() { + StatisticsStore.incrementHits(); + StatisticsStore.incrementHits(); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(2, record.requests().total()); + } + + @Test + public void testIncrementMultipleAttacksDetected() { + String operation = "testOperation"; + StatisticsStore.incrementAttacksDetected(operation); + StatisticsStore.incrementAttacksDetected(operation); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(2, record.requests().attacksDetected().get("total")); + } + + @Test + public void testIncrementMultipleAttacksBlocked() { + String operation = "testOperation"; + StatisticsStore.incrementAttacksBlocked(operation); + StatisticsStore.incrementAttacksBlocked(operation); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(2, record.requests().attacksDetected().get("blocked")); + } + + @Test + public void testRegisterMultipleCalls() { + String operation = "testOperation"; + StatisticsStore.registerCall(operation, OperationKind.FS_OP); + StatisticsStore.registerCall(operation, OperationKind.FS_OP); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertTrue(record.operations().containsKey(operation)); + assertEquals(2, record.operations().get(operation).total()); + } + + @Test + public void testIncrementIpHitsMultipleTimes() { + String ip = "192.168.1.1"; + StatisticsStore.incrementIpHits(ip, false); + StatisticsStore.incrementIpHits(ip, true); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertNotNull(record.ipAddresses().get(ip)); + assertEquals(2, record.ipAddresses().get(ip).total()); + assertEquals(1, record.ipAddresses().get(ip).blocked()); + } + + @Test + public void testIncrementUAHitsMultipleTimes() { + String userAgent = "Mozilla/5.0"; + StatisticsStore.incrementUAHits(userAgent, true); + StatisticsStore.incrementUAHits(userAgent, false); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertNotNull(record.userAgents().get(userAgent)); + assertEquals(2, record.userAgents().get(userAgent).total()); + assertEquals(1, record.userAgents().get(userAgent).blocked()); + } + + @Test + public void testClearAfterMultipleIncrements() { + StatisticsStore.incrementHits(); + StatisticsStore.incrementIpHits("192.168.1.1", false); + StatisticsStore.incrementUAHits("Mozilla/5.0", true); + String operation = "testOperation"; + StatisticsStore.registerCall(operation, OperationKind.FS_OP); + + StatisticsStore.clear(); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + + assertNotNull(record); + assertEquals(0, record.requests().total()); + assertEquals(0, record.operations().size()); + assertEquals(0, record.ipAddresses().size()); + assertEquals(0, record.userAgents().size()); + } + + @Test + public void testConcurrentAccess() throws InterruptedException { + Runnable incrementHitsTask = () -> { + for (int i = 0; i < 100; i++) { + StatisticsStore.incrementHits(); + } + }; + + Thread thread1 = new Thread(incrementHitsTask); + Thread thread2 = new Thread(incrementHitsTask); + thread1.start(); + thread2.start(); + thread1.join(); + thread2.join(); + + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(200, record.requests().total()); + } + + @Test + public void testIncrementIpHitsWithDifferentIPs() { + StatisticsStore.incrementIpHits("192.168.1.1", false); + StatisticsStore.incrementIpHits("192.168.1.2", true); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(1, record.ipAddresses().get("192.168.1.1").total()); + assertEquals(0, record.ipAddresses().get("192.168.1.1").blocked()); + assertEquals(1, record.ipAddresses().get("192.168.1.2").total()); + assertEquals(1, record.ipAddresses().get("192.168.1.2").blocked()); + } + + @Test + public void testIncrementUAHitsWithDifferentUserAgents() { + StatisticsStore.incrementUAHits("Mozilla/5.0", true); + StatisticsStore.incrementUAHits("Chrome/91.0", false); + + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(1, record.userAgents().get("Mozilla/5.0").total()); + assertEquals(1, record.userAgents().get("Mozilla/5.0").blocked()); + assertEquals(1, record.userAgents().get("Chrome/91.0").total()); + assertEquals(0, record.userAgents().get("Chrome/91.0").blocked()); + } + + @Test + public void testClearAfterIncrementingDifferentMetrics() { + StatisticsStore.incrementHits(); + StatisticsStore.incrementAttacksDetected("operation1"); + StatisticsStore.incrementAttacksBlocked("operation1"); + StatisticsStore.incrementIpHits("192.168.1.1", false); + StatisticsStore.incrementUAHits("Mozilla/5.0", true); + + StatisticsStore.clear(); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + + assertNotNull(record); + assertEquals(0, record.requests().total()); + assertEquals(0, record.requests().attacksDetected().get("total")); + assertEquals(0, record.requests().attacksDetected().get("blocked")); + assertEquals(0, record.ipAddresses().size()); + assertEquals(0, record.userAgents().size()); + } + + @Test + public void testIncrementTotalIpHits() { + StatisticsStore.incrementTotalIpHits(false); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(1, record.ipAddresses().getTotal()); + assertEquals(0, record.ipAddresses().getBlocked()); + } + + @Test + public void testIncrementTotalUAHits() { + StatisticsStore.incrementTotalUAHits(true); + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(1, record.userAgents().getTotal()); + assertEquals(1, record.userAgents().getBlocked()); + } + + @Test + public void testConcurrentIncrementHitsAndClear() throws InterruptedException { + Runnable incrementHitsTask = () -> { + for (int i = 0; i < 50; i++) { + StatisticsStore.incrementHits(); + } + }; + + Thread incrementThread = new Thread(incrementHitsTask); + Thread clearThread = new Thread(() -> { + try { + Thread.sleep(100); // Ensure this runs after some hits are incremented + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + StatisticsStore.clear(); + }); + + incrementThread.start(); + clearThread.start(); + incrementThread.join(); + clearThread.join(); + + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(0, record.requests().total()); // Expecting 0 due to clear + } + + @Test + public void testIncrementMultipleMetricsSimultaneously() { + String ip = "192.168.1.1"; + String userAgent = "Mozilla/5.0"; + String operation = "testOperation"; + + StatisticsStore.incrementHits(); + StatisticsStore.incrementAttacksDetected(operation); + StatisticsStore.incrementAttacksBlocked(operation); + StatisticsStore.incrementIpHits(ip, false); + StatisticsStore.incrementUAHits(userAgent, true); + + Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); + assertNotNull(record); + assertEquals(1, record.requests().total()); + assertEquals(1, record.requests().attacksDetected().get("total")); + assertEquals(1, record.requests().attacksDetected().get("blocked")); + assertNotNull(record.ipAddresses().get(ip)); + assertEquals(1, record.ipAddresses().get(ip).total()); + assertEquals(0, record.ipAddresses().get(ip).blocked()); + assertNotNull(record.userAgents().get(userAgent)); + assertEquals(1, record.userAgents().get(userAgent).total()); + assertEquals(1, record.userAgents().get(userAgent).blocked()); + } } From 9af073d0a2888c36a9ab6c044059f891808acba7 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Thu, 17 Apr 2025 15:47:54 +0200 Subject: [PATCH 09/45] Update reporting api --- .../aikido/agent_api/background/cloud/api/ReportingApi.java | 5 +++-- .../agent_api/background/cloud/api/ReportingApiHTTP.java | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java index 4ef6af34..577141a9 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java @@ -31,9 +31,10 @@ public ReportingApi(int timeoutInSec) { public record APIListsResponse( List blockedIPAddresses, List allowedIPAddresses, - String blockedUserAgents + BotBlocklist blockedUserAgents ) {} - public record ListsResponseEntry(String source, String description, List ips) {} + public record ListsResponseEntry(boolean monitor, String source, String description, List ips) {} + public record BotBlocklist(boolean monitor, String key, String pattern) {} /** * Fetch blocked lists using a separate API call, these can include : * -> blocked IP Addresses (e.g. geo restrictions) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java index e7391769..d822b2bb 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java @@ -81,6 +81,8 @@ public Optional fetchBlockedLists() { // Set the Accept-Encoding header to gzip connection.setRequestProperty("Accept-Encoding", "gzip"); connection.setRequestProperty("Authorization", token.get()); + // Indicates to the server that this agent supports the new format with monitoring + connection.setRequestProperty("x-supports-monitoring", "true"); if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { return Optional.empty(); From 488bdc4988e308649f3dc93a5316943a0191af60 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Thu, 17 Apr 2025 15:49:07 +0200 Subject: [PATCH 10/45] Revert "Update reporting api" This reverts commit 9af073d0a2888c36a9ab6c044059f891808acba7. --- .../aikido/agent_api/background/cloud/api/ReportingApi.java | 5 ++--- .../agent_api/background/cloud/api/ReportingApiHTTP.java | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java index 577141a9..4ef6af34 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java @@ -31,10 +31,9 @@ public ReportingApi(int timeoutInSec) { public record APIListsResponse( List blockedIPAddresses, List allowedIPAddresses, - BotBlocklist blockedUserAgents + String blockedUserAgents ) {} - public record ListsResponseEntry(boolean monitor, String source, String description, List ips) {} - public record BotBlocklist(boolean monitor, String key, String pattern) {} + public record ListsResponseEntry(String source, String description, List ips) {} /** * Fetch blocked lists using a separate API call, these can include : * -> blocked IP Addresses (e.g. geo restrictions) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java index d822b2bb..e7391769 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java @@ -81,8 +81,6 @@ public Optional fetchBlockedLists() { // Set the Accept-Encoding header to gzip connection.setRequestProperty("Accept-Encoding", "gzip"); connection.setRequestProperty("Authorization", token.get()); - // Indicates to the server that this agent supports the new format with monitoring - connection.setRequestProperty("x-supports-monitoring", "true"); if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { return Optional.empty(); From 9bb2f8f9491b3e879bde82cede683b40d821155c Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Thu, 17 Apr 2025 15:50:07 +0200 Subject: [PATCH 11/45] Update API call that fetches firewall lists --- .../agent_api/background/cloud/api/ReportingApi.java | 9 +++++++-- .../agent_api/background/cloud/api/ReportingApiHTTP.java | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java index 4ef6af34..299f8543 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java @@ -31,9 +31,14 @@ public ReportingApi(int timeoutInSec) { public record APIListsResponse( List blockedIPAddresses, List allowedIPAddresses, - String blockedUserAgents + BotBlocklist blockedUserAgents ) {} - public record ListsResponseEntry(String source, String description, List ips) {} + + public record ListsResponseEntry(boolean monitor, String key, String source, String description, List ips) { + } + + public record BotBlocklist(boolean monitor, String key, String pattern) { + } /** * Fetch blocked lists using a separate API call, these can include : * -> blocked IP Addresses (e.g. geo restrictions) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java index e7391769..d822b2bb 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java @@ -81,6 +81,8 @@ public Optional fetchBlockedLists() { // Set the Accept-Encoding header to gzip connection.setRequestProperty("Accept-Encoding", "gzip"); connection.setRequestProperty("Authorization", token.get()); + // Indicates to the server that this agent supports the new format with monitoring + connection.setRequestProperty("x-supports-monitoring", "true"); if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { return Optional.empty(); From 41a5610884d77d51e25931dca0f49a27d3c8883e Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Fri, 18 Apr 2025 14:27:07 +0200 Subject: [PATCH 12/45] Store the response in a special ParsedFirewallLists object --- .../background/cloud/api/ReportingApi.java | 2 +- .../storage/ServiceConfiguration.java | 32 ++------- .../ParsedFirewallLists.java | 65 +++++++++++++++++++ 3 files changed, 70 insertions(+), 29 deletions(-) create mode 100644 agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java index 299f8543..b88ada8c 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java @@ -31,7 +31,7 @@ public ReportingApi(int timeoutInSec) { public record APIListsResponse( List blockedIPAddresses, List allowedIPAddresses, - BotBlocklist blockedUserAgents + List blockedUserAgents ) {} public record ListsResponseEntry(boolean monitor, String key, String source, String description, List ips) { diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java index e08d4b8f..8410b14d 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java @@ -4,6 +4,7 @@ import dev.aikido.agent_api.background.cloud.api.APIResponse; import dev.aikido.agent_api.background.cloud.api.ReportingApi; import dev.aikido.agent_api.helpers.net.IPList; +import dev.aikido.agent_api.storage.service_configuration.ParsedFirewallLists; import java.util.ArrayList; import java.util.HashSet; @@ -18,16 +19,13 @@ * It is essential for e.g. rate limiting */ public class ServiceConfiguration { - private final List blockedIps = new ArrayList<>(); - private final List allowedIps = new ArrayList<>(); + private final ParsedFirewallLists firewallLists = new ParsedFirewallLists(); private boolean blockingEnabled; private boolean receivedAnyStats; private boolean middlewareInstalled; private IPList bypassedIPs = new IPList(); private HashSet blockedUserIDs = new HashSet<>(); private List endpoints = new ArrayList<>(); - // User-Agent Blocking (e.g. bot blocking) : - private Pattern blockedUserAgentRegex; public ServiceConfiguration() { this.receivedAnyStats = true; // true by default, waiting for the startup event @@ -112,29 +110,7 @@ public BlockedResult isIpBlocked(String ip) { } public void updateBlockedLists(ReportingApi.APIListsResponse res) { - // Update blocked IP addresses (e.g. for geo restrictions) : - blockedIps.clear(); - if (res.blockedIPAddresses() != null) { - for (ReportingApi.ListsResponseEntry entry : res.blockedIPAddresses()) { - IPList ipList = createIPList(entry.ips()); - blockedIps.add(new IPListEntry(ipList, entry.description())); - } - } - - // Update allowed IP addresses (e.g. for geo restrictions) : - allowedIps.clear(); - if (res.allowedIPAddresses() != null) { - for (ReportingApi.ListsResponseEntry entry : res.allowedIPAddresses()) { - IPList ipList = createIPList(entry.ips()); - this.allowedIps.add(new IPListEntry(ipList, entry.description())); - } - } - - // Update Blocked User-Agents regex - blockedUserAgentRegex = null; - if (res.blockedUserAgents() != null && !res.blockedUserAgents().isEmpty()) { - this.blockedUserAgentRegex = Pattern.compile(res.blockedUserAgents(), Pattern.CASE_INSENSITIVE); - } + this.firewallLists.update(res); } /** @@ -148,7 +124,7 @@ public boolean isBlockedUserAgent(String userAgent) { } // IP restrictions (e.g. Geo-IP Restrictions) : - public record IPListEntry(IPList ipList, String description) { + public record IPListEntry(IPList ipList, String description, boolean monitor, String key) { } public record BlockedResult(boolean blocked, String description) { diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java new file mode 100644 index 00000000..1f7e17c2 --- /dev/null +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java @@ -0,0 +1,65 @@ +package dev.aikido.agent_api.storage.service_configuration; + +import dev.aikido.agent_api.background.cloud.api.ReportingApi; +import dev.aikido.agent_api.helpers.net.IPList; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import static dev.aikido.agent_api.helpers.IPListBuilder.createIPList; + +public class ParsedFirewallLists { + public final List blockedIps = new ArrayList<>(); + public final List allowedIps = new ArrayList<>(); + public final List blockedUserAgents = new ArrayList<>(); + + public ParsedFirewallLists() { + + } + + public void update(ReportingApi.APIListsResponse response) { + updateBlockedIps(response.blockedIPAddresses()); + updateAllowedIps(response.allowedIPAddresses()); + updateBlockedUserAgents(response.blockedUserAgents()); + } + + public void updateBlockedIps(List blockedIpsList) { + blockedIps.clear(); + if (blockedIpsList != null) + return; + for (ReportingApi.ListsResponseEntry entry : blockedIpsList) { + IPList ipList = createIPList(entry.ips()); + blockedIps.add(new BlockedIPEntry(entry.monitor(), entry.key(), entry.source(), entry.description(), ipList)); + } + } + + public void updateAllowedIps(List allowedIpsList) { + allowedIps.clear(); + if (allowedIpsList != null) + return; + for (ReportingApi.ListsResponseEntry entry : allowedIpsList) { + IPList ipList = createIPList(entry.ips()); + allowedIps.add(new AllowedIPEntry(entry.key(), entry.source(), entry.description(), ipList)); + } + } + + public void updateBlockedUserAgents(List blockedUserAgentsList) { + blockedUserAgents.clear(); + if (blockedUserAgentsList == null) + return; + for (ReportingApi.BotBlocklist entry : blockedUserAgentsList) { + Pattern pattern = Pattern.compile(entry.pattern(), Pattern.CASE_INSENSITIVE); + blockedUserAgents.add(new BlockedUAEntry(entry.monitor(), entry.key(), pattern)); + } + } + + public record BlockedIPEntry(boolean monitor, String key, String source, String description, IPList ips) { + } + + public record AllowedIPEntry(String key, String source, String description, IPList ips) { + } + + public record BlockedUAEntry(boolean monitor, String key, Pattern pattern) { + } +} From bdaf3bfa4136508d491a11329cef0bb6c2d5e7aa Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Fri, 18 Apr 2025 14:43:56 +0200 Subject: [PATCH 13/45] WebRequestCollector run all scans --- .../collectors/WebRequestCollector.java | 47 +++++++++++++------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java b/agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java index 29c4cab8..58ef79ca 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java @@ -3,6 +3,7 @@ import dev.aikido.agent_api.background.Endpoint; import dev.aikido.agent_api.context.Context; import dev.aikido.agent_api.context.ContextObject; +import dev.aikido.agent_api.context.RouteMetadata; import dev.aikido.agent_api.storage.ServiceConfiguration; import dev.aikido.agent_api.storage.statistics.StatisticsStore; @@ -38,31 +39,47 @@ public static Res report(ContextObject newContext) { // Increment total hits : StatisticsStore.incrementHits(); - // Per-route IP allowlists : - List matchedEndpoints = matchEndpoints(newContext.getRouteMetadata(), config.getEndpoints()); - if (!ipAllowedToAccessRoute(newContext.getRemoteAddress(), matchedEndpoints)) { + Res endpointAllowlistRes = checkEndpointAllowlist(newContext.getRouteMetadata(), newContext.getRemoteAddress(), config); + Res blockedIpsRes = checkBlockedIps(newContext.getRemoteAddress(), config); + Res blockedUserAgentsRes = checkBlockedUserAgents(newContext.getHeader("user-agent"), config); + + // make sure to follow a certain order when giving error messages. + if (endpointAllowlistRes != null) + return endpointAllowlistRes; + else if (blockedIpsRes != null) + return blockedIpsRes; + else return blockedUserAgentsRes; + } + + private static Res checkEndpointAllowlist(RouteMetadata routeMetadata, String remoteAddress, ServiceConfiguration config) { + List matchedEndpoints = matchEndpoints(routeMetadata, config.getEndpoints()); + if (!ipAllowedToAccessRoute(remoteAddress, matchedEndpoints)) { String msg = "Your IP address is not allowed to access this resource."; - msg += " (Your IP: " + newContext.getRemoteAddress() + ")"; + msg += " (Your IP: " + remoteAddress + ")"; return new Res(msg, 403); } + return null; // not blocked + } - // Blocked IP lists (e.g. Geo restrictions) - ServiceConfiguration.BlockedResult ipBlocked = config.isIpBlocked(newContext.getRemoteAddress()); + private static Res checkBlockedIps(String remoteAddress, ServiceConfiguration config) { + ServiceConfiguration.BlockedResult ipBlocked = config.isIpBlocked(remoteAddress); if (ipBlocked.blocked()) { String msg = "Your IP address is blocked. Reason: " + ipBlocked.description(); - msg += " (Your IP: " + newContext.getRemoteAddress() + ")"; + msg += " (Your IP: " + remoteAddress + ")"; return new Res(msg, 403); } + return null; // not blocked + } - // User-Agent blocking (e.g. blocking bots) - String userAgent = newContext.getHeader("user-agent"); - if (userAgent != null && !userAgent.isEmpty()) { - if (config.isBlockedUserAgent(userAgent)) { - String msg = "You are not allowed to access this resource because you have been identified as a bot."; - return new Res(msg, 403); - } + private static Res checkBlockedUserAgents(String userAgent, ServiceConfiguration config) { + if (userAgent == null || userAgent.isEmpty()) { + return null; // not blocked + } + if (config.isBlockedUserAgent(userAgent)) { + String msg = "You are not allowed to access this resource because you have been identified as a bot."; + return new Res(msg, 403); } - return null; + return null; // not blocked } public record Res(String msg, Integer status) { From 94cf59366f96eba71e90c2532b568e9ba028c58b Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Mon, 28 Apr 2025 16:10:23 +0200 Subject: [PATCH 14/45] Add matching code to ParsedFirewallLists --- .../ParsedFirewallLists.java | 48 +++++++++++++++---- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java index 1f7e17c2..fc8a4453 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java @@ -10,14 +10,42 @@ import static dev.aikido.agent_api.helpers.IPListBuilder.createIPList; public class ParsedFirewallLists { - public final List blockedIps = new ArrayList<>(); - public final List allowedIps = new ArrayList<>(); - public final List blockedUserAgents = new ArrayList<>(); + private final List blockedIps = new ArrayList<>(); + private final List allowedIps = new ArrayList<>(); + private final List blockedUserAgents = new ArrayList<>(); public ParsedFirewallLists() { } + private static List matchIpEntries(String ip, List ipEntries) { + List matches = new ArrayList<>(); + for (IPEntry entry : ipEntries) { + if (entry.ips().matches(ip)) { + matches.add(new Match(entry.key(), !entry.monitor(), entry.description())); + } + } + return matches; + } + + public List matchBlockedIps(String ip) { + return matchIpEntries(ip, this.blockedIps); + } + + public List matchAllowedIps(String ip) { + return matchIpEntries(ip, this.allowedIps); + } + + public List matchBlockedUserAgents(String userAgent) { + List matches = new ArrayList<>(); + for (BlockedUAEntry entry : this.blockedUserAgents) { + if (entry.pattern().matcher(userAgent).find()) { + matches.add(new Match(entry.key(), !entry.monitor(), null)); + } + } + return matches; + } + public void update(ReportingApi.APIListsResponse response) { updateBlockedIps(response.blockedIPAddresses()); updateAllowedIps(response.allowedIPAddresses()); @@ -26,21 +54,21 @@ public void update(ReportingApi.APIListsResponse response) { public void updateBlockedIps(List blockedIpsList) { blockedIps.clear(); - if (blockedIpsList != null) + if (blockedIpsList == null) return; for (ReportingApi.ListsResponseEntry entry : blockedIpsList) { IPList ipList = createIPList(entry.ips()); - blockedIps.add(new BlockedIPEntry(entry.monitor(), entry.key(), entry.source(), entry.description(), ipList)); + blockedIps.add(new IPEntry(entry.monitor(), entry.key(), entry.source(), entry.description(), ipList)); } } public void updateAllowedIps(List allowedIpsList) { allowedIps.clear(); - if (allowedIpsList != null) + if (allowedIpsList == null) return; for (ReportingApi.ListsResponseEntry entry : allowedIpsList) { IPList ipList = createIPList(entry.ips()); - allowedIps.add(new AllowedIPEntry(entry.key(), entry.source(), entry.description(), ipList)); + allowedIps.add(new IPEntry(entry.monitor(), entry.key(), entry.source(), entry.description(), ipList)); } } @@ -54,12 +82,12 @@ public void updateBlockedUserAgents(List blockedUserA } } - public record BlockedIPEntry(boolean monitor, String key, String source, String description, IPList ips) { + public record Match(String key, boolean block, String description) { } - public record AllowedIPEntry(String key, String source, String description, IPList ips) { + private record IPEntry(boolean monitor, String key, String source, String description, IPList ips) { } - public record BlockedUAEntry(boolean monitor, String key, Pattern pattern) { + private record BlockedUAEntry(boolean monitor, String key, Pattern pattern) { } } From 1eff096414685872d912219112272d5c4e225cf4 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Mon, 28 Apr 2025 16:15:36 +0200 Subject: [PATCH 15/45] Use the new ParsedFirewallLists, keep boolean match for allowed ips --- .../storage/ServiceConfiguration.java | 35 +++++++++---------- .../ParsedFirewallLists.java | 18 ++++++++-- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java index 8410b14d..f425fc45 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java @@ -85,28 +85,23 @@ public boolean isIpBypassed(String ip) { * Check if the IP is blocked (e.g. Geo IP Restrictions) */ public BlockedResult isIpBlocked(String ip) { + BlockedResult blockedResult = new BlockedResult(false, null); + // Check for allowed ip addresses (i.e. only one country is allowed to visit the site) // Always allow access from private IP addresses (those include local IP addresses) - if (!allowedIps.isEmpty() && !isPrivateIp(ip)) { - boolean ipAllowed = false; - for (IPListEntry entry : allowedIps) { - if (entry.ipList.matches(ip)) { - ipAllowed = true; // We allow IP addresses as long as they match with one of the lists. - break; - } - } - if (!ipAllowed) { - return new BlockedResult(true, "not in allowlist"); - } + if (!isPrivateIp(ip) && firewallLists.matchesAllowedIps(ip)) { + blockedResult = new BlockedResult(true, "not in allowlist"); } // Check for blocked ip addresses - for (IPListEntry entry : blockedIps) { - if (entry.ipList.matches(ip)) { - return new BlockedResult(true, entry.description); + for (ParsedFirewallLists.Match match: firewallLists.matchBlockedIps(ip)) { + // when a blocking match is found, set blocked result if it hasn't been set already. + if (match.block() && !blockedResult.blocked()) { + blockedResult = new BlockedResult(true, match.description()); } } - return new BlockedResult(false, null); + + return blockedResult; } public void updateBlockedLists(ReportingApi.APIListsResponse res) { @@ -117,10 +112,14 @@ public void updateBlockedLists(ReportingApi.APIListsResponse res) { * Check if a given User-Agent is blocked or not : */ public boolean isBlockedUserAgent(String userAgent) { - if (blockedUserAgentRegex != null) { - return blockedUserAgentRegex.matcher(userAgent).find(); + boolean blocked = false; + for(ParsedFirewallLists.Match match: this.firewallLists.matchBlockedUserAgents(userAgent)) { + if (match.block()) { + blocked = true; + } } - return false; + + return blocked; } // IP restrictions (e.g. Geo-IP Restrictions) : diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java index fc8a4453..8a108093 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java @@ -29,11 +29,23 @@ private static List matchIpEntries(String ip, List ipEntries) { } public List matchBlockedIps(String ip) { - return matchIpEntries(ip, this.blockedIps); + List matches = new ArrayList<>(); + for (IPEntry entry : this.blockedIps) { + if (entry.ips().matches(ip)) { + matches.add(new Match(entry.key(), !entry.monitor(), entry.description())); + } + } + return matches; } - public List matchAllowedIps(String ip) { - return matchIpEntries(ip, this.allowedIps); + // returns true if one or more matches has been found with allowlist. + public boolean matchesAllowedIps(String ip) { + for (IPEntry entry : this.allowedIps) { + if (entry.ips().matches(ip)) { + return true; + } + } + return false; } public List matchBlockedUserAgents(String userAgent) { From 73a68f9c1fe32d60e55249009d56ed082da60593 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Mon, 28 Apr 2025 16:16:02 +0200 Subject: [PATCH 16/45] refactor for ServiceConfiguration.java --- .../dev/aikido/agent_api/storage/ServiceConfiguration.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java index f425fc45..1ab893f1 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java @@ -9,7 +9,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.regex.Pattern; import static dev.aikido.agent_api.helpers.IPListBuilder.createIPList; import static dev.aikido.agent_api.vulnerabilities.ssrf.IsPrivateIP.isPrivateIp; @@ -94,7 +93,7 @@ public BlockedResult isIpBlocked(String ip) { } // Check for blocked ip addresses - for (ParsedFirewallLists.Match match: firewallLists.matchBlockedIps(ip)) { + for (ParsedFirewallLists.Match match : firewallLists.matchBlockedIps(ip)) { // when a blocking match is found, set blocked result if it hasn't been set already. if (match.block() && !blockedResult.blocked()) { blockedResult = new BlockedResult(true, match.description()); @@ -113,9 +112,10 @@ public void updateBlockedLists(ReportingApi.APIListsResponse res) { */ public boolean isBlockedUserAgent(String userAgent) { boolean blocked = false; - for(ParsedFirewallLists.Match match: this.firewallLists.matchBlockedUserAgents(userAgent)) { + for (ParsedFirewallLists.Match match : this.firewallLists.matchBlockedUserAgents(userAgent)) { if (match.block()) { blocked = true; + break; } } From 30f0438ad53c9df95bb06a46153dcced5e985fcf Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 01:08:24 +0200 Subject: [PATCH 17/45] Update test cases and remove unused code --- .../statistics/FirewallListsRecord.java | 20 ------ .../storage/statistics/StatisticsStore.java | 24 ------- .../collectors/WebRequestCollectorTest.java | 36 +++++------ .../storage/ServiceConfigurationTest.java | 63 ++++++++++--------- .../java/storage/StatisticsStoreTest.java | 18 ------ .../test/java/utils/EmptyAPIResponses.java | 2 +- 6 files changed, 52 insertions(+), 111 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java index 02240a22..5d3ded12 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java @@ -5,8 +5,6 @@ public class FirewallListsRecord { private final Map breakdown = new HashMap<>(); - private int total = 0; - private int blocked = 0; public FirewallListsRecord() { } @@ -27,25 +25,7 @@ public BreakdownEntry get(String key) { return this.breakdown.get(key); } - public void incrementTotal() { - this.total += 1; - } - - public int getTotal() { - return this.total; - } - - public void incrementBlocked() { - this.blocked += 1; - } - - public int getBlocked() { - return this.blocked; - } - public void clear() { - this.total = 0; - this.blocked = 0; this.breakdown.clear(); } diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java index eb4fb0d5..12177fcc 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java @@ -71,18 +71,6 @@ public static void incrementIpHits(String key, boolean blocked) { } } - public static void incrementTotalIpHits(boolean blocked) { - mutex.lock(); - try { - stats.getIpAddresses().incrementTotal(); - if (blocked) { - stats.getIpAddresses().incrementBlocked(); - } - } finally { - mutex.unlock(); - } - } - public static void incrementUAHits(String key, boolean blocked) { mutex.lock(); try { @@ -92,18 +80,6 @@ public static void incrementUAHits(String key, boolean blocked) { } } - public static void incrementTotalUAHits(boolean blocked) { - mutex.lock(); - try { - stats.getUserAgents().incrementTotal(); - if (blocked) { - stats.getUserAgents().incrementBlocked(); - } - } finally { - mutex.unlock(); - } - } - public static void clear() { mutex.lock(); try { diff --git a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java index ca036c2e..a539b6de 100644 --- a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java +++ b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java @@ -84,8 +84,8 @@ void testReport_no_endpoints() { @Test void testReport_ipBlockedTwice() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.1")) - ), null, ""); + new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.1")) + ), null, List.of()); setEmptyConfigWithEndpointList(List.of( new Endpoint( "GET", "/api/resource", 100, 100, @@ -105,8 +105,8 @@ void testReport_ipBlockedTwice() { @Test void testReport_ipBlockedUsingLists() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) - ), null, ""); + new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) + ), null, List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); @@ -120,8 +120,8 @@ void testReport_ipBlockedUsingLists() { @Test void testReport_ipNotAllowedUsingLists() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(null, List.of( - new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.2.1")) - ), ""); + new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.2.1")) + ), List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); @@ -139,8 +139,8 @@ void testReport_ipNotAllowedUsingLists() { @Test void testReport_ipInAllowlist() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(null, List.of( - new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.1", "10.0.0.0/24")) - ), ""); + new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.1", "10.0.0.0/24")) + ), List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); @@ -152,8 +152,8 @@ void testReport_ipInAllowlist() { @Test void testReport_ipNotBlockedUsingListsNorUserAgent() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) - ), null, "Unrelated|random"); + new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) + ), null, List.of(new ReportingApi.BotBlocklist(false, "block", "Unrelated|random"))); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); @@ -165,8 +165,8 @@ void testReport_ipNotBlockedUsingListsNorUserAgent() { @Test void testReport_userAgentBlocked() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) - ), null, "AI2Bot|hacker"); + new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) + ), null, List.of(new ReportingApi.BotBlocklist(false, "block", "AI2Bot|hacker"))); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); WebRequestCollector.Res response = WebRequestCollector.report(contextObject); @@ -179,8 +179,8 @@ void testReport_userAgentBlocked() { @Test void testReport_userAgentBlocked_Ip_Bypassed() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) - ), null, "AI2Bot|hacker"); + new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) + ), null, List.of(new ReportingApi.BotBlocklist(false, "block", "AI2Bot|hacker"))); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); List bypassedIps = List.of("192.168.1.1"); @@ -198,8 +198,8 @@ void testReport_userAgentBlocked_Ip_Bypassed() { @Test void testReport_ipBlockedUsingLists_Ip_Bypassed() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) - ), null, ""); + new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) + ), null, List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); List bypassedIps = List.of("192.168.1.1"); @@ -217,8 +217,8 @@ void testReport_ipBlockedUsingLists_Ip_Bypassed() { void testReport_ipNotAllowedUsingLists_Ip_Bypassed() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse( null, - List.of(new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("1.2.3.4"))), - "" + List.of(new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("1.2.3.4"))), + List.of() ); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); diff --git a/agent_api/src/test/java/storage/ServiceConfigurationTest.java b/agent_api/src/test/java/storage/ServiceConfigurationTest.java index 520bfce5..8b71bb3e 100644 --- a/agent_api/src/test/java/storage/ServiceConfigurationTest.java +++ b/agent_api/src/test/java/storage/ServiceConfigurationTest.java @@ -69,11 +69,11 @@ public void testUpdateConfigWithUnsuccessfulResponse() { @Test public void testIsIpBlocked() { - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), Collections.emptyList(), - null + Collections.emptyList() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -85,11 +85,11 @@ public void testIsIpBlocked() { @Test public void testIsIpAllowed() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry("source", "allowed", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( Collections.emptyList(), List.of(allowedEntry), - null + Collections.emptyList() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -103,7 +103,7 @@ public void testIsBlockedUserAgent() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( Collections.emptyList(), Collections.emptyList(), - "blocked-agent" + List.of(new ReportingApi.BotBlocklist(false, "blocked-bots", "blocked-agent")) ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -133,7 +133,7 @@ public void testIsIpBlockedWithEmptyAllowedList() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( Collections.emptyList(), Collections.emptyList(), - null + Collections.emptyList() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -144,12 +144,12 @@ public void testIsIpBlockedWithEmptyAllowedList() { @Test public void testIsIpBlockedWithMultipleBlockedEntries() { - ReportingApi.ListsResponseEntry blockedEntry1 = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("192.168.1.1")); - ReportingApi.ListsResponseEntry blockedEntry2 = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("192.168.1.2")); + ReportingApi.ListsResponseEntry blockedEntry1 = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry blockedEntry2 = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.2")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry1, blockedEntry2), Collections.emptyList(), - null + Collections.emptyList() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -162,12 +162,12 @@ public void testIsIpBlockedWithMultipleBlockedEntries() { @Test public void testIsIpBlockedWithMixedAllowedAndBlockedEntries() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry("source", "allowed", List.of("192.168.1.1")); - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("192.168.1.2")); + ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.2")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), List.of(allowedEntry), - null + Collections.emptyList() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -183,7 +183,7 @@ public void testIsBlockedUserAgentWithNullRegex() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( Collections.emptyList(), Collections.emptyList(), - null + Collections.emptyList() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -196,7 +196,7 @@ public void testIsBlockedUserAgentWithCaseInsensitiveMatch() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( Collections.emptyList(), Collections.emptyList(), - "blocked-agent" + List.of(new ReportingApi.BotBlocklist(false, "blocked-bots", "blocked-agent")) ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -339,11 +339,11 @@ public void testGetEndpointsWithEmptyList() { @Test public void testIsIpBlockedWithSubnet() { - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("192.168.1.0/24")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.0/24")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), - List.of(), - null + Collections.emptyList(), + Collections.emptyList() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -355,12 +355,12 @@ public void testIsIpBlockedWithSubnet() { @Test public void testIsIpBlockedWithMixedSubnetAndSingleIP() { - ReportingApi.ListsResponseEntry blockedEntry1 = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("192.168.1.0/24")); - ReportingApi.ListsResponseEntry blockedEntry2 = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("10.0.0.1")); + ReportingApi.ListsResponseEntry blockedEntry1 = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.0/24")); + ReportingApi.ListsResponseEntry blockedEntry2 = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("10.0.0.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry1, blockedEntry2), - List.of(), - null + Collections.emptyList(), + Collections.emptyList() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -376,7 +376,7 @@ public void testIsIpBlockedWithEmptyBlockedList() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(), List.of(), - null + Collections.emptyList() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -390,7 +390,7 @@ public void testIsBlockedUserAgentWithEmptyRegex() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(), List.of(), - "" + List.of(new ReportingApi.BotBlocklist(false, "empty", "")) ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -403,7 +403,10 @@ public void testIsBlockedUserAgentWithComplexRegex() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(), List.of(), - "blocked.*agent|another.*pattern" + List.of( + new ReportingApi.BotBlocklist(false, "block1", "blocked.*agent"), + new ReportingApi.BotBlocklist(false, "block2", "another.*pattern") + ) ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -415,8 +418,8 @@ public void testIsBlockedUserAgentWithComplexRegex() { @Test public void testIsIpBlockedWithAllowedAndBlockedIPs() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry("source", "allowed", List.of("10.0.0.1")); - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), List.of(allowedEntry), @@ -433,7 +436,7 @@ public void testIsIpBlockedWithAllowedAndBlockedIPs() { @Test public void testIsIpBlockedWithOnlyAllowedIPs() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry("source", "allowed", List.of("10.0.0.1")); + ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(), List.of(allowedEntry), @@ -450,7 +453,7 @@ public void testIsIpBlockedWithOnlyAllowedIPs() { @Test public void testIsIpBlockedWithOnlyBlockedIPs() { - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), List.of(), @@ -467,8 +470,8 @@ public void testIsIpBlockedWithOnlyBlockedIPs() { @Test public void testIsIpBlockedWithAllowedIPsAndBlockedIPs() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry("source", "allowed", List.of("10.0.0.1")); - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), List.of(allowedEntry), diff --git a/agent_api/src/test/java/storage/StatisticsStoreTest.java b/agent_api/src/test/java/storage/StatisticsStoreTest.java index df710fbb..b37d95f9 100644 --- a/agent_api/src/test/java/storage/StatisticsStoreTest.java +++ b/agent_api/src/test/java/storage/StatisticsStoreTest.java @@ -238,24 +238,6 @@ public void testClearAfterIncrementingDifferentMetrics() { assertEquals(0, record.userAgents().size()); } - @Test - public void testIncrementTotalIpHits() { - StatisticsStore.incrementTotalIpHits(false); - Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); - assertNotNull(record); - assertEquals(1, record.ipAddresses().getTotal()); - assertEquals(0, record.ipAddresses().getBlocked()); - } - - @Test - public void testIncrementTotalUAHits() { - StatisticsStore.incrementTotalUAHits(true); - Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); - assertNotNull(record); - assertEquals(1, record.userAgents().getTotal()); - assertEquals(1, record.userAgents().getBlocked()); - } - @Test public void testConcurrentIncrementHitsAndClear() throws InterruptedException { Runnable incrementHitsTask = () -> { diff --git a/agent_api/src/test/java/utils/EmptyAPIResponses.java b/agent_api/src/test/java/utils/EmptyAPIResponses.java index 799478d2..ea259b28 100644 --- a/agent_api/src/test/java/utils/EmptyAPIResponses.java +++ b/agent_api/src/test/java/utils/EmptyAPIResponses.java @@ -15,7 +15,7 @@ public class EmptyAPIResponses { true, "", UnixTimeMS.getUnixTimeMS(), List.of(), List.of(), List.of(), true, false ); public final static ReportingApi.APIListsResponse emptyAPIListsResponse = new ReportingApi.APIListsResponse( - List.of(), List.of(), "" + List.of(), List.of(), List.of() ); public static void setEmptyConfigWithEndpointList(List endpoints) { ServiceConfigStore.updateFromAPIResponse(new APIResponse( From 7d5537e4a730079805cf891c747fe73baeca2deb Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 01:10:50 +0200 Subject: [PATCH 18/45] Report hits, fixes bug with test cases --- .../dev/aikido/agent_api/storage/ServiceConfiguration.java | 5 ++++- .../src/test/java/storage/ServiceConfigurationTest.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java index 1ab893f1..39f62442 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java @@ -5,6 +5,7 @@ import dev.aikido.agent_api.background.cloud.api.ReportingApi; import dev.aikido.agent_api.helpers.net.IPList; import dev.aikido.agent_api.storage.service_configuration.ParsedFirewallLists; +import dev.aikido.agent_api.storage.statistics.StatisticsStore; import java.util.ArrayList; import java.util.HashSet; @@ -88,12 +89,13 @@ public BlockedResult isIpBlocked(String ip) { // Check for allowed ip addresses (i.e. only one country is allowed to visit the site) // Always allow access from private IP addresses (those include local IP addresses) - if (!isPrivateIp(ip) && firewallLists.matchesAllowedIps(ip)) { + if (!isPrivateIp(ip) && !firewallLists.matchesAllowedIps(ip)) { blockedResult = new BlockedResult(true, "not in allowlist"); } // Check for blocked ip addresses for (ParsedFirewallLists.Match match : firewallLists.matchBlockedIps(ip)) { + StatisticsStore.incrementIpHits(match.key(), match.block()); // when a blocking match is found, set blocked result if it hasn't been set already. if (match.block() && !blockedResult.blocked()) { blockedResult = new BlockedResult(true, match.description()); @@ -113,6 +115,7 @@ public void updateBlockedLists(ReportingApi.APIListsResponse res) { public boolean isBlockedUserAgent(String userAgent) { boolean blocked = false; for (ParsedFirewallLists.Match match : this.firewallLists.matchBlockedUserAgents(userAgent)) { + StatisticsStore.incrementUAHits(match.key(), match.block()); if (match.block()) { blocked = true; break; diff --git a/agent_api/src/test/java/storage/ServiceConfigurationTest.java b/agent_api/src/test/java/storage/ServiceConfigurationTest.java index 8b71bb3e..11625071 100644 --- a/agent_api/src/test/java/storage/ServiceConfigurationTest.java +++ b/agent_api/src/test/java/storage/ServiceConfigurationTest.java @@ -390,7 +390,7 @@ public void testIsBlockedUserAgentWithEmptyRegex() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(), List.of(), - List.of(new ReportingApi.BotBlocklist(false, "empty", "")) + List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); From e8dcf3d07117d7af91cd20743a5f1563a0c19440 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 01:24:36 +0200 Subject: [PATCH 19/45] Fix bug for uas and update service config unit tests to test for stats --- .../storage/ServiceConfiguration.java | 1 - .../storage/ServiceConfigurationTest.java | 69 ++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java index 39f62442..2ca0c354 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java @@ -118,7 +118,6 @@ public boolean isBlockedUserAgent(String userAgent) { StatisticsStore.incrementUAHits(match.key(), match.block()); if (match.block()) { blocked = true; - break; } } diff --git a/agent_api/src/test/java/storage/ServiceConfigurationTest.java b/agent_api/src/test/java/storage/ServiceConfigurationTest.java index 11625071..0c90b049 100644 --- a/agent_api/src/test/java/storage/ServiceConfigurationTest.java +++ b/agent_api/src/test/java/storage/ServiceConfigurationTest.java @@ -4,6 +4,7 @@ import dev.aikido.agent_api.background.cloud.api.APIResponse; import dev.aikido.agent_api.background.cloud.api.ReportingApi; import dev.aikido.agent_api.storage.ServiceConfiguration; +import dev.aikido.agent_api.storage.statistics.StatisticsStore; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -20,6 +21,7 @@ public class ServiceConfigurationTest { @BeforeEach public void setUp() { serviceConfiguration = new ServiceConfiguration(); + StatisticsStore.clear(); } @Test @@ -69,7 +71,7 @@ public void testUpdateConfigWithUnsuccessfulResponse() { @Test public void testIsIpBlocked() { - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "my-key", "source", "blocked", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), Collections.emptyList(), @@ -81,6 +83,39 @@ public void testIsIpBlocked() { ServiceConfiguration.BlockedResult result = serviceConfiguration.isIpBlocked("192.168.1.1"); assertTrue(result.blocked()); assertEquals("blocked", result.description()); + assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("my-key").total()); + assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("my-key").blocked()); + } + + @Test + public void testIsIpBlockedOrMonitored() { + var blockedEntry = new ReportingApi.ListsResponseEntry(false, "blocked-key", "source", "blocked", List.of("192.168.1.1")); + var monitoredEntry = new ReportingApi.ListsResponseEntry(true, "monitored-key", "source", "blocked", List.of("192.168.2.*")); + + ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( + List.of(blockedEntry, monitoredEntry), + Collections.emptyList(), + Collections.emptyList() + ); + + serviceConfiguration.updateBlockedLists(listsResponse); + + ServiceConfiguration.BlockedResult result = serviceConfiguration.isIpBlocked("192.168.1.1"); + assertTrue(result.blocked()); + assertEquals("blocked", result.description()); + assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("blocked-key").total()); + assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("blocked-key").blocked()); + assertNull(StatisticsStore.getStatsRecord().ipAddresses().get("monitored-key")); + + // now test with multiple ip that are only monitored + assertFalse(serviceConfiguration.isIpBlocked("192.168.2.20").blocked()); + assertFalse(serviceConfiguration.isIpBlocked("192.168.2.128").blocked()); + assertFalse(serviceConfiguration.isIpBlocked("192.168.3.20").blocked()); + assertFalse(serviceConfiguration.isIpBlocked("192.168.2.5").blocked()); + assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("blocked-key").total()); + assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("blocked-key").blocked()); + assertEquals(3, StatisticsStore.getStatsRecord().ipAddresses().get("monitored-key").total()); + assertEquals(0, StatisticsStore.getStatsRecord().ipAddresses().get("monitored-key").blocked()); } @Test @@ -110,6 +145,38 @@ public void testIsBlockedUserAgent() { assertTrue(serviceConfiguration.isBlockedUserAgent("blocked-agent")); assertFalse(serviceConfiguration.isBlockedUserAgent("allowed-agent")); + assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("blocked-bots").total()); + assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("blocked-bots").blocked()); + } + + @Test + public void testMonitoredAndBlockedUserAgents() { + var monitoring1 = new ReportingApi.BotBlocklist(true, "monitoring1", "blocked-agent"); + var blocking1 = new ReportingApi.BotBlocklist(false, "blocking1", "blocked-agent"); + var monitoring2 = new ReportingApi.BotBlocklist(true, "monitoring2", "allowed*"); + var blocking2 = new ReportingApi.BotBlocklist(false, "blocking2", "blocked*"); + + ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( + Collections.emptyList(), + Collections.emptyList(), + List.of(monitoring1, blocking1, monitoring2, blocking2) + ); + + serviceConfiguration.updateBlockedLists(listsResponse); + + assertTrue(serviceConfiguration.isBlockedUserAgent("blocked-agent")); + assertTrue(serviceConfiguration.isBlockedUserAgent("blocked-a2")); + assertFalse(serviceConfiguration.isBlockedUserAgent("allowed-agent")); + assertFalse(serviceConfiguration.isBlockedUserAgent("allowed-a2")); + assertFalse(serviceConfiguration.isBlockedUserAgent("allowed-a3")); + assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("blocking1").total()); + assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("blocking1").blocked()); + assertEquals(2, StatisticsStore.getStatsRecord().userAgents().get("blocking2").total()); + assertEquals(2, StatisticsStore.getStatsRecord().userAgents().get("blocking2").blocked()); + assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("monitoring1").total()); + assertEquals(0, StatisticsStore.getStatsRecord().userAgents().get("monitoring1").blocked()); + assertEquals(3, StatisticsStore.getStatsRecord().userAgents().get("monitoring2").total()); + assertEquals(0, StatisticsStore.getStatsRecord().userAgents().get("monitoring2").blocked()); } @Test From 60e0cb05f95ca24075daffe928e965681cb8c56b Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 01:29:48 +0200 Subject: [PATCH 20/45] Update the mock core server --- end2end/server/mock_aikido_core.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/end2end/server/mock_aikido_core.py b/end2end/server/mock_aikido_core.py index c04f4b03..85c15553 100644 --- a/end2end/server/mock_aikido_core.py +++ b/end2end/server/mock_aikido_core.py @@ -50,12 +50,25 @@ "success": True, "blockedIPAddresses": [ { + "monitor": False, + "key": "geo-1", "source": "geoip", "description": "geo restrictions", "ips": ["1.2.3.4"] } ], - "blockedUserAgents": "AI2Bot|Bytespider", + "blockedUserAgents": [ + { + "monitor": False, + "key": "ai-agents", + "pattern": "AI2Bot" + }, + { + "monitor": False, + "key": "crawlers", + "pattern": "Bytespider" + } + ], }, "configUpdatedAt": 0, } @@ -145,5 +158,5 @@ def mock_set_protection(): else: print(f"Error: File {config_file} not found") sys.exit(1) - + app.run(host='0.0.0.0', port=port) From 882cf7a6505a278f05b36054460b122b9a10f72b Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 01:37:43 +0200 Subject: [PATCH 21/45] Update e2e tests to also check monitoring is monitored --- end2end/server/mock_aikido_core.py | 12 ++++++++++++ end2end/utils/test_bot_blocking.py | 17 +++++++++++++++++ end2end/utils/test_ip_blocking.py | 8 +++++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/end2end/server/mock_aikido_core.py b/end2end/server/mock_aikido_core.py index 85c15553..ef8475ec 100644 --- a/end2end/server/mock_aikido_core.py +++ b/end2end/server/mock_aikido_core.py @@ -55,6 +55,13 @@ "source": "geoip", "description": "geo restrictions", "ips": ["1.2.3.4"] + }, + { + "monitor": True, + "key": "geo-2", + "source": "geoip", + "description": "should not be blocked", + "ips": ["5.6.7.8"] } ], "blockedUserAgents": [ @@ -67,6 +74,11 @@ "monitor": False, "key": "crawlers", "pattern": "Bytespider" + }, + { + "monitor": True, + "key": "crawlers-monitor", + "pattern": "ClaudeUser" } ], }, diff --git a/end2end/utils/test_bot_blocking.py b/end2end/utils/test_bot_blocking.py index 12509786..a1a721f7 100644 --- a/end2end/utils/test_bot_blocking.py +++ b/end2end/utils/test_bot_blocking.py @@ -34,3 +34,20 @@ def test_bot_blocking(url): }) assert_eq(res.status_code, equals=403) assert_eq(res.text, equals="You are not allowed to access this resource because you have been identified as a bot.") + + # Monitored User-Agent : + res = requests.get(url, headers={ + 'User-Agent': "Mozilla/5.0 ClaudeUser 2025" + }) + assert_eq(res.status_code, equals=200) + res = requests.get(url, headers={ + 'User-Agent': "ClaudeUser" + }) + assert_eq(res.status_code, equals=200) + + # Still blocks if 2 matches are found : + res = requests.get(url, headers={ + 'User-Agent': "BYTESPIDER and ClaudeUser" + }) + assert_eq(res.status_code, equals=403) + assert_eq(res.text, equals="You are not allowed to access this resource because you have been identified as a bot.") diff --git a/end2end/utils/test_ip_blocking.py b/end2end/utils/test_ip_blocking.py index fed972fc..deccbe01 100644 --- a/end2end/utils/test_ip_blocking.py +++ b/end2end/utils/test_ip_blocking.py @@ -38,4 +38,10 @@ def test_ip_blocking(url): res = requests.get(url, headers={ 'X-Forwarded-For': "" }) - assert_eq(res.status_code, equals=200) \ No newline at end of file + assert_eq(res.status_code, equals=200) + + # Monitored IP + res = requests.get(url, headers={ + 'X-Forwarded-For': "5.6.7.8" + }) + assert_eq(res.status_code, equals=200) From c206ee0ba12453c854cb971bafa141544b4a3530 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 01:46:13 +0200 Subject: [PATCH 22/45] Fix bug for when allowed ips are empty, and add regression tests --- .../ParsedFirewallLists.java | 3 +++ .../collectors/WebRequestCollectorTest.java | 22 ++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java index 8a108093..e19a5ee2 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java @@ -40,6 +40,9 @@ public List matchBlockedIps(String ip) { // returns true if one or more matches has been found with allowlist. public boolean matchesAllowedIps(String ip) { + if (this.allowedIps.isEmpty()) { + return true; // Empty allowed is means all ips match + } for (IPEntry entry : this.allowedIps) { if (entry.ips().matches(ip)) { return true; diff --git a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java index a539b6de..b262bfde 100644 --- a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java +++ b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java @@ -89,7 +89,7 @@ void testReport_ipBlockedTwice() { setEmptyConfigWithEndpointList(List.of( new Endpoint( "GET", "/api/resource", 100, 100, - List.of("192.168.0.1"), false, false, false + List.of("192.168.0.1", "5.6.7.8"), false, false, false ) )); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); @@ -100,6 +100,11 @@ void testReport_ipBlockedTwice() { assertNotNull(response); assertEquals("Your IP address is not allowed to access this resource. (Your IP: 192.168.1.1)", response.msg()); assertEquals(403, response.status()); + + contextObject.setIp("5.6.7.8"); + WebRequestCollector.Res response2 = WebRequestCollector.report(contextObject); + assertNull(response2); + } @Test @@ -116,6 +121,21 @@ void testReport_ipBlockedUsingLists() { assertEquals("Your IP address is blocked. Reason: geoip restrictions (Your IP: 192.168.1.1)", response.msg()); assertEquals(403, response.status()); } + @Test + void testReport_publicIpBlockedUsingLists() { + ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( + new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("bullshit.ip", "2.2.2.0/24")) + ), null, List.of()); + ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); + + contextObject.setIp("2.2.2.7"); + WebRequestCollector.Res response = WebRequestCollector.report(contextObject); + + assertNotNull(response); + assertEquals("Your IP address is blocked. Reason: geoip restrictions (Your IP: 2.2.2.7)", response.msg()); + assertEquals(403, response.status()); + } + @Test void testReport_ipNotAllowedUsingLists() { From 3b43133cb50005b68d4d612aa36e8af447488b50 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 01:54:21 +0200 Subject: [PATCH 23/45] Create a copy so the clear action has no effect --- .../agent_api/storage/statistics/FirewallListsRecord.java | 4 ++++ .../dev/aikido/agent_api/storage/statistics/Statistics.java | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java index 5d3ded12..7d049279 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java @@ -9,6 +9,10 @@ public class FirewallListsRecord { public FirewallListsRecord() { } + public FirewallListsRecord(FirewallListsRecord previous) { + this.breakdown.putAll(previous.breakdown); + } + public void increment(String key, boolean blocked) { breakdown.computeIfAbsent(key, k -> new BreakdownEntry(0, 0)); diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java index e76d0346..838a1e49 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java @@ -92,7 +92,9 @@ public StatsRecord getRecord() { "total", attacksDetected, "blocked", attacksBlocked )), - getOperations(), getIpAddresses(), getUserAgents() + getOperations(), + new FirewallListsRecord(getIpAddresses()), + new FirewallListsRecord(getUserAgents()) ); } From a61a0eca3ff8e21431810486f5e120392bcfa732 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 14:32:56 +0200 Subject: [PATCH 24/45] Fix broken test which checks lists response --- .../src/test/java/background/cloud/ReportingAPITest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/agent_api/src/test/java/background/cloud/ReportingAPITest.java b/agent_api/src/test/java/background/cloud/ReportingAPITest.java index df317671..cae078ed 100644 --- a/agent_api/src/test/java/background/cloud/ReportingAPITest.java +++ b/agent_api/src/test/java/background/cloud/ReportingAPITest.java @@ -70,10 +70,13 @@ public void testListsResponseWithWrongEndpoint(StdOut out) { public void testListsResponse() { Optional res = api.fetchBlockedLists(); assertTrue(res.isPresent()); - assertEquals(1, res.get().blockedIPAddresses().size()); + assertEquals(2, res.get().blockedIPAddresses().size()); assertEquals("geoip", res.get().blockedIPAddresses().get(0).source()); assertEquals("geo restrictions", res.get().blockedIPAddresses().get(0).description()); assertEquals("1.2.3.4", res.get().blockedIPAddresses().get(0).ips().get(0)); - assertEquals("AI2Bot|Bytespider", res.get().blockedUserAgents()); + assertEquals(3, res.get().blockedUserAgents().size()); + assertEquals("AI2Bot", res.get().blockedUserAgents().get(0).pattern()); + assertEquals("Bytespider", res.get().blockedUserAgents().get(1).pattern()); + } } From 14bc88a8aa9b484cfe857cf0778f2416b05c1f0f Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 15:16:48 +0200 Subject: [PATCH 25/45] Remove unused code for ParsedFirewallLists --- .../service_configuration/ParsedFirewallLists.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java index e19a5ee2..d0951da5 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java @@ -18,16 +18,6 @@ public ParsedFirewallLists() { } - private static List matchIpEntries(String ip, List ipEntries) { - List matches = new ArrayList<>(); - for (IPEntry entry : ipEntries) { - if (entry.ips().matches(ip)) { - matches.add(new Match(entry.key(), !entry.monitor(), entry.description())); - } - } - return matches; - } - public List matchBlockedIps(String ip) { List matches = new ArrayList<>(); for (IPEntry entry : this.blockedIps) { From df63de2bbe21092c86d23b677aec5679162ac5a4 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Wed, 30 Apr 2025 15:18:52 +0200 Subject: [PATCH 26/45] Update test cases for better coverage --- .../test/java/storage/ServiceConfigurationTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/agent_api/src/test/java/storage/ServiceConfigurationTest.java b/agent_api/src/test/java/storage/ServiceConfigurationTest.java index 0c90b049..551ad157 100644 --- a/agent_api/src/test/java/storage/ServiceConfigurationTest.java +++ b/agent_api/src/test/java/storage/ServiceConfigurationTest.java @@ -537,20 +537,24 @@ public void testIsIpBlockedWithOnlyBlockedIPs() { @Test public void testIsIpBlockedWithAllowedIPsAndBlockedIPs() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.1")); - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry allowedEntry1 = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.1")); + ReportingApi.ListsResponseEntry allowedEntry2 = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.2")); + ReportingApi.ListsResponseEntry allowedEntry3 = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.3")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("10.0.0.2")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), - List.of(allowedEntry), + List.of(allowedEntry1, allowedEntry2, allowedEntry3), null ); serviceConfiguration.updateBlockedLists(listsResponse); ServiceConfiguration.BlockedResult resultAllowed = serviceConfiguration.isIpBlocked("10.0.0.1"); - ServiceConfiguration.BlockedResult resultBlocked = serviceConfiguration.isIpBlocked("192.168.1.1"); + ServiceConfiguration.BlockedResult resultAllowed2 = serviceConfiguration.isIpBlocked("10.0.0.3"); + ServiceConfiguration.BlockedResult resultBlocked = serviceConfiguration.isIpBlocked("10.0.0.2"); ServiceConfiguration.BlockedResult resultNotAllowedLocal = serviceConfiguration.isIpBlocked("192.168.1.2"); assertFalse(resultAllowed.blocked()); + assertFalse(resultAllowed2.blocked()); assertTrue(resultBlocked.blocked()); assertFalse(resultNotAllowedLocal.blocked()); } From 9b651e0b9495492f514c651e3c698abc43064bef Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 11:29:40 +0200 Subject: [PATCH 27/45] Change the APIListsResponse to reflect the current API --- .../agent_api/background/cloud/api/ReportingApi.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java index b88ada8c..8293e6d6 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java @@ -30,14 +30,17 @@ public ReportingApi(int timeoutInSec) { public record APIListsResponse( List blockedIPAddresses, + List monitoredIPAddresses, List allowedIPAddresses, - List blockedUserAgents + String blockedUserAgents, + String monitoredUserAgents, + List userAgentDetails ) {} - public record ListsResponseEntry(boolean monitor, String key, String source, String description, List ips) { + public record ListsResponseEntry(String key, String source, String description, List ips) { } - public record BotBlocklist(boolean monitor, String key, String pattern) { + public record UserAgentDetail(String key, String pattern) { } /** * Fetch blocked lists using a separate API call, these can include : From fdd70913a221610b8f4a20e1c3e09dbbfcd05c3a Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 11:30:03 +0200 Subject: [PATCH 28/45] Change ParsedFirewallLists to save data and return results from new api --- .../ParsedFirewallLists.java | 73 +++++++++++++++---- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java index d0951da5..ae461a4a 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java @@ -12,7 +12,9 @@ public class ParsedFirewallLists { private final List blockedIps = new ArrayList<>(); private final List allowedIps = new ArrayList<>(); - private final List blockedUserAgents = new ArrayList<>(); + private final List uaDetails = new ArrayList<>(); + private Pattern blockedUserAgents = null; + private Pattern monitoredUserAgents = null; public ParsedFirewallLists() { @@ -41,20 +43,35 @@ public boolean matchesAllowedIps(String ip) { return false; } - public List matchBlockedUserAgents(String userAgent) { - List matches = new ArrayList<>(); - for (BlockedUAEntry entry : this.blockedUserAgents) { + public UABlockedResult matchBlockedUserAgents(String userAgent) { + boolean isBlocked = false; + if (blockedUserAgents != null) + isBlocked = blockedUserAgents.matcher(userAgent).find(); + + boolean isMonitored = false; + if (monitoredUserAgents != null) + isMonitored = monitoredUserAgents.matcher(userAgent).find(); + + if (!isMonitored && !isBlocked) + // only run the more detailed matches if it's an actual attack/monitored. + return new UABlockedResult(false, List.of()); + + List matchedUAKeys = new ArrayList<>(); + for (UADetailsEntry entry : this.uaDetails) { if (entry.pattern().matcher(userAgent).find()) { - matches.add(new Match(entry.key(), !entry.monitor(), null)); + matchedUAKeys.add(entry.key()); } } - return matches; + return new UABlockedResult(isBlocked, matchedUAKeys); } public void update(ReportingApi.APIListsResponse response) { updateBlockedIps(response.blockedIPAddresses()); + updateMonitoredIps(response.monitoredIPAddresses()); updateAllowedIps(response.allowedIPAddresses()); - updateBlockedUserAgents(response.blockedUserAgents()); + + updateBlockedAndMonitoredUAs(response.blockedUserAgents(), response.monitoredUserAgents()); + updateUADetails(response.userAgentDetails()); } public void updateBlockedIps(List blockedIpsList) { @@ -63,7 +80,16 @@ public void updateBlockedIps(List blockedIpsLis return; for (ReportingApi.ListsResponseEntry entry : blockedIpsList) { IPList ipList = createIPList(entry.ips()); - blockedIps.add(new IPEntry(entry.monitor(), entry.key(), entry.source(), entry.description(), ipList)); + blockedIps.add(new IPEntry(/* monitor */ false, entry.key(), entry.source(), entry.description(), ipList)); + } + } + + public void updateMonitoredIps(List monitoredIpsList) { + if (monitoredIpsList == null) + return; + for (ReportingApi.ListsResponseEntry entry : monitoredIpsList) { + IPList ipList = createIPList(entry.ips()); + blockedIps.add(new IPEntry(/* monitor */ true, entry.key(), entry.source(), entry.description(), ipList)); } } @@ -73,26 +99,43 @@ public void updateAllowedIps(List allowedIpsLis return; for (ReportingApi.ListsResponseEntry entry : allowedIpsList) { IPList ipList = createIPList(entry.ips()); - allowedIps.add(new IPEntry(entry.monitor(), entry.key(), entry.source(), entry.description(), ipList)); + boolean shouldMonitor = false; // we don't monitor allowed ips + allowedIps.add(new IPEntry(shouldMonitor, entry.key(), entry.source(), entry.description(), ipList)); } } - public void updateBlockedUserAgents(List blockedUserAgentsList) { - blockedUserAgents.clear(); - if (blockedUserAgentsList == null) + public void updateUADetails(List userAgentDetails) { + this.uaDetails.clear(); + if (userAgentDetails == null) return; - for (ReportingApi.BotBlocklist entry : blockedUserAgentsList) { + for (ReportingApi.UserAgentDetail entry : userAgentDetails) { Pattern pattern = Pattern.compile(entry.pattern(), Pattern.CASE_INSENSITIVE); - blockedUserAgents.add(new BlockedUAEntry(entry.monitor(), entry.key(), pattern)); + this.uaDetails.add(new UADetailsEntry(entry.key(), pattern)); } } + public void updateBlockedAndMonitoredUAs(String blockedUAs, String monitoredUAs) { + this.blockedUserAgents = null; + if (blockedUAs != null && !blockedUAs.isEmpty()) { + this.blockedUserAgents = Pattern.compile(blockedUAs, Pattern.CASE_INSENSITIVE); + } + + this.monitoredUserAgents = null; + if (monitoredUAs != null && !monitoredUAs.isEmpty()) { + this.monitoredUserAgents = Pattern.compile(monitoredUAs, Pattern.CASE_INSENSITIVE); + } + } + + public record Match(String key, boolean block, String description) { } + public record UABlockedResult(boolean block, List matchedKeys) { + } + private record IPEntry(boolean monitor, String key, String source, String description, IPList ips) { } - private record BlockedUAEntry(boolean monitor, String key, Pattern pattern) { + private record UADetailsEntry(String key, Pattern pattern) { } } From 7f4855c62cf34ae30ad5c5d222af76772a811842 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 11:54:26 +0200 Subject: [PATCH 29/45] Update statistics to not include total/blocked breakdown --- .../storage/statistics/Statistics.java | 42 +++++++++++++------ .../storage/statistics/StatisticsStore.java | 8 ++-- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java index 838a1e49..ea8ddaa8 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java @@ -7,8 +7,8 @@ public class Statistics { private final Map operations = new HashMap<>(); - private final FirewallListsRecord ipAddresses = new FirewallListsRecord(); - private final FirewallListsRecord userAgents = new FirewallListsRecord(); + private final Map ipAddressMatches = new HashMap<>(); + private final Map userAgentMatches = new HashMap<>(); private int totalHits; private int attacksDetected; private int attacksBlocked; @@ -75,13 +75,31 @@ public Map getOperations() { // firewall lists - public FirewallListsRecord getIpAddresses() { - return this.ipAddresses; + public Map getIpAddresses() { + return new HashMap<>(this.ipAddressMatches); } - public FirewallListsRecord getUserAgents() { - return this.userAgents; + + public Map getUserAgents() { + return new HashMap<>(this.userAgentMatches); + } + + public void addMatchToIpAddresses(String key) { + if (this.ipAddressMatches.containsKey(key)) { + int currentHitCount = this.ipAddressMatches.get(key); + this.ipAddressMatches.put(key, currentHitCount + 1); + } else { + this.ipAddressMatches.put(key, 0); + } } + public void addMatchToUserAgents(String key) { + if (this.userAgentMatches.containsKey(key)) { + int currentHitCount = this.userAgentMatches.get(key); + this.userAgentMatches.put(key, currentHitCount + 1); + } else { + this.userAgentMatches.put(key, 0); + } + } public StatsRecord getRecord() { long endedAt = UnixTimeMS.getUnixTimeMS(); @@ -93,8 +111,8 @@ public StatsRecord getRecord() { "blocked", attacksBlocked )), getOperations(), - new FirewallListsRecord(getIpAddresses()), - new FirewallListsRecord(getUserAgents()) + Map.of("breakdown", getIpAddresses()), + Map.of("breakdown", getUserAgents()) ); } @@ -104,8 +122,8 @@ public void clear() { this.attacksDetected = 0; this.startedAt = UnixTimeMS.getUnixTimeMS(); this.operations.clear(); - this.ipAddresses.clear(); - this.userAgents.clear(); + this.ipAddressMatches.clear(); + this.userAgentMatches.clear(); } // Stats records for sending out the heartbeat : @@ -114,7 +132,7 @@ public record StatsRequestsRecord(long total, long aborted, Map public record StatsRecord(long startedAt, long endedAt, StatsRequestsRecord requests, Map operations, - FirewallListsRecord ipAddresses, - FirewallListsRecord userAgents) { + Map> ipAddresses, + Map> userAgents) { } } diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java index 12177fcc..3d1fad42 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/StatisticsStore.java @@ -62,19 +62,19 @@ public static void registerCall(String sink, OperationKind kind) { } } - public static void incrementIpHits(String key, boolean blocked) { + public static void incrementIpHits(String key) { mutex.lock(); try { - stats.getIpAddresses().increment(key, blocked); + stats.addMatchToIpAddresses(key); } finally { mutex.unlock(); } } - public static void incrementUAHits(String key, boolean blocked) { + public static void incrementUAHits(String key) { mutex.lock(); try { - stats.getUserAgents().increment(key, blocked); + stats.addMatchToUserAgents(key); } finally { mutex.unlock(); } From 5eb3a8b28397519355cc5c1663347068d7b93553 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 11:54:54 +0200 Subject: [PATCH 30/45] Remove unused FirewallListsRecord --- .../statistics/FirewallListsRecord.java | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java deleted file mode 100644 index 7d049279..00000000 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/FirewallListsRecord.java +++ /dev/null @@ -1,42 +0,0 @@ -package dev.aikido.agent_api.storage.statistics; - -import java.util.HashMap; -import java.util.Map; - -public class FirewallListsRecord { - private final Map breakdown = new HashMap<>(); - - public FirewallListsRecord() { - } - - public FirewallListsRecord(FirewallListsRecord previous) { - this.breakdown.putAll(previous.breakdown); - } - - public void increment(String key, boolean blocked) { - breakdown.computeIfAbsent(key, k -> new BreakdownEntry(0, 0)); - - int newTotal = breakdown.get(key).total + 1; - int newBlocked = breakdown.get(key).blocked; - if (blocked) { - newBlocked += 1; - } - - breakdown.put(key, new BreakdownEntry(newTotal, newBlocked)); - } - - public BreakdownEntry get(String key) { - return this.breakdown.get(key); - } - - public void clear() { - this.breakdown.clear(); - } - - public int size() { - return this.breakdown.size(); - } - - public record BreakdownEntry(int total, int blocked) { - } -} From c3229838bbd00c7d9daa0ddea4f5192bf6cefa88 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 11:55:17 +0200 Subject: [PATCH 31/45] Change how the ServiceConfiguration increments work --- .../storage/ServiceConfiguration.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java index 2ca0c354..e01f2392 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java @@ -95,7 +95,7 @@ public BlockedResult isIpBlocked(String ip) { // Check for blocked ip addresses for (ParsedFirewallLists.Match match : firewallLists.matchBlockedIps(ip)) { - StatisticsStore.incrementIpHits(match.key(), match.block()); + StatisticsStore.incrementIpHits(match.key()); // when a blocking match is found, set blocked result if it hasn't been set already. if (match.block() && !blockedResult.blocked()) { blockedResult = new BlockedResult(true, match.description()); @@ -113,19 +113,11 @@ public void updateBlockedLists(ReportingApi.APIListsResponse res) { * Check if a given User-Agent is blocked or not : */ public boolean isBlockedUserAgent(String userAgent) { - boolean blocked = false; - for (ParsedFirewallLists.Match match : this.firewallLists.matchBlockedUserAgents(userAgent)) { - StatisticsStore.incrementUAHits(match.key(), match.block()); - if (match.block()) { - blocked = true; - } + ParsedFirewallLists.UABlockedResult result = this.firewallLists.matchBlockedUserAgents(userAgent); + for (String matchedKey : result.matchedKeys()) { + StatisticsStore.incrementUAHits(matchedKey); } - - return blocked; - } - - // IP restrictions (e.g. Geo-IP Restrictions) : - public record IPListEntry(IPList ipList, String description, boolean monitor, String key) { + return result.block(); } public record BlockedResult(boolean blocked, String description) { From 4496003df25b5aaa04013a3d82543c29d9bf7733 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 12:11:53 +0200 Subject: [PATCH 32/45] Update mock aikido core api spec --- end2end/server/mock_aikido_core.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/end2end/server/mock_aikido_core.py b/end2end/server/mock_aikido_core.py index ef8475ec..2f3a03b6 100644 --- a/end2end/server/mock_aikido_core.py +++ b/end2end/server/mock_aikido_core.py @@ -50,33 +50,32 @@ "success": True, "blockedIPAddresses": [ { - "monitor": False, "key": "geo-1", "source": "geoip", "description": "geo restrictions", "ips": ["1.2.3.4"] }, + ], + "monitoredIPAddresses": [ { - "monitor": True, "key": "geo-2", "source": "geoip", "description": "should not be blocked", "ips": ["5.6.7.8"] } ], - "blockedUserAgents": [ + "blockedUserAgents": "AI2Bot|Bytespider", + "monitoredUserAgents": "ClaudeUser", + "userAgentDetails": [ { - "monitor": False, "key": "ai-agents", "pattern": "AI2Bot" }, { - "monitor": False, "key": "crawlers", "pattern": "Bytespider" }, { - "monitor": True, "key": "crawlers-monitor", "pattern": "ClaudeUser" } From 72c6d099d83866bba91a85d57ed4b63db49ae08c Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 13:07:44 +0200 Subject: [PATCH 33/45] Fix WebRequestCollectorTest test cases --- .../collectors/WebRequestCollectorTest.java | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java index b262bfde..855c52f0 100644 --- a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java +++ b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java @@ -84,8 +84,8 @@ void testReport_no_endpoints() { @Test void testReport_ipBlockedTwice() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.1")) - ), null, List.of()); + new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("192.168.1.1")) + ), List.of(), List.of(), null, null, List.of()); setEmptyConfigWithEndpointList(List.of( new Endpoint( "GET", "/api/resource", 100, 100, @@ -110,8 +110,8 @@ void testReport_ipBlockedTwice() { @Test void testReport_ipBlockedUsingLists() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) - ), null, List.of()); + new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) + ), List.of(), List.of(), null, null, List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); @@ -124,8 +124,8 @@ void testReport_ipBlockedUsingLists() { @Test void testReport_publicIpBlockedUsingLists() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("bullshit.ip", "2.2.2.0/24")) - ), null, List.of()); + new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("bullshit.ip", "2.2.2.0/24")) + ), List.of(), List.of(), null, null, List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); contextObject.setIp("2.2.2.7"); @@ -139,9 +139,9 @@ void testReport_publicIpBlockedUsingLists() { @Test void testReport_ipNotAllowedUsingLists() { - ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(null, List.of( - new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.2.1")) - ), List.of()); + ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of(), List.of(), List.of( + new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("192.168.2.1")) + ), null, null, List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); @@ -158,9 +158,9 @@ void testReport_ipNotAllowedUsingLists() { @Test void testReport_ipInAllowlist() { - ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(null, List.of( - new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.1", "10.0.0.0/24")) - ), List.of()); + ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of(), List.of(), List.of( + new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("192.168.1.1", "10.0.0.0/24")) + ), null, null, List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); @@ -172,8 +172,9 @@ void testReport_ipInAllowlist() { @Test void testReport_ipNotBlockedUsingListsNorUserAgent() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) - ), null, List.of(new ReportingApi.BotBlocklist(false, "block", "Unrelated|random"))); + new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) + ), List.of(), List.of(), + "Unrelated|random", "test", List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); @@ -185,8 +186,9 @@ void testReport_ipNotBlockedUsingListsNorUserAgent() { @Test void testReport_userAgentBlocked() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) - ), null, List.of(new ReportingApi.BotBlocklist(false, "block", "AI2Bot|hacker"))); + new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) + ), List.of(), List.of(), + "AI2Bot|hacker", "", List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); WebRequestCollector.Res response = WebRequestCollector.report(contextObject); @@ -199,8 +201,8 @@ void testReport_userAgentBlocked() { @Test void testReport_userAgentBlocked_Ip_Bypassed() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) - ), null, List.of(new ReportingApi.BotBlocklist(false, "block", "AI2Bot|hacker"))); + new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) + ), List.of(), List.of(), "AI2Bot|hacker", "", List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); List bypassedIps = List.of("192.168.1.1"); @@ -218,8 +220,8 @@ void testReport_userAgentBlocked_Ip_Bypassed() { @Test void testReport_ipBlockedUsingLists_Ip_Bypassed() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( - new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) - ), null, List.of()); + new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) + ), List.of(), List.of(), null, null, List.of()); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); List bypassedIps = List.of("192.168.1.1"); @@ -236,9 +238,9 @@ void testReport_ipBlockedUsingLists_Ip_Bypassed() { @Test void testReport_ipNotAllowedUsingLists_Ip_Bypassed() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse( - null, - List.of(new ReportingApi.ListsResponseEntry(false, "key", "geoip", "geoip restrictions", List.of("1.2.3.4"))), - List.of() + List.of(), List.of(), + List.of(new ReportingApi.ListsResponseEntry("key", "geoip", "geoip restrictions", List.of("1.2.3.4"))), + null, null, List.of() ); ServiceConfigStore.updateFromAPIListsResponse(blockedListsRes); From 3532124d16b024c355d4bb2edb681ca2ee1b9f95 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 13:07:56 +0200 Subject: [PATCH 34/45] Fix EmptyAPIResponses not having correct fields --- agent_api/src/test/java/utils/EmptyAPIResponses.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent_api/src/test/java/utils/EmptyAPIResponses.java b/agent_api/src/test/java/utils/EmptyAPIResponses.java index ea259b28..091631a8 100644 --- a/agent_api/src/test/java/utils/EmptyAPIResponses.java +++ b/agent_api/src/test/java/utils/EmptyAPIResponses.java @@ -15,7 +15,7 @@ public class EmptyAPIResponses { true, "", UnixTimeMS.getUnixTimeMS(), List.of(), List.of(), List.of(), true, false ); public final static ReportingApi.APIListsResponse emptyAPIListsResponse = new ReportingApi.APIListsResponse( - List.of(), List.of(), List.of() + List.of(), List.of(), List.of(), null, null, List.of() ); public static void setEmptyConfigWithEndpointList(List endpoints) { ServiceConfigStore.updateFromAPIResponse(new APIResponse( From 2450ba8a3eb41dfb4ae2709b80ae6afbf14708d8 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 13:10:02 +0200 Subject: [PATCH 35/45] Statistics, baseline on new element should be 1 for hits --- .../dev/aikido/agent_api/storage/statistics/Statistics.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java index ea8ddaa8..ce53f76a 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/statistics/Statistics.java @@ -88,7 +88,7 @@ public void addMatchToIpAddresses(String key) { int currentHitCount = this.ipAddressMatches.get(key); this.ipAddressMatches.put(key, currentHitCount + 1); } else { - this.ipAddressMatches.put(key, 0); + this.ipAddressMatches.put(key, 1); } } @@ -97,7 +97,7 @@ public void addMatchToUserAgents(String key) { int currentHitCount = this.userAgentMatches.get(key); this.userAgentMatches.put(key, currentHitCount + 1); } else { - this.userAgentMatches.put(key, 0); + this.userAgentMatches.put(key, 1); } } From 4dd53e8cf35700400206eabaca7cc0c8378a1ee5 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 13:10:10 +0200 Subject: [PATCH 36/45] Update StatisticsStore test cases --- .../java/storage/StatisticsStoreTest.java | 85 ++++++++----------- 1 file changed, 36 insertions(+), 49 deletions(-) diff --git a/agent_api/src/test/java/storage/StatisticsStoreTest.java b/agent_api/src/test/java/storage/StatisticsStoreTest.java index b37d95f9..f3b86a93 100644 --- a/agent_api/src/test/java/storage/StatisticsStoreTest.java +++ b/agent_api/src/test/java/storage/StatisticsStoreTest.java @@ -55,29 +55,26 @@ public void testRegisterCall() { @Test public void testIncrementIpHits() { String ip = "192.168.1.1"; - StatisticsStore.incrementIpHits(ip, false); + StatisticsStore.incrementIpHits(ip); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); assertNotNull(record); - assertNotNull(record.ipAddresses().get(ip)); - assertEquals(1, record.ipAddresses().get(ip).total()); + assertEquals(1, record.ipAddresses().get("breakdown").get(ip)); } @Test public void testIncrementUAHits() { String userAgent = "Mozilla/5.0"; - StatisticsStore.incrementUAHits(userAgent, true); + StatisticsStore.incrementUAHits(userAgent); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); assertNotNull(record); - assertNotNull(record.userAgents().get(userAgent)); - assertEquals(1, record.userAgents().get(userAgent).total()); - assertEquals(1, record.userAgents().get(userAgent).blocked()); + assertEquals(1, record.userAgents().get("breakdown").get(userAgent)); } @Test public void testClear() { StatisticsStore.incrementHits(); - StatisticsStore.incrementIpHits("ip", false); - StatisticsStore.incrementUAHits("ip", false); + StatisticsStore.incrementIpHits("ip"); + StatisticsStore.incrementUAHits("ip"); String operation = "testOperation"; StatisticsStore.registerCall(operation, OperationKind.FS_OP); @@ -87,8 +84,8 @@ public void testClear() { assertNotNull(record); assertEquals(0, record.requests().total()); assertEquals(0, record.operations().size()); - assertEquals(0, record.ipAddresses().size()); - assertEquals(0, record.userAgents().size()); + assertEquals(0, record.ipAddresses().get("breakdown").size()); + assertEquals(0, record.userAgents().get("breakdown").size()); } @@ -135,32 +132,28 @@ public void testRegisterMultipleCalls() { @Test public void testIncrementIpHitsMultipleTimes() { String ip = "192.168.1.1"; - StatisticsStore.incrementIpHits(ip, false); - StatisticsStore.incrementIpHits(ip, true); + StatisticsStore.incrementIpHits(ip); + StatisticsStore.incrementIpHits(ip); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); assertNotNull(record); - assertNotNull(record.ipAddresses().get(ip)); - assertEquals(2, record.ipAddresses().get(ip).total()); - assertEquals(1, record.ipAddresses().get(ip).blocked()); + assertEquals(2, record.ipAddresses().get("breakdown").get(ip)); } @Test public void testIncrementUAHitsMultipleTimes() { String userAgent = "Mozilla/5.0"; - StatisticsStore.incrementUAHits(userAgent, true); - StatisticsStore.incrementUAHits(userAgent, false); + StatisticsStore.incrementUAHits(userAgent); + StatisticsStore.incrementUAHits(userAgent); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); assertNotNull(record); - assertNotNull(record.userAgents().get(userAgent)); - assertEquals(2, record.userAgents().get(userAgent).total()); - assertEquals(1, record.userAgents().get(userAgent).blocked()); + assertEquals(2, record.userAgents().get("breakdown").get(userAgent)); } @Test public void testClearAfterMultipleIncrements() { StatisticsStore.incrementHits(); - StatisticsStore.incrementIpHits("192.168.1.1", false); - StatisticsStore.incrementUAHits("Mozilla/5.0", true); + StatisticsStore.incrementIpHits("192.168.1.1"); + StatisticsStore.incrementUAHits("Mozilla/5.0"); String operation = "testOperation"; StatisticsStore.registerCall(operation, OperationKind.FS_OP); @@ -170,8 +163,8 @@ public void testClearAfterMultipleIncrements() { assertNotNull(record); assertEquals(0, record.requests().total()); assertEquals(0, record.operations().size()); - assertEquals(0, record.ipAddresses().size()); - assertEquals(0, record.userAgents().size()); + assertEquals(0, record.ipAddresses().get("breakdown").size()); + assertEquals(0, record.userAgents().get("breakdown").size()); } @Test @@ -196,27 +189,23 @@ public void testConcurrentAccess() throws InterruptedException { @Test public void testIncrementIpHitsWithDifferentIPs() { - StatisticsStore.incrementIpHits("192.168.1.1", false); - StatisticsStore.incrementIpHits("192.168.1.2", true); + StatisticsStore.incrementIpHits("192.168.1.1"); + StatisticsStore.incrementIpHits("192.168.1.2"); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); assertNotNull(record); - assertEquals(1, record.ipAddresses().get("192.168.1.1").total()); - assertEquals(0, record.ipAddresses().get("192.168.1.1").blocked()); - assertEquals(1, record.ipAddresses().get("192.168.1.2").total()); - assertEquals(1, record.ipAddresses().get("192.168.1.2").blocked()); + assertEquals(1, record.ipAddresses().get("breakdown").get("192.168.1.1")); + assertEquals(1, record.ipAddresses().get("breakdown").get("192.168.1.2")); } @Test public void testIncrementUAHitsWithDifferentUserAgents() { - StatisticsStore.incrementUAHits("Mozilla/5.0", true); - StatisticsStore.incrementUAHits("Chrome/91.0", false); + StatisticsStore.incrementUAHits("Mozilla/5.0"); + StatisticsStore.incrementUAHits("Chrome/91.0"); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); assertNotNull(record); - assertEquals(1, record.userAgents().get("Mozilla/5.0").total()); - assertEquals(1, record.userAgents().get("Mozilla/5.0").blocked()); - assertEquals(1, record.userAgents().get("Chrome/91.0").total()); - assertEquals(0, record.userAgents().get("Chrome/91.0").blocked()); + assertEquals(1, record.userAgents().get("breakdown").get("Mozilla/5.0")); + assertEquals(1, record.userAgents().get("breakdown").get("Chrome/91.0")); } @Test @@ -224,8 +213,8 @@ public void testClearAfterIncrementingDifferentMetrics() { StatisticsStore.incrementHits(); StatisticsStore.incrementAttacksDetected("operation1"); StatisticsStore.incrementAttacksBlocked("operation1"); - StatisticsStore.incrementIpHits("192.168.1.1", false); - StatisticsStore.incrementUAHits("Mozilla/5.0", true); + StatisticsStore.incrementIpHits("192.168.1.1"); + StatisticsStore.incrementUAHits("Mozilla/5.0"); StatisticsStore.clear(); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); @@ -234,8 +223,8 @@ public void testClearAfterIncrementingDifferentMetrics() { assertEquals(0, record.requests().total()); assertEquals(0, record.requests().attacksDetected().get("total")); assertEquals(0, record.requests().attacksDetected().get("blocked")); - assertEquals(0, record.ipAddresses().size()); - assertEquals(0, record.userAgents().size()); + assertEquals(0, record.ipAddresses().get("breakdown").size()); + assertEquals(0, record.userAgents().get("breakdown").size()); } @Test @@ -275,19 +264,17 @@ public void testIncrementMultipleMetricsSimultaneously() { StatisticsStore.incrementHits(); StatisticsStore.incrementAttacksDetected(operation); StatisticsStore.incrementAttacksBlocked(operation); - StatisticsStore.incrementIpHits(ip, false); - StatisticsStore.incrementUAHits(userAgent, true); + StatisticsStore.incrementIpHits(ip); + StatisticsStore.incrementUAHits(userAgent); Statistics.StatsRecord record = StatisticsStore.getStatsRecord(); assertNotNull(record); assertEquals(1, record.requests().total()); assertEquals(1, record.requests().attacksDetected().get("total")); assertEquals(1, record.requests().attacksDetected().get("blocked")); - assertNotNull(record.ipAddresses().get(ip)); - assertEquals(1, record.ipAddresses().get(ip).total()); - assertEquals(0, record.ipAddresses().get(ip).blocked()); - assertNotNull(record.userAgents().get(userAgent)); - assertEquals(1, record.userAgents().get(userAgent).total()); - assertEquals(1, record.userAgents().get(userAgent).blocked()); + assertNotNull(record.ipAddresses().get("breakdown").get(ip)); + assertEquals(1, record.ipAddresses().get("breakdown").get(ip)); + assertNotNull(record.userAgents().get("breakdown").get(userAgent)); + assertEquals(1, record.userAgents().get("breakdown").get(userAgent)); } } From d460e689cb741f55dc2847c1fc6f4bd28c42e784 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 13:11:45 +0200 Subject: [PATCH 37/45] Fix ServiceConfigurationTest cases --- .../storage/ServiceConfigurationTest.java | 178 +++++++++--------- 1 file changed, 84 insertions(+), 94 deletions(-) diff --git a/agent_api/src/test/java/storage/ServiceConfigurationTest.java b/agent_api/src/test/java/storage/ServiceConfigurationTest.java index 551ad157..d9f685d8 100644 --- a/agent_api/src/test/java/storage/ServiceConfigurationTest.java +++ b/agent_api/src/test/java/storage/ServiceConfigurationTest.java @@ -71,11 +71,10 @@ public void testUpdateConfigWithUnsuccessfulResponse() { @Test public void testIsIpBlocked() { - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "my-key", "source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("my-key", "source", "blocked", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(blockedEntry), - Collections.emptyList(), - Collections.emptyList() + List.of(blockedEntry), List.of(), List.of(), + "", "", List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -83,19 +82,17 @@ public void testIsIpBlocked() { ServiceConfiguration.BlockedResult result = serviceConfiguration.isIpBlocked("192.168.1.1"); assertTrue(result.blocked()); assertEquals("blocked", result.description()); - assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("my-key").total()); - assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("my-key").blocked()); + assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("breakdown").get("my-key")); } @Test public void testIsIpBlockedOrMonitored() { - var blockedEntry = new ReportingApi.ListsResponseEntry(false, "blocked-key", "source", "blocked", List.of("192.168.1.1")); - var monitoredEntry = new ReportingApi.ListsResponseEntry(true, "monitored-key", "source", "blocked", List.of("192.168.2.*")); + var blockedEntry = new ReportingApi.ListsResponseEntry("blocked-key", "source", "blocked", List.of("192.168.1.1")); + var monitoredEntry = new ReportingApi.ListsResponseEntry("monitored-key", "source", "blocked", List.of("192.168.2.*")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(blockedEntry, monitoredEntry), - Collections.emptyList(), - Collections.emptyList() + List.of(blockedEntry), List.of(monitoredEntry), + List.of(), null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -103,8 +100,7 @@ public void testIsIpBlockedOrMonitored() { ServiceConfiguration.BlockedResult result = serviceConfiguration.isIpBlocked("192.168.1.1"); assertTrue(result.blocked()); assertEquals("blocked", result.description()); - assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("blocked-key").total()); - assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("blocked-key").blocked()); + assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("breakdown").get("blocked-key")); assertNull(StatisticsStore.getStatsRecord().ipAddresses().get("monitored-key")); // now test with multiple ip that are only monitored @@ -112,19 +108,17 @@ public void testIsIpBlockedOrMonitored() { assertFalse(serviceConfiguration.isIpBlocked("192.168.2.128").blocked()); assertFalse(serviceConfiguration.isIpBlocked("192.168.3.20").blocked()); assertFalse(serviceConfiguration.isIpBlocked("192.168.2.5").blocked()); - assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("blocked-key").total()); - assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("blocked-key").blocked()); - assertEquals(3, StatisticsStore.getStatsRecord().ipAddresses().get("monitored-key").total()); - assertEquals(0, StatisticsStore.getStatsRecord().ipAddresses().get("monitored-key").blocked()); + assertEquals(1, StatisticsStore.getStatsRecord().ipAddresses().get("breakdown").get("blocked-key")); + assertEquals(3, StatisticsStore.getStatsRecord().ipAddresses().get("breakdown").get("monitored-key")); } @Test public void testIsIpAllowed() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry("key", "source", "allowed", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - Collections.emptyList(), + List.of(), List.of(), List.of(allowedEntry), - Collections.emptyList() + null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -136,30 +130,31 @@ public void testIsIpAllowed() { @Test public void testIsBlockedUserAgent() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - Collections.emptyList(), - Collections.emptyList(), - List.of(new ReportingApi.BotBlocklist(false, "blocked-bots", "blocked-agent")) + List.of(), List.of(), List.of(), + "blocked-agent", null, List.of( + new ReportingApi.UserAgentDetail("blocked-bots", "blocked-agent") + ) ); serviceConfiguration.updateBlockedLists(listsResponse); assertTrue(serviceConfiguration.isBlockedUserAgent("blocked-agent")); assertFalse(serviceConfiguration.isBlockedUserAgent("allowed-agent")); - assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("blocked-bots").total()); - assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("blocked-bots").blocked()); + assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("breakdown").get("blocked-bots")); } @Test public void testMonitoredAndBlockedUserAgents() { - var monitoring1 = new ReportingApi.BotBlocklist(true, "monitoring1", "blocked-agent"); - var blocking1 = new ReportingApi.BotBlocklist(false, "blocking1", "blocked-agent"); - var monitoring2 = new ReportingApi.BotBlocklist(true, "monitoring2", "allowed*"); - var blocking2 = new ReportingApi.BotBlocklist(false, "blocking2", "blocked*"); - ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - Collections.emptyList(), - Collections.emptyList(), - List.of(monitoring1, blocking1, monitoring2, blocking2) + List.of(), List.of(), List.of(), + "blocked-agent|blocked*", + "blocked-agent|allowed*", + List.of( + new ReportingApi.UserAgentDetail("monitoring1", "blocked-agent"), + new ReportingApi.UserAgentDetail("blocking1", "blocked-agent"), + new ReportingApi.UserAgentDetail("monitoring2", "allowed*"), + new ReportingApi.UserAgentDetail("blocking2", "blocked*") + ) ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -169,14 +164,11 @@ public void testMonitoredAndBlockedUserAgents() { assertFalse(serviceConfiguration.isBlockedUserAgent("allowed-agent")); assertFalse(serviceConfiguration.isBlockedUserAgent("allowed-a2")); assertFalse(serviceConfiguration.isBlockedUserAgent("allowed-a3")); - assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("blocking1").total()); - assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("blocking1").blocked()); - assertEquals(2, StatisticsStore.getStatsRecord().userAgents().get("blocking2").total()); - assertEquals(2, StatisticsStore.getStatsRecord().userAgents().get("blocking2").blocked()); - assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("monitoring1").total()); - assertEquals(0, StatisticsStore.getStatsRecord().userAgents().get("monitoring1").blocked()); - assertEquals(3, StatisticsStore.getStatsRecord().userAgents().get("monitoring2").total()); - assertEquals(0, StatisticsStore.getStatsRecord().userAgents().get("monitoring2").blocked()); + var uas = StatisticsStore.getStatsRecord().userAgents().get("breakdown"); + assertEquals(1, uas.get("blocking1")); + assertEquals(2, uas.get("blocking2")); + assertEquals(1, uas.get("monitoring1")); + assertEquals(3, uas.get("monitoring2")); } @Test @@ -198,9 +190,8 @@ public void testIsIpBlockedWithPrivateIp() { public void testIsIpBlockedWithEmptyAllowedList() { // If allowed IPs list is empty, any IP should not be blocked by allowed list logic ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList() + List.of(), List.of(), List.of(), + null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -211,12 +202,11 @@ public void testIsIpBlockedWithEmptyAllowedList() { @Test public void testIsIpBlockedWithMultipleBlockedEntries() { - ReportingApi.ListsResponseEntry blockedEntry1 = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); - ReportingApi.ListsResponseEntry blockedEntry2 = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.2")); + ReportingApi.ListsResponseEntry blockedEntry1 = new ReportingApi.ListsResponseEntry("key", "source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry blockedEntry2 = new ReportingApi.ListsResponseEntry("key", "source", "blocked", List.of("192.168.1.2")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(blockedEntry1, blockedEntry2), - Collections.emptyList(), - Collections.emptyList() + List.of(blockedEntry1, blockedEntry2), List.of(), List.of(), + null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -229,12 +219,11 @@ public void testIsIpBlockedWithMultipleBlockedEntries() { @Test public void testIsIpBlockedWithMixedAllowedAndBlockedEntries() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("192.168.1.1")); - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.2")); + ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry("key", "source", "allowed", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("key", "source", "blocked", List.of("192.168.1.2")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(blockedEntry), - List.of(allowedEntry), - Collections.emptyList() + List.of(blockedEntry), List.of(), List.of(allowedEntry), + null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -248,9 +237,10 @@ public void testIsIpBlockedWithMixedAllowedAndBlockedEntries() { @Test public void testIsBlockedUserAgentWithNullRegex() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList() + List.of(), List.of(), List.of(), + null, null, List.of( + new ReportingApi.UserAgentDetail("key", "any-agent") + ) ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -261,9 +251,9 @@ public void testIsBlockedUserAgentWithNullRegex() { @Test public void testIsBlockedUserAgentWithCaseInsensitiveMatch() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - Collections.emptyList(), - Collections.emptyList(), - List.of(new ReportingApi.BotBlocklist(false, "blocked-bots", "blocked-agent")) + List.of(), List.of(), List.of(), + "blocked-agent|blockwhatever", + "", List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -406,11 +396,10 @@ public void testGetEndpointsWithEmptyList() { @Test public void testIsIpBlockedWithSubnet() { - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.0/24")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("key", "source", "blocked", List.of("192.168.1.0/24")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), - Collections.emptyList(), - Collections.emptyList() + List.of(), List.of(), null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -422,12 +411,11 @@ public void testIsIpBlockedWithSubnet() { @Test public void testIsIpBlockedWithMixedSubnetAndSingleIP() { - ReportingApi.ListsResponseEntry blockedEntry1 = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.0/24")); - ReportingApi.ListsResponseEntry blockedEntry2 = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("10.0.0.1")); + ReportingApi.ListsResponseEntry blockedEntry1 = new ReportingApi.ListsResponseEntry("key", "source", "blocked", List.of("192.168.1.0/24")); + ReportingApi.ListsResponseEntry blockedEntry2 = new ReportingApi.ListsResponseEntry("key", "source", "blocked", List.of("10.0.0.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry1, blockedEntry2), - Collections.emptyList(), - Collections.emptyList() + List.of(), List.of(), null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -441,9 +429,8 @@ public void testIsIpBlockedWithMixedSubnetAndSingleIP() { @Test public void testIsIpBlockedWithEmptyBlockedList() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(), - List.of(), - Collections.emptyList() + List.of(), List.of(), List.of(), + "", "", List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -455,9 +442,10 @@ public void testIsIpBlockedWithEmptyBlockedList() { @Test public void testIsBlockedUserAgentWithEmptyRegex() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(), - List.of(), - List.of() + List.of(), List.of(), List.of(), + "", "", List.of( + new ReportingApi.UserAgentDetail("key", "any-agent") + ) ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -468,29 +456,32 @@ public void testIsBlockedUserAgentWithEmptyRegex() { @Test public void testIsBlockedUserAgentWithComplexRegex() { ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(), - List.of(), + List.of(), List.of(), List.of(), + "blocked.*agent|another.*pattern", "", List.of( - new ReportingApi.BotBlocklist(false, "block1", "blocked.*agent"), - new ReportingApi.BotBlocklist(false, "block2", "another.*pattern") + new ReportingApi.UserAgentDetail("block1", "blocked.*agent"), + new ReportingApi.UserAgentDetail("block2", "another.*pattern") ) ); serviceConfiguration.updateBlockedLists(listsResponse); assertTrue(serviceConfiguration.isBlockedUserAgent("blocked-agent")); + assertTrue(serviceConfiguration.isBlockedUserAgent("blocked-agent2")); assertTrue(serviceConfiguration.isBlockedUserAgent("another-pattern")); assertFalse(serviceConfiguration.isBlockedUserAgent("allowed-agent")); + assertEquals(2, StatisticsStore.getStatsRecord().userAgents().get("breakdown").get("block1")); + assertEquals(1, StatisticsStore.getStatsRecord().userAgents().get("breakdown").get("block2")); + assertNull(StatisticsStore.getStatsRecord().userAgents().get("breakdown").get("unknownblock")); } @Test public void testIsIpBlockedWithAllowedAndBlockedIPs() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.1")); - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry("key", "source", "allowed", List.of("10.0.0.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("key", "source", "blocked", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(blockedEntry), - List.of(allowedEntry), - null + List.of(blockedEntry), List.of(), List.of(allowedEntry), + null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -503,11 +494,11 @@ public void testIsIpBlockedWithAllowedAndBlockedIPs() { @Test public void testIsIpBlockedWithOnlyAllowedIPs() { - ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.1")); + ReportingApi.ListsResponseEntry allowedEntry = new ReportingApi.ListsResponseEntry("key", "source", "allowed", List.of("10.0.0.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(), + List.of(), List.of(), List.of(allowedEntry), - null + null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -520,11 +511,10 @@ public void testIsIpBlockedWithOnlyAllowedIPs() { @Test public void testIsIpBlockedWithOnlyBlockedIPs() { - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("192.168.1.1")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("key", "source", "blocked", List.of("192.168.1.1")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( List.of(blockedEntry), - List.of(), - null + List.of(), List.of(), null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); @@ -537,14 +527,14 @@ public void testIsIpBlockedWithOnlyBlockedIPs() { @Test public void testIsIpBlockedWithAllowedIPsAndBlockedIPs() { - ReportingApi.ListsResponseEntry allowedEntry1 = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.1")); - ReportingApi.ListsResponseEntry allowedEntry2 = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.2")); - ReportingApi.ListsResponseEntry allowedEntry3 = new ReportingApi.ListsResponseEntry(false, "key", "source", "allowed", List.of("10.0.0.3")); - ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry(false, "key", "source", "blocked", List.of("10.0.0.2")); + ReportingApi.ListsResponseEntry allowedEntry1 = new ReportingApi.ListsResponseEntry("key", "source", "allowed", List.of("10.0.0.1")); + ReportingApi.ListsResponseEntry allowedEntry2 = new ReportingApi.ListsResponseEntry("key", "source", "allowed", List.of("10.0.0.2")); + ReportingApi.ListsResponseEntry allowedEntry3 = new ReportingApi.ListsResponseEntry("key", "source", "allowed", List.of("10.0.0.3")); + ReportingApi.ListsResponseEntry blockedEntry = new ReportingApi.ListsResponseEntry("key", "source", "blocked", List.of("10.0.0.2")); ReportingApi.APIListsResponse listsResponse = new ReportingApi.APIListsResponse( - List.of(blockedEntry), + List.of(blockedEntry), List.of(), List.of(allowedEntry1, allowedEntry2, allowedEntry3), - null + null, null, List.of() ); serviceConfiguration.updateBlockedLists(listsResponse); From 1b951ae2a2af36962dc73d4c13931db94989a318 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 13:12:12 +0200 Subject: [PATCH 38/45] Remove the setting of a now useless header --- .../aikido/agent_api/background/cloud/api/ReportingApiHTTP.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java index d822b2bb..e7391769 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApiHTTP.java @@ -81,8 +81,6 @@ public Optional fetchBlockedLists() { // Set the Accept-Encoding header to gzip connection.setRequestProperty("Accept-Encoding", "gzip"); connection.setRequestProperty("Authorization", token.get()); - // Indicates to the server that this agent supports the new format with monitoring - connection.setRequestProperty("x-supports-monitoring", "true"); if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { return Optional.empty(); From 8e606245c2cd1edc67f10bf4b3945390168d32d6 Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 13:24:48 +0200 Subject: [PATCH 39/45] Update test cases for ReportingAPITest --- .../background/cloud/ReportingAPITest.java | 60 +++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/agent_api/src/test/java/background/cloud/ReportingAPITest.java b/agent_api/src/test/java/background/cloud/ReportingAPITest.java index cae078ed..b088501d 100644 --- a/agent_api/src/test/java/background/cloud/ReportingAPITest.java +++ b/agent_api/src/test/java/background/cloud/ReportingAPITest.java @@ -3,6 +3,7 @@ import dev.aikido.agent_api.background.cloud.api.APIResponse; import dev.aikido.agent_api.background.cloud.api.ReportingApiHTTP; import dev.aikido.agent_api.helpers.env.Token; +import dev.aikido.agent_api.storage.service_configuration.ParsedFirewallLists; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junitpioneer.jupiter.SetEnvironmentVariable; @@ -70,13 +71,64 @@ public void testListsResponseWithWrongEndpoint(StdOut out) { public void testListsResponse() { Optional res = api.fetchBlockedLists(); assertTrue(res.isPresent()); - assertEquals(2, res.get().blockedIPAddresses().size()); + assertEquals(1, res.get().blockedIPAddresses().size()); assertEquals("geoip", res.get().blockedIPAddresses().get(0).source()); assertEquals("geo restrictions", res.get().blockedIPAddresses().get(0).description()); assertEquals("1.2.3.4", res.get().blockedIPAddresses().get(0).ips().get(0)); - assertEquals(3, res.get().blockedUserAgents().size()); - assertEquals("AI2Bot", res.get().blockedUserAgents().get(0).pattern()); - assertEquals("Bytespider", res.get().blockedUserAgents().get(1).pattern()); + assertEquals(1, res.get().monitoredIPAddresses().size()); + assertEquals("geoip", res.get().monitoredIPAddresses().get(0).source()); + assertEquals("should not be blocked", res.get().monitoredIPAddresses().get(0).description()); + assertEquals("5.6.7.8", res.get().monitoredIPAddresses().get(0).ips().get(0)); + + assertNull(res.get().allowedIPAddresses()); + + assertEquals(3, res.get().userAgentDetails().size()); + assertEquals("ai-agents", res.get().userAgentDetails().get(0).key()); + assertEquals("AI2Bot", res.get().userAgentDetails().get(0).pattern()); + + assertEquals("crawlers", res.get().userAgentDetails().get(1).key()); + assertEquals("Bytespider", res.get().userAgentDetails().get(1).pattern()); + + assertEquals("crawlers-monitor", res.get().userAgentDetails().get(2).key()); + assertEquals("ClaudeUser", res.get().userAgentDetails().get(2).pattern()); + + assertEquals("AI2Bot|Bytespider", res.get().blockedUserAgents()); + assertEquals("ClaudeUser", res.get().monitoredUserAgents()); + } + + @Test + public void testParsedListsResponse() { + Optional res = api.fetchBlockedLists(); + assertTrue(res.isPresent()); + + ParsedFirewallLists parsed = new ParsedFirewallLists(); + parsed.update(res.get()); + + var matches = parsed.matchBlockedIps("1.2.3.4"); + assertEquals(1, matches.size()); + assertEquals("geo-1", matches.get(0).key()); + assertEquals("geo restrictions", matches.get(0).description()); + assertTrue(matches.get(0).block()); + + matches = parsed.matchBlockedIps("5.6.7.8"); + assertEquals(1, matches.size()); + assertEquals("geo-2", matches.get(0).key()); + assertEquals("should not be blocked", matches.get(0).description()); + assertFalse(matches.get(0).block()); + + matches = parsed.matchBlockedIps("2.3.4.5"); + assertEquals(0, matches.size()); + + var matchUa = parsed.matchBlockedUserAgents("ClaudeUser/2.0 Mozilla"); + assertFalse(matchUa.block()); + assertEquals(1, matchUa.matchedKeys().size()); + assertEquals("crawlers-monitor", matchUa.matchedKeys().get(0)); + + matchUa = parsed.matchBlockedUserAgents("ClaudeUser/2.0 Mozilla, AI2Bot"); + assertTrue(matchUa.block()); + assertEquals(2, matchUa.matchedKeys().size()); + assertEquals("ai-agents", matchUa.matchedKeys().get(0)); + assertEquals("crawlers-monitor", matchUa.matchedKeys().get(1)); } } From 5775bd9aa521dc042b53ff9188ca45639ce4cbae Mon Sep 17 00:00:00 2001 From: BitterPanda Date: Fri, 9 May 2025 13:29:20 +0200 Subject: [PATCH 40/45] empty commit, run build jobs From 15a6700710417ad739264e15b011382d3cfc63b1 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Tue, 13 May 2025 00:15:58 +0200 Subject: [PATCH 41/45] SafePatternCompiler: create, use and add unit test --- .../helpers/patterns/SafePatternCompiler.java | 20 +++++ .../ParsedFirewallLists.java | 18 ++-- .../java/helpers/SafePatternCompilerTest.java | 86 +++++++++++++++++++ .../storage/ServiceConfigurationTest.java | 22 +++++ 4 files changed, 135 insertions(+), 11 deletions(-) create mode 100644 agent_api/src/main/java/dev/aikido/agent_api/helpers/patterns/SafePatternCompiler.java create mode 100644 agent_api/src/test/java/helpers/SafePatternCompilerTest.java diff --git a/agent_api/src/main/java/dev/aikido/agent_api/helpers/patterns/SafePatternCompiler.java b/agent_api/src/main/java/dev/aikido/agent_api/helpers/patterns/SafePatternCompiler.java new file mode 100644 index 00000000..61462bba --- /dev/null +++ b/agent_api/src/main/java/dev/aikido/agent_api/helpers/patterns/SafePatternCompiler.java @@ -0,0 +1,20 @@ +package dev.aikido.agent_api.helpers.patterns; + +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +public final class SafePatternCompiler { + private SafePatternCompiler() { + } + + public static Pattern compilePatternSafely(String regex, int flags) { + if (regex == null || regex.isEmpty()) { + return null; + } + try { + return Pattern.compile(regex, flags); + } catch (PatternSyntaxException ignored) { + return null; + } + } +} diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java index ae461a4a..872f3cea 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java @@ -8,6 +8,7 @@ import java.util.regex.Pattern; import static dev.aikido.agent_api.helpers.IPListBuilder.createIPList; +import static dev.aikido.agent_api.helpers.patterns.SafePatternCompiler.compilePatternSafely; public class ParsedFirewallLists { private final List blockedIps = new ArrayList<>(); @@ -109,21 +110,16 @@ public void updateUADetails(List userAgentDetails) if (userAgentDetails == null) return; for (ReportingApi.UserAgentDetail entry : userAgentDetails) { - Pattern pattern = Pattern.compile(entry.pattern(), Pattern.CASE_INSENSITIVE); - this.uaDetails.add(new UADetailsEntry(entry.key(), pattern)); + Pattern pattern = compilePatternSafely(entry.pattern(), Pattern.CASE_INSENSITIVE); + if (pattern != null) { + this.uaDetails.add(new UADetailsEntry(entry.key(), pattern)); + } } } public void updateBlockedAndMonitoredUAs(String blockedUAs, String monitoredUAs) { - this.blockedUserAgents = null; - if (blockedUAs != null && !blockedUAs.isEmpty()) { - this.blockedUserAgents = Pattern.compile(blockedUAs, Pattern.CASE_INSENSITIVE); - } - - this.monitoredUserAgents = null; - if (monitoredUAs != null && !monitoredUAs.isEmpty()) { - this.monitoredUserAgents = Pattern.compile(monitoredUAs, Pattern.CASE_INSENSITIVE); - } + this.blockedUserAgents = compilePatternSafely(blockedUAs, Pattern.CASE_INSENSITIVE); + this.monitoredUserAgents = compilePatternSafely(monitoredUAs, Pattern.CASE_INSENSITIVE); } diff --git a/agent_api/src/test/java/helpers/SafePatternCompilerTest.java b/agent_api/src/test/java/helpers/SafePatternCompilerTest.java new file mode 100644 index 00000000..f91f70b6 --- /dev/null +++ b/agent_api/src/test/java/helpers/SafePatternCompilerTest.java @@ -0,0 +1,86 @@ +package helpers; + +import dev.aikido.agent_api.helpers.patterns.SafePatternCompiler; +import org.junit.jupiter.api.Test; + +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.*; + +public class SafePatternCompilerTest { + + @Test + public void testCompilePatternSafely_ValidRegex() { + String regex = "^[a-zA-Z0-9]+$"; // Valid regex + Pattern pattern = SafePatternCompiler.compilePatternSafely(regex, 0); + assertNotNull(pattern); + assertTrue(pattern.matcher("Test123").matches()); + assertFalse(pattern.matcher("Test 123").matches()); + } + + @Test + public void testCompilePatternSafely_EmptyRegex() { + String regex = ""; // Empty regex + Pattern pattern = SafePatternCompiler.compilePatternSafely(regex, 0); + assertNull(pattern); + } + + @Test + public void testCompilePatternSafely_NullRegex() { + String regex = null; // Null regex + Pattern pattern = SafePatternCompiler.compilePatternSafely(regex, 0); + assertNull(pattern); + } + + @Test + public void testCompilePatternSafely_InvalidRegex_UnmatchedParentheses() { + String regex = "(abc"; // Invalid regex + Pattern pattern = SafePatternCompiler.compilePatternSafely(regex, 0); + assertNull(pattern); + } + + @Test + public void testCompilePatternSafely_InvalidRegex_UnmatchedBrackets() { + String regex = "[abc"; // Invalid regex + Pattern pattern = SafePatternCompiler.compilePatternSafely(regex, 0); + assertNull(pattern); + } + + @Test + public void testCompilePatternSafely_InvalidRegex_EmptyCharacterClass() { + String regex = "[]"; // Invalid regex + Pattern pattern = SafePatternCompiler.compilePatternSafely(regex, 0); + assertNull(pattern); + } + + @Test + public void testCompilePatternSafely_InvalidRegex_InvalidQuantifier() { + String regex = "a{2,1}"; // Invalid regex + Pattern pattern = SafePatternCompiler.compilePatternSafely(regex, 0); + assertNull(pattern); + } + + @Test + public void testCompilePatternSafely_ValidRegex_WithFlags() { + String regex = "abc"; // Valid regex + Pattern pattern = SafePatternCompiler.compilePatternSafely(regex, Pattern.CASE_INSENSITIVE); + assertNotNull(pattern); + assertTrue(pattern.matcher("ABC").matches()); + } + + @Test + public void testCompilePatternSafely_InvalidRegex_NamedGroup() { + String regex = "(? Date: Tue, 13 May 2025 00:18:39 +0200 Subject: [PATCH 42/45] Rename allowedIpsList/blockedIpsList -> allowedIpLists/blockedIpLists --- .../service_configuration/ParsedFirewallLists.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java index 872f3cea..5fa92073 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java @@ -75,11 +75,11 @@ public void update(ReportingApi.APIListsResponse response) { updateUADetails(response.userAgentDetails()); } - public void updateBlockedIps(List blockedIpsList) { + public void updateBlockedIps(List blockedIpLists) { blockedIps.clear(); - if (blockedIpsList == null) + if (blockedIpLists == null) return; - for (ReportingApi.ListsResponseEntry entry : blockedIpsList) { + for (ReportingApi.ListsResponseEntry entry : blockedIpLists) { IPList ipList = createIPList(entry.ips()); blockedIps.add(new IPEntry(/* monitor */ false, entry.key(), entry.source(), entry.description(), ipList)); } @@ -94,11 +94,11 @@ public void updateMonitoredIps(List monitoredIp } } - public void updateAllowedIps(List allowedIpsList) { + public void updateAllowedIps(List allowedIpLists) { allowedIps.clear(); - if (allowedIpsList == null) + if (allowedIpLists == null) return; - for (ReportingApi.ListsResponseEntry entry : allowedIpsList) { + for (ReportingApi.ListsResponseEntry entry : allowedIpLists) { IPList ipList = createIPList(entry.ips()); boolean shouldMonitor = false; // we don't monitor allowed ips allowedIps.add(new IPEntry(shouldMonitor, entry.key(), entry.source(), entry.description(), ipList)); From b0d2d842f899944e063c5d31f6ce7b1acf43f8a6 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Tue, 13 May 2025 00:25:48 +0200 Subject: [PATCH 43/45] Modify isIpBlocked by early-returning for readability --- .../agent_api/storage/ServiceConfiguration.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java index e01f2392..390be6aa 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java @@ -85,24 +85,24 @@ public boolean isIpBypassed(String ip) { * Check if the IP is blocked (e.g. Geo IP Restrictions) */ public BlockedResult isIpBlocked(String ip) { - BlockedResult blockedResult = new BlockedResult(false, null); - // Check for allowed ip addresses (i.e. only one country is allowed to visit the site) // Always allow access from private IP addresses (those include local IP addresses) if (!isPrivateIp(ip) && !firewallLists.matchesAllowedIps(ip)) { - blockedResult = new BlockedResult(true, "not in allowlist"); + return new BlockedResult(true, "not in allowlist"); } // Check for blocked ip addresses - for (ParsedFirewallLists.Match match : firewallLists.matchBlockedIps(ip)) { + List blockedIpMatches = firewallLists.matchBlockedIps(ip); + for (ParsedFirewallLists.Match match : blockedIpMatches) { StatisticsStore.incrementIpHits(match.key()); - // when a blocking match is found, set blocked result if it hasn't been set already. - if (match.block() && !blockedResult.blocked()) { - blockedResult = new BlockedResult(true, match.description()); + } + for (ParsedFirewallLists.Match match : firewallLists.matchBlockedIps(ip)) { + if (match.block()) { + return new BlockedResult(true, match.description()); } } - return blockedResult; + return new BlockedResult(false, null); } public void updateBlockedLists(ReportingApi.APIListsResponse res) { From 261ce0dad7dbbd99686e4634d46ded1000f4fa7b Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Tue, 13 May 2025 00:30:34 +0200 Subject: [PATCH 44/45] extract then blockedIps.clear() from the updateBlockedIps --- .../storage/service_configuration/ParsedFirewallLists.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java index 5fa92073..8a6d05f0 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java @@ -67,8 +67,10 @@ public UABlockedResult matchBlockedUserAgents(String userAgent) { } public void update(ReportingApi.APIListsResponse response) { + blockedIps.clear(); updateBlockedIps(response.blockedIPAddresses()); updateMonitoredIps(response.monitoredIPAddresses()); + updateAllowedIps(response.allowedIPAddresses()); updateBlockedAndMonitoredUAs(response.blockedUserAgents(), response.monitoredUserAgents()); @@ -76,7 +78,6 @@ public void update(ReportingApi.APIListsResponse response) { } public void updateBlockedIps(List blockedIpLists) { - blockedIps.clear(); if (blockedIpLists == null) return; for (ReportingApi.ListsResponseEntry entry : blockedIpLists) { From fc3cea34624847bfff2199047479f663d79bda34 Mon Sep 17 00:00:00 2001 From: BitterPanda63 Date: Tue, 13 May 2025 00:33:03 +0200 Subject: [PATCH 45/45] WebRequestCollector simplify with early returns --- .../agent_api/collectors/WebRequestCollector.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java b/agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java index 58ef79ca..483c97c1 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java @@ -40,15 +40,14 @@ public static Res report(ContextObject newContext) { StatisticsStore.incrementHits(); Res endpointAllowlistRes = checkEndpointAllowlist(newContext.getRouteMetadata(), newContext.getRemoteAddress(), config); - Res blockedIpsRes = checkBlockedIps(newContext.getRemoteAddress(), config); - Res blockedUserAgentsRes = checkBlockedUserAgents(newContext.getHeader("user-agent"), config); - - // make sure to follow a certain order when giving error messages. if (endpointAllowlistRes != null) return endpointAllowlistRes; - else if (blockedIpsRes != null) + + Res blockedIpsRes = checkBlockedIps(newContext.getRemoteAddress(), config); + if (blockedIpsRes != null) return blockedIpsRes; - else return blockedUserAgentsRes; + + return checkBlockedUserAgents(newContext.getHeader("user-agent"), config); } private static Res checkEndpointAllowlist(RouteMetadata routeMetadata, String remoteAddress, ServiceConfiguration config) {