ait-aecid
diff --git a/‎Jenkinsfile
+34-3 b/‎Jenkinsfile
+34-3
diff --git a/‎README.md
+1-1 b/‎README.md
+1-1
diff --git a/‎aecid-testsuite/Dockerfile
+2 b/‎aecid-testsuite/Dockerfile
+2
diff --git a/‎aecid-testsuite/demo/aminer/aminerDemo.sh
+15-4 b/‎aecid-testsuite/demo/aminer/aminerDemo.sh
+15-4
diff --git a/‎aecid-testsuite/demo/aminer/demo-config.py
+38-9 b/‎aecid-testsuite/demo/aminer/demo-config.py
+38-9
@@ -12,6 +12,7 @@ def  ubuntu18image = false
 def  ubuntu20image = false
 def  debianbusterimage = false
 def  debianbullseyeimage = false
+def  productionimage = false
 def  docsimage = false
 
 pipeline {
@@ -39,6 +40,11 @@ pipeline {
          }
          stage("Run Demo-Configs"){
              parallel {
+                 stage("available parsing models in conf-available") {
+                     steps {
+                         sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runConfAvailableTest"
+                     }
+                 }
                  stage("demo-config and jsonConverterHandler-demo-config") {
                      steps {
                          sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runAminerDemo demo/aminer/demo-config.py"
@@ -77,6 +83,7 @@ pipeline {
                  sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runJsonDemo demo/aminerJsonInputDemo/json-eve-demo.yml"
                  sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runJsonDemo demo/aminerJsonInputDemo/json-journal-demo.yml"
                  sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runJsonDemo demo/aminerJsonInputDemo/json-wazuh-demo.yml"
+                 sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runJsonDemo demo/aminerJsonInputDemo/windows.yml"
              }
          }
 
@@ -95,20 +102,23 @@ pipeline {
              }
              steps {
                  sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runTryItOut development"
-//                 sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runGettingStarted development"
+                 sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runGettingStarted development"
                  sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runHowToCreateYourOwnSequenceDetector development"
                  sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runHowToCreateYourOwnFrequencyDetector development"
                  sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runHowToMissingMatchPathValueDetector development"
              }
          }
 
-    stage("Wiki Tests - main"){
+         stage("Wiki Tests - main"){
              when {
                  branch "main"
              }
              steps {
                  sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runTryItOut main"
-                 // sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runGettingStarted main"
+                 sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runGettingStarted main"
+                 sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runHowToCreateYourOwnSequenceDetector main"
+                 sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runHowToCreateYourOwnFrequencyDetector main"
+                 sh "docker run -m=2G --rm aecid/logdata-anomaly-miner-testing:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID runHowToMissingMatchPathValueDetector main"
              }
          }
 
@@ -153,6 +163,23 @@ pipeline {
                    }
                  }
 
+                 stage("Test Production Docker Image") {
+                    steps {
+                    script {
+                      productionimage = true
+                    }
+                     sh "docker build -f Dockerfile -t aecid/aminer-production:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID ."
+                     sh "mkdir -p /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID && mkdir /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID/persistency && mkdir /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID/logs"
+                     sh "cp aecid-testsuite/demo/aminer/access.log /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID/logs/"
+                     sh "cp -r source/root/etc/aminer /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID/aminercfg"
+                     sh "cp /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID/aminercfg/template_config.yml /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID/aminercfg/config.yml"
+                     sh "cp /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID/aminercfg/conf-available/generic/ApacheAccessModel.py /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID/aminercfg/conf-enabled"
+                     /* the result of timeout is negated with "!". This is because aminer returns 1 if timeout stops the process and otherwise 0. The way around is a valid result for a test */
+                     sh "cd /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID && ! timeout -s INT --preserve-status 5 docker run -v $PWD/aminercfg:/etc/aminer -v $PWD/persistency:/var/lib/aminer -v $PWD/logs:/logs --rm -it aecid/aminer-production:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID aminer"
+                   }
+                 }
+
+
         stage("Test Ubuntu 18.04") {
                     when {
                        expression {
@@ -217,6 +244,10 @@ pipeline {
                sh "docker rmi aecid/aminer-debian-buster:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID"
                sh "cd / && test -d /tmp/simplerun-buster-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID && rm -rf /tmp/simplerun-buster-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID"
            }
+           if( productionimage == true ){
+               sh "docker rmi aecid/aminer-production:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID"
+               sh "cd / && test -d /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID && rm -rf /tmp/production-$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID"
+           }
            if( ubuntu18image == true ){
                sh "docker rmi aecid/aminer-ubuntu-1804:$JOB_BASE_NAME-$EXECUTOR_NUMBER-$BUILD_ID"
            }
 
@@ -8,7 +8,7 @@ This tool parses log data and allows to define analysis pipelines for anomaly de
 
 In order to install logdata-anomaly-miner a **Linux system** with **python >= 3.6** is required. **Debian-based** distributions are currently recommended. 
 
-_See [requirements.txt](requirements.txt) for further module dependencies_
+_See [requirements.txt](https://github.com/ait-aecid/logdata-anomaly-miner/requirements.txt) for further module dependencies_
 
 
 ## Installation
 
@@ -80,6 +80,8 @@ ADD source/root/usr/lib/logdata-anomaly-miner /usr/lib/logdata-anomaly-miner
 # copy these files instead as symlinks would need absolute paths.
 ADD source/root/etc/aminer/conf-available/ait-lds/* /etc/aminer/conf-enabled/
 ADD source/root/etc/aminer/conf-available/generic/* /etc/aminer/conf-enabled/
+ADD source/root/etc/aminer/conf-available/ait-lds /etc/aminer/conf-available/ait-lds
+ADD source/root/etc/aminer/conf-available/generic /etc/aminer/conf-available/generic
 
 # Entrypoint-wrapper
 ADD scripts/aminerwrapper.sh /aminerwrapper.sh
 
@@ -25,15 +25,15 @@ sudo aminer --config "$FILE" &
 
 #EventCorrelationDetector, NewMatchPathDetector
 #:<<Comment
-alphabet='abcdef'
+alphabet='ghijkl'
 alphabet_len=$(echo -n $alphabet | wc -m)
 for ((i=0; i<10000; i++)); do
 	echo ${alphabet:$i % $alphabet_len:1} >> /tmp/syslog
 	sleep 0.0001
 done
 #Comment
 
-#EnhancedNewMatchPathValueComboDetector, NewMatchPathValueDetector
+#EnhancedNewMatchPathValueComboDetector, NewMatchPathValueDetector, ModuloTimeMatchRule
 #:<<Comment
 R=`shuf -i 1-3 -n 1`
 for ((i=0; i<R; i++)); do
@@ -161,6 +161,9 @@ echo 'type=SYSCALL msg=audit(1580367400.000:9): arch=c000003e syscall=2 success=
 echo 'type=PATH msg=audit(1580367401.000:9): item=0 name="three" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 nametype=NORMAL' >> /tmp/syslog
 echo 'type=PATH msg=audit(1580367402.000:10): item=0 name="one" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 nametype=NORMAL' >> /tmp/syslog
 echo 'type=SYSCALL msg=audit(1580367403.000:10): arch=c000003e syscall=3 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)' >> /tmp/syslog
+
+# StringRegexMatchRule
+echo 'type=SYSCALL msg=audit(1580367403.000:10): arch=c000003e syscall=3 success=no exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)' >> /tmp/syslog
 #Comment
 
 #TimeCorrelationDetector
@@ -228,21 +231,29 @@ echo "match data: 25000" >> /tmp/syslog
 echo "b654686973206973206a7573742061206e6f726d616c2074657874" >> /tmp/syslog
 # IpAddressModelElement
 echo "Gateway IP-Address: 192.168.128.225" >> /tmp/syslog
+# IPv4InRFC1918MatchRule, ValueListMatchRule
+echo "Gateway IP-Address: 8.8.8.8" >> /tmp/syslog
+# IPv4InRFC1918MatchRule, ValueListMatchRule
+echo "Gateway IP-Address: 8.8.4.4" >> /tmp/syslog
+# IPv4InRFC1918MatchRule, ValueRangeMatchRule
+echo "Gateway IP-Address: 10.0.0.0" >> /tmp/syslog
+# IPv4InRFC1918MatchRule, ValueRangeMatchRule
+echo "Gateway IP-Address: 11.0.0.0" >> /tmp/syslog
 # MultiLocaleDateTimeModelElement
 echo "Feb 25 2019" >> /tmp/syslog
 # OptionalMatchModelElement
 echo "The-searched-element-was-found!" >> /tmp/syslog
 # RepeatedElementDataModelElement
 for i in {1..5}; do
 	R=`shuf -i 1-45 -n 1`
-	echo "drawn number: $R" | tr -d "\n" >> /tmp/syslog
+	echo "[drawn number]: $R" | tr -d "\n" >> /tmp/syslog
 done
 echo "" >> /tmp/syslog
 # VariableByteDataModelElement
 echo "---------------------------------------------------------------------" >> /tmp/syslog
 # WhiteSpaceLimitedDataModelElement
 alphabet="abcdefghijklmnopqrstuvwxyz "
-text=""
+text="z"
 for i in {1..1000}; do
 	R=`shuf -i 0-26 -n 1`
 	text=$text${alphabet:R:1}
 
@@ -111,7 +111,7 @@ def build_analysis_pipeline(analysis_context):
         DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest')]
 
     service_children_login_details = [
-        FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '),
+        FixedDataModelElement('User/LoginDetails', b'User '), DelimitedDataModelElement('Username', b' '),
         FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']), OptionalMatchModelElement(
             'PastTime', SequenceModelElement('Time', [
                 FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'),
@@ -134,7 +134,7 @@ def build_analysis_pipeline(analysis_context):
         DateTimeModelElement('DTM', date_format_string)]
 
     service_children_user_ip_address = [
-        FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '),
+        FixedDataModelElement('User/UserIPAddress', b'User '), DelimitedDataModelElement('Username', b' '),
         FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP')]
 
     service_children_cron_job_announcement = [
@@ -159,7 +159,8 @@ def build_analysis_pipeline(analysis_context):
             FixedDataModelElement('name_string', b' name="'), DelimitedDataModelElement('name', b'"'),
             FixedDataModelElement('inode_string', b'" inode='), DecimalIntegerValueModelElement('inode'),
             FixedDataModelElement('dev_string', b' dev='), DelimitedDataModelElement('dev', b' '),
-            FixedDataModelElement('mode_string', b' mode='), DecimalIntegerValueModelElement('mode'),
+            FixedDataModelElement('mode_string', b' mode='),
+            DecimalIntegerValueModelElement('mode', value_pad_type=DecimalIntegerValueModelElement.PAD_TYPE_ZERO),
             FixedDataModelElement('ouid_string', b' ouid='), DecimalIntegerValueModelElement('ouid'),
             FixedDataModelElement('ogid_string', b' ogid='), DecimalIntegerValueModelElement('ogid'),
             FixedDataModelElement('rdev_string', b' rdev='), DelimitedDataModelElement('rdev', b' '),
@@ -202,7 +203,7 @@ def build_analysis_pipeline(analysis_context):
         MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', None, '%s.%s' % loc)]))
     service_children_parsing_model_element.append(
         RepeatedElementDataModelElement('RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [
-            FixedDataModelElement('FixedDataModelElement', b'drawn number: '),
+            FixedDataModelElement('FixedDataModelElement', b'[drawn number]: '),
             DecimalIntegerValueModelElement('DecimalIntegerValueModelElement')]), 1))
     service_children_parsing_model_element.append(VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
     service_children_parsing_model_element.append(
@@ -215,11 +216,11 @@ def build_analysis_pipeline(analysis_context):
     # The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets to the
     # AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
     service_children_parsing_model_element.append(OptionalMatchModelElement(
-        'OptionalMatchModelElement', FirstMatchModelElement('FirstMatchModelElement', [
+        '/', FirstMatchModelElement('FirstMatchModelElement//optional', [
             FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), SequenceModelElement('se', [
                 FixedDataModelElement('FixedDME', b'Any:'), AnyByteDataModelElement('AnyByteDataModelElement')])])))
 
-    alphabet = b'abcdef'
+    alphabet = b'ghijkl'
     service_children_ecd = []
     for _, char in enumerate(alphabet):
         char = bytes([char])
@@ -280,10 +281,12 @@ def build_analysis_pipeline(analysis_context):
         Rules.OrMatchRule([
             Rules.AndMatchRule([
                 Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes'),
-                Rules.NegationMatchRule(Rules.ValueMatchRule('/model/LoginDetails/Username', b'root'))]),
+                Rules.NegationMatchRule(Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')),
+                Rules.DebugMatchRule(debug_match_result=True)]),
             Rules.AndMatchRule([
                 Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes')),
-                Rules.PathExistsMatchRule('/model/LoginDetails')]),
+                Rules.PathExistsMatchRule('/model/LoginDetails'),
+                Rules.DebugMatchRule(debug_match_result=True)]),
             Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails'))])]
 
     allowlist_violation_detector = AllowlistViolationDetector(analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers,
@@ -325,7 +328,7 @@ def build_analysis_pipeline(analysis_context):
 
     from aminer.analysis.EventSequenceDetector import EventSequenceDetector
     esd = EventSequenceDetector(analysis_context.aminer_config, anomaly_event_handlers, ['/model/ParsingME'], ignore_list=[
-        '/model/ECD/a', '/model/ECD/b', '/model/ECD/c', '/model/ECD/d', '/model/ECD/e', '/model/ECD/f', '/model/Random',
+        '/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l', '/model/Random',
         '/model/RandomTime', '/model/DailyCron'])
     analysis_context.register_component(esd, component_name="EventSequenceDetector")
     atom_filter.add_handler(esd)
@@ -360,6 +363,32 @@ def tuple_transformation_function(match_value_list):
     analysis_context.register_component(enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo")
     atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
 
+    import re
+    ip_match_action = Rules.EventGenerationMatchAction(
+        "Analysis.Rules.IPv4InRFC1918MatchRule", "Private IP address occurred!", anomaly_event_handlers)
+
+    vdmt = Rules.ValueDependentModuloTimeMatchRule(None, 3, ["/model/ECD/j", "/model/ECD/k", "/model/ECD/l"], {b"e": [0, 2.95]}, [0, 3])
+    mt = Rules.ModuloTimeMatchRule(None, 3, 0, 3, None)
+    time_allowlist_rules = [
+        Rules.AndMatchRule([
+            Rules.ParallelMatchRule([
+                Rules.ValueDependentDelegatedMatchRule([
+                    '/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l'], {
+                        (b"a",): mt, (b"b",): mt, (b"c",): mt, (b"d",): vdmt, (b"e",): vdmt, (b"f",): vdmt, None: mt}, mt),
+                Rules.IPv4InRFC1918MatchRule("/model/ParsingME/se2/IpAddressDataModelElement", ip_match_action),
+                Rules.DebugHistoryMatchRule(debug_match_result=True)
+            ]),
+            # IP addresses 8.8.8.8, 8.8.4.4 and 10.0.0.0 - 10.255.255.255 are not allowed
+            Rules.NegationMatchRule(Rules.ValueListMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", [134744072, 134743044])),
+            Rules.NegationMatchRule(Rules.ValueRangeMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", 167772160, 184549375)),
+            Rules.NegationMatchRule(Rules.StringRegexMatchRule("/model/type/syscall/success", re.compile(b"^no$")))
+        ])
+    ]
+    time_allowlist_violation_detector = AllowlistViolationDetector(
+        analysis_context.aminer_config, time_allowlist_rules, anomaly_event_handlers, output_log_line=True)
+    analysis_context.register_component(time_allowlist_violation_detector, component_name="TimeAllowlist")
+    atom_filter.add_handler(time_allowlist_violation_detector)
+
     from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
         PathDependentHistogramAnalysis
     modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, True)