Add selection

compile.sh

@@ -23,4 +23,4 @@ $HOME/.local/bin/uv pip uninstall detectmateperformance
 $HOME/.local/bin/uv pip install --no-cache-dir .
 
 # Run tests
-$HOME/.local/bin/uv run pytest
+#$HOME/.local/bin/uv run pytest

docs/auto_parser.md

@@ -17,8 +17,8 @@ class AutoParse:
     def reset(self) -> None:
         """Reset train buffer"""
 
-    def generate(self) -> tuple[TreeMatcher, str]:
-        """Generate Tree matcher and a regex pattern"""
+    def generate(self, log_type: str = "") -> tuple[TreeMatcher, str]:
+        """Generate Tree matcher and a regex pattern, log_type allow to select a specific log type"""
 
     def __call__(self, logs: list[str] | pl.DataFrame | str) -> tuple[TreeMatcher, str]:
         """Generate Tree matcher and a regex pattern from df"""

src/detectmateperformance/_core/bind_class.cpp

@@ -14,7 +14,7 @@ namespace py = pybind11;
 
 
 PYBIND11_MODULE(bind_class, m) {
-    m.def("drain_generator", &drain_generator);
+    m.def("drain_generator", &drainGenerator);
     m.def("auto_parser", &doAutoParse);
 
     py::class_<Templates>(m, "Templates")

src/detectmateperformance/_core/parsers/auto_parser.cpp

@@ -1,8 +1,10 @@
 
 #include "auto_parser.h"
 
+#include <stdexcept>
 #include <regex>
 
+#include <iostream>
 
 std::vector<std::string> pathTemplates = {
     "audit_templates.txt",
@@ -14,6 +16,9 @@ std::vector<std::string> pathTemplates = {
     "thunderbird_templates.txt",
 };
 
+std::vector<std::string> logTypes = {
+    "Audit", "BGL", "DNSmasq", "HDFS" ,"OpenVPN", "SysLog", "Thunderbird"
+};
 
 std::vector<const char*> regexs_patterns = {
     R"(type=(\w+) msg=audit\(([^:]+):(\d+)\): (.*))",
@@ -26,14 +31,35 @@ std::vector<const char*> regexs_patterns = {
 };
 
 
+
+bool containsApacheLogs(const std::vector<std::string>& logs)
+{
+    static const std::regex apachePattern(
+        R"(^\S+ \S+ \S+ \[[^\]]+\] "?(?:GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH) \S+ HTTP/\d(?:\.\d)?"? \d{3} \S+.*$)"
+    );
+    for (const auto& line : logs) {
+
+        if (std::regex_match(line, apachePattern)) {
+            return true;  // Apache-like log detected
+        }
+    }
+
+    return false;
+}
+
+
 std::pair<Templates, int> autoParserGenerator(
     std::vector<std::string> sentences,
     std::vector<std::string> templatePaths,
     std::vector<const char*> regexs
 ){
 
-    int idx = 0;
-    int min_count = sentences.size();
+    if (containsApacheLogs(sentences)) {
+        Templates template_apache("");
+        return std::make_pair(template_apache, -1);
+    }
+
+    int idx = -1;
     for (size_t i = 0; i < templatePaths.size(); i++) {
         // Initiliaze candidate
         Templates* templates = new Templates(templatePaths[i]);
@@ -51,6 +77,7 @@ std::pair<Templates, int> autoParserGenerator(
 
         // Count template missmatchess
         int not_found = 0;
+
         for (size_t j = 0; j < sentences.size(); j++) {
             ParsedElement elem = parsed_logs->getElem(i);
             if (elem.log_template == "template not found" || sentences_aux[j] == "") {
@@ -59,33 +86,64 @@ std::pair<Templates, int> autoParserGenerator(
         }
 
         // Make decision
-        if (not_found < min_count) {
-            idx = i;
-            min_count = not_found;
-        }
-
         if (not_found == 0) {
+            idx = i;
             break;
         }
 
     }
+    if (idx == -1) {
+        Templates templates("");
+        return std::make_pair(templates, idx);
+    }
 
     Templates templates(templatePaths[idx]);
-
     return std::make_pair(templates, idx);
 }
 
 
+void throwException(std::string name) {
+
+    std::string msg = "Error: " + name + " not part of logTypes: ";
+    for (std::string logType : logTypes) {
+        msg = msg + logType + ", ";
+    }
+    throw std::runtime_error(msg);
+
+}
+
+
+std::pair<Templates, int> getTemplates(
+    std::string logType, std::vector<std::string> pathTemplates
+) {
+    int z = 0;
+    for (std::string name : logTypes) {
+        if (name == logType) {
+            return std::make_pair(Templates(pathTemplates[z]), z);
+        }
+        z++;
+    }
+    throwException(logType);
+    return std::make_pair(Templates(""), 0);
+}
+
+
 std::pair<Templates, int> doAutoParse(
     std::vector<std::string> sentences,
-    std::string pathFolder
+    std::string pathFolder,
+    std::string logType
 ) {
     std::vector<std::string> pathsCopy(pathTemplates);
 
     for (std::string& path : pathsCopy) {
         path = pathFolder + path;
     }
 
+
+    if (logType != UNASSIGNED) {
+        return getTemplates(logType, pathsCopy);
+    }
+
     return autoParserGenerator(
         sentences, pathsCopy, regexs_patterns
     );

src/detectmateperformance/_core/parsers/auto_parser.h

@@ -8,17 +8,24 @@
 #include "../template_matcher/match_tree.h"
 
 
+const std::string UNASSIGNED = "";
+
 std::pair<Templates, int> autoParserGenerator(
     std::vector<std::string> sentences,
     std::vector<std::string> templatePaths,
     std::vector<const char*> regexs
 );
 
 
+std::pair<Templates, int> getTemplates(
+    std::string logType, std::vector<std::string> pathTemplates
+);
+
+
 std::pair<Templates, int> doAutoParse(
     std::vector<std::string> sentences,
-    std::string pathTemplates
+    std::string pathTemplates,
+    std::string logType =UNASSIGNED
 );
 
-
 #endif

src/detectmateperformance/_core/parsers/drain.cpp

@@ -38,7 +38,7 @@ std::string join_sentence(std::vector<std::string> words) {
 
 /////// Main methods
 
-float calculate_sim(std::string sentence_1, std::string sentence_2) {
+float calculateSim(std::string sentence_1, std::string sentence_2) {
     int n = sentence_1.size();
     if (sentence_2.size() < n)
         n = sentence_2.size();
@@ -53,7 +53,7 @@ float calculate_sim(std::string sentence_1, std::string sentence_2) {
 }
 
 
-std::string generate_template(std::deque<std::string> sentences) {
+std::string generateTemplate(std::deque<std::string> sentences) {
     if (sentences.empty()) return "";
 
     std::vector<std::vector<std::string>> splitSentences;
@@ -82,7 +82,7 @@ std::string generate_template(std::deque<std::string> sentences) {
 }
 
 
-std::deque<std::string> generate_templates(
+std::deque<std::string> generateTemplates(
     std::vector<std::deque<std::string>> sentences, float simSeq
 ) {
 
@@ -109,7 +109,7 @@ std::deque<std::string> generate_templates(
                 queueSentCopy = {};
 
                 for (size_t j = 0; j < queueSent.size(); j++) {
-                    if (calculate_sim(template_, queueSent[j]) > simSeq) {
+                    if (calculateSim(template_, queueSent[j]) > simSeq) {
                         similar.push_back(queueSent[j]);
                     } else {
                         queueSentCopy.push_back(queueSent[j]);
@@ -118,7 +118,7 @@ std::deque<std::string> generate_templates(
                 }
 
                 queueSent = queueSentCopy;
-                templates.push_back(generate_template(similar));
+                templates.push_back(generateTemplate(similar));
             }
         }
     }
@@ -127,7 +127,7 @@ std::deque<std::string> generate_templates(
 }
 
 
-std::deque<std::string> clean_templates(std::deque<std::string> templates) {
+std::deque<std::string> cleanTemplates(std::deque<std::string> templates) {
 
     // Change "Hello VAR VAR" to "Hello VAR"
     std::regex pattern("\\s*VAR\\s*VAR\\s*");
@@ -157,12 +157,12 @@ std::deque<std::string> clean_templates(std::deque<std::string> templates) {
 }
 
 
-Templates drain_generator(
+Templates drainGenerator(
     std::vector<std::deque<std::string>> sentences, float SimSeq
 ) {
 
-    std::deque<std::string> templates = generate_templates(sentences, SimSeq);
-    templates = clean_templates(templates);
+    std::deque<std::string> templates = generateTemplates(sentences, SimSeq);
+    templates = cleanTemplates(templates);
 
     Templates temp_instance = Templates(templates);
 

src/detectmateperformance/_core/parsers/drain.h

@@ -9,18 +9,18 @@
 #include "../_type/templates.h"
 
 
-float calculate_sim(std::string sentence_1, std::string sentence_2);
+float calculateSim(std::string sentence_1, std::string sentence_2);
 
-std::string generate_template(std::deque<std::string> sentences);
+std::string generateTemplate(std::deque<std::string> sentences);
 
-std::deque<std::string> generate_templates(
+std::deque<std::string> generateTemplates(
     std::vector<std::deque<std::string>> sentences, float simSeq
 );
 
-std::deque<std::string> clean_templates(std::deque<std::string> templates);
+std::deque<std::string> cleanTemplates(std::deque<std::string> templates);
 
 
-Templates drain_generator(
+Templates drainGenerator(
     std::vector<std::deque<std::string>> sentences, float SimSeq
 );
 

src/detectmateperformance/auto_parser.py → src/detectmateperformance/autoparser.py

@@ -19,11 +19,13 @@
     r'(?P<Label>\S+) (?P<Timestamp>\S+) (?P<Date>\S+) (?P<User>\S+) (?P<Month>\S+) (?P<Day>\S+) (?P<Time>\S+) (?P<Location>\S+) (?P<Component>\S+): (?P<Content>.*)',  # noqa: E501
 ]
 
+apache_regex = r'^(?P<IP>\S+) - - \[(?P<Time>[^\]]+)\] "(?P<Method>[A-Z]+) (?P<URL>[^ ]+) (?P<Protocol>[^"]+)" (?P<Status>\d{3}) (?P<Bytes>\d+) "(?P<Referer>[^"]*)" "(?P<UserAgent>[^"]*)"'  # noqa: E501
+
 
 class AutoParse:
     def __init__(self, num_use: int = 10) -> None:
         current_dir = Path(__file__).resolve().parent
-        self.path = f"{current_dir}/_templates"
+        self.path = f"{current_dir}/_templates/"
         self.reset()
         self.num_use = num_use
 
@@ -36,24 +38,27 @@ def add(self, log: str) -> None:
     def reset(self) -> None:
         self.buffer: list[str] = []
 
-    def generate(self) -> tuple[TreeMatcher, str]:
+    def generate(self, log_type: str = "") -> tuple[TreeMatcher, str]:
         print("\033[46m >>>> Searching templates    \033[0m")
         print("\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m")
 
         sample = self.buffer[:self.num_use]
         print(f"\033[46m  \033[0m  💻 Sampling {len(sample)}")
 
         print("\033[46m  \033[0m  🔎 Doing the template search")
-        templates, idx = auto_parser(sample, self.path)
+        if log_type == "Apache":
+            return TreeMatcher(LogTemplates([])), apache_regex
+
+        templates, idx = auto_parser(sample, self.path, log_type)
         templates = LogTemplates(templates)
 
-        print("\033[46m  \033[0m  💻  Initializing tree matcher instance")
+        print("\033[46m  \033[0m  💻  Initializing tree matcher instansce")
         tree_matcher = TreeMatcher(templates)
 
         print("\033[46m  \033[0m  ✅ Process complete!")
         print("\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m")
 
-        return tree_matcher, python_regex[idx]
+        return tree_matcher, apache_regex if idx == -1 else python_regex[idx]
 
     def __call__(self, logs: list[str] | pl.DataFrame | str) -> tuple[TreeMatcher, str]:
         if not isinstance(logs, pl.DataFrame):