Development

CMakeLists.txt

@@ -85,6 +85,11 @@ add_library(
   src/detectmateperformance/_core/parsers/drain.cpp
 )
 
+add_library(
+  auto_parser_op
+  src/detectmateperformance/_core/parsers/auto_parser.h
+  src/detectmateperformance/_core/parsers/auto_parser.cpp
+)
 # match_tree depends on variable and tree_op; make those transitive so
 # consumers of match_tree (executables/tests) don't need to worry about
 # static library link ordering.
@@ -95,6 +100,7 @@ target_link_libraries(
   tree_op
   tree
   drain_op
+  auto_parser_op
 )
 
 add_executable(
@@ -122,6 +128,11 @@ add_executable(
   tests/test_c/test_drain_op.cpp
 )
 
+add_executable(
+  test_auto_parser
+  tests/test_c/test_auto_parser.cpp
+)
+
 pybind11_add_module(
   bind_class
   src/detectmateperformance/_core/bind_class.cpp
@@ -143,6 +154,8 @@ pybind11_add_module(
   src/detectmateperformance/_core/template_matcher/match_tree.cpp
   src/detectmateperformance/_core/parsers/drain.h
   src/detectmateperformance/_core/parsers/drain.cpp
+  src/detectmateperformance/_core/parsers/auto_parser.h
+  src/detectmateperformance/_core/parsers/auto_parser.cpp
 )
 
 target_link_libraries(
@@ -200,9 +213,24 @@ target_link_libraries(
   aux
 )
 
+target_link_libraries(
+  test_auto_parser
+  GTest::gtest_main
+  auto_parser_op
+  templates
+  parsedelement
+  parsed
+  tree
+  variable
+  tree_op
+  match_tree
+  aux
+)
+
 include(GoogleTest)
 gtest_discover_tests(test_c)
 gtest_discover_tests(test_type)
 gtest_discover_tests(test_matcher_tree)
 gtest_discover_tests(test_matcher_vars)
 gtest_discover_tests(test_drain_op)
+gtest_discover_tests(test_auto_parser)

docs/auto_parser.md

@@ -0,0 +1,36 @@
+
+# Auto Parser
+
+Check all the templates in the dataset and select the most fitting one.
+
+```python
+class AutoParse:
+    def __init__(self, num_use: int = 10) -> None:
+        pass
+
+    def __len__(self) -> int:
+       """Buffer size"""
+
+    def add(self, log: str) -> None:
+        """Add log to buffer"""
+
+    def reset(self) -> None:
+        """Reset train buffer"""
+
+    def generate(self) -> tuple[TreeMatcher, str]:
+        """Generate Tree matcher and a regex pattern"""
+
+    def __call__(self, logs: list[str] | pl.DataFrame | str) -> tuple[TreeMatcher, str]:
+        """Generate Tree matcher and a regex pattern from df"""
+```
+
+
+## Usage
+
+```python
+from detectmateperformance.auto_parser import AutoParse
+
+
+auto_parser = AutoParse(10)
+tree_matcher, regex = auto_parser("tests/test_data/audit.log")
+```

mkdocs.yml

@@ -9,6 +9,7 @@ nav:
   - Methods:
       - TreeMatcher: tree_matcher.md
       - Drain: drain.md
+      - AutoParser: auto_parser.md
   - Auxiliar:
       - Types: types.md
       - Metrics: metrics.md

pyproject.toml

@@ -3,7 +3,7 @@ name = "detectmateperformance"
 description = "A Python package for DetectMate performance evaluation."
 requires-python = ">=3.12"
 license = "EUPL-1.2"
-version="0.1.5"
+version="0.1.6"
 dependencies = [
     "levenshtein>=0.27.3",
     "numpy>=2.4.4",
@@ -41,4 +41,7 @@ include-package-data = true
 license-files = ["LICENSE.md"]
 
 [tool.setuptools.package-data]
-"*" = ["lib/bind_class.*"]
+"*" = [
+    "lib/bind_class.*",
+    "_templates/*.txt"
+]

src/detectmateperformance/_core/_type/templates.cpp

@@ -2,7 +2,9 @@
 #include <utility>
 #include <string>
 #include <deque>
+#include <regex>
 #include "templates.h"
+
 #include "../aux.h"
 
 
@@ -28,8 +30,15 @@ Templates::Templates(std::deque<std::string> templates) {
     resetCount();
 }
 
-Templates::Templates(std::string message) {
-    this->messages.push_back(te_preprocess(preprocessing(message)));
+Templates::Templates(std::string filename) {
+    std::regex pattern("<[^>]*>");
+    std::deque<std::string> templates = readFileToLines(filename);
+
+    for (const auto& message : templates) {
+        this->messages.push_back(te_preprocess(preprocessing(
+            std::regex_replace(message, pattern, "VAR")
+        )));
+    }
     resetCount();
 }
 

src/detectmateperformance/_core/_type/templates.h

@@ -15,7 +15,7 @@ class Templates {
 
 public:
     Templates(std::deque<std::string> templates);
-    Templates(std::string message);
+    Templates(std::string filename);
 
     ~Templates();
 

src/detectmateperformance/_core/aux.cpp

@@ -1,6 +1,20 @@
+#include <regex>
+#include <iostream>
+#include <fstream>
+#include <string>
+
 #include "aux.h"
 
 
+std::string clean_string(const std::string& input) {
+    std::regex pattern(
+        "[!\"#$%&'()*+,:;<=>?@\\[\\]^`{|}~\\s]|\\.(?![a-zA-Z0-9])|-(?![a-zA-Z0-9])"
+    );
+    std::string result = std::regex_replace(input, pattern, " ");
+
+    return result;
+}
+
 bool do_split(const char* str) {
     return *str == ' ';
 }
@@ -14,14 +28,11 @@ void remove_empty(std::deque<std::string>& words) {
 std::deque<std::string> preprocessing(std::string message) {
     std::deque<std::string> words;
 
+    message = clean_string(message);
     const char* start = message.data();
     const char* end = start;
 
     while (*end) {
-        if (std::ispunct(*end)) {
-            *const_cast<char*>(end) = ' ';
-        }
-
         if (do_split(end)) {
             words.emplace_back(start, end);
             start = end + 1;
@@ -34,3 +45,21 @@ std::deque<std::string> preprocessing(std::string message) {
 
     return words;
 }
+
+
+std::deque<std::string> readFileToLines(const std::string& filename) {
+    std::deque<std::string> lines;
+    std::ifstream file(filename);
+
+    if (!file.is_open()) {
+        std::cerr << "Error: Could not open file " << filename << std::endl;
+        return lines;
+    }
+
+    std::string line;
+    while (std::getline(file, line)) {
+        lines.push_back(line);
+    }
+
+    return lines;
+}

src/detectmateperformance/_core/aux.h

@@ -7,8 +7,9 @@
 #include <deque>
 #include <algorithm>
 
-std::deque<std::string> preprocessing(std::string message);
 
+std::deque<std::string> preprocessing(std::string message);
 
+std::deque<std::string> readFileToLines(const std::string& filename);
 
 #endif

src/detectmateperformance/_core/bind_class.cpp

@@ -2,6 +2,8 @@
 #include <pybind11/stl.h>
 
 #include "parsers/drain.h"
+#include "parsers/auto_parser.h"
+
 #include "template_matcher/match_tree.h"
 #include "_type/element.h"
 #include "_type/templates.h"
@@ -13,6 +15,7 @@ namespace py = pybind11;
 
 PYBIND11_MODULE(bind_class, m) {
     m.def("drain_generator", &drain_generator);
+    m.def("auto_parser", &doAutoParse);
 
     py::class_<Templates>(m, "Templates")
         .def(py::init<std::deque<std::string>>())

src/detectmateperformance/_core/parsers/auto_parser.cpp

@@ -0,0 +1,92 @@
+
+#include "auto_parser.h"
+
+#include <regex>
+
+
+std::vector<std::string> pathTemplates = {
+    "audit_templates.txt",
+    "bgl_templates.txt",
+    "dnsmasq_templates.txt",
+    "hdfs_templates.txt",
+    "openvpn.txt",
+    "syslog_templates.txt",
+    "thunderbird_templates.txt",
+};
+
+
+std::vector<const char*> regexs_patterns = {
+    R"(type=(\w+) msg=audit\(([^:]+):(\d+)\): (.*))",
+    R"((\S+) (\S+) (\S+) (\S+) (\S+) (\S+) (\S+) (\S+) (\S+) (.*))",
+    R"((\S+) (\S+) (\S+) (\S+) (.*))",
+    R"((\S+) (\S+) (\d+) (\S+) ([^:]+): (.*))",
+    R"((\S+) (\S+) (\S+) (.*))",
+    R"((\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+?)\[(\d+)\]:\s+(.*))",
+    R"((\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)(?:\[(\d+)\])?:\s+(.*))",
+};
+
+
+std::pair<Templates, int> autoParserGenerator(
+    std::vector<std::string> sentences,
+    std::vector<std::string> templatePaths,
+    std::vector<const char*> regexs
+){
+
+    int idx = 0;
+    int min_count = sentences.size();
+    for (size_t i = 0; i < templatePaths.size(); i++) {
+        // Initiliaze candidate
+        Templates* templates = new Templates(templatePaths[i]);
+        MatchTree* tree = new MatchTree(templates);
+
+        // Process format and parse logs
+        std::regex log_regex(regexs[i]);
+        std::smatch match_results;
+        std::vector<std::string> sentences_aux(sentences.size());
+        for (size_t j = 0; j < sentences.size(); j++) {
+            std::regex_search(sentences[j], match_results, log_regex);
+            sentences_aux[j] = match_results[match_results.size() - 1].str();
+        }
+        ParsedMessages* parsed_logs = tree->match_batch(sentences_aux, 1);
+
+        // Count template missmatchess
+        int not_found = 0;
+        for (size_t j = 0; j < sentences.size(); j++) {
+            ParsedElement elem = parsed_logs->getElem(i);
+            if (elem.log_template == "template not found" || sentences_aux[j] == "") {
+                not_found += 1;
+            }
+        }
+
+        // Make decision
+        if (not_found < min_count) {
+            idx = i;
+            min_count = not_found;
+        }
+
+        if (not_found == 0) {
+            break;
+        }
+
+    }
+
+    Templates templates(templatePaths[idx]);
+
+    return std::make_pair(templates, idx);
+}
+
+
+std::pair<Templates, int> doAutoParse(
+    std::vector<std::string> sentences,
+    std::string pathFolder
+) {
+    std::vector<std::string> pathsCopy(pathTemplates);
+
+    for (std::string& path : pathsCopy) {
+        path = pathFolder + path;
+    }
+
+    return autoParserGenerator(
+        sentences, pathsCopy, regexs_patterns
+    );
+}

src/detectmateperformance/_core/parsers/auto_parser.h

@@ -0,0 +1,24 @@
+#ifndef M_AUTOPARSER_H
+#define M_AUTOPARSER_H
+
+#include <vector>
+#include <string>
+
+#include "../_type/templates.h"
+#include "../template_matcher/match_tree.h"
+
+
+std::pair<Templates, int> autoParserGenerator(
+    std::vector<std::string> sentences,
+    std::vector<std::string> templatePaths,
+    std::vector<const char*> regexs
+);
+
+
+std::pair<Templates, int> doAutoParse(
+    std::vector<std::string> sentences,
+    std::string pathTemplates
+);
+
+
+#endif