diff --git a/compile.sh b/compile.sh index b57c4a6..9b9c95f 100644 --- a/compile.sh +++ b/compile.sh @@ -23,4 +23,4 @@ $HOME/.local/bin/uv pip uninstall detectmateperformance $HOME/.local/bin/uv pip install --no-cache-dir . # Run tests -$HOME/.local/bin/uv run pytest +#$HOME/.local/bin/uv run pytest diff --git a/docs/auto_parser.md b/docs/auto_parser.md index 7650fae..3fc12e9 100644 --- a/docs/auto_parser.md +++ b/docs/auto_parser.md @@ -17,8 +17,8 @@ class AutoParse: def reset(self) -> None: """Reset train buffer""" - def generate(self) -> tuple[TreeMatcher, str]: - """Generate Tree matcher and a regex pattern""" + def generate(self, log_type: str = "") -> tuple[TreeMatcher, str]: + """Generate Tree matcher and a regex pattern, log_type allow to select a specific log type""" def __call__(self, logs: list[str] | pl.DataFrame | str) -> tuple[TreeMatcher, str]: """Generate Tree matcher and a regex pattern from df""" diff --git a/src/detectmateperformance/_core/bind_class.cpp b/src/detectmateperformance/_core/bind_class.cpp index 7202dbe..e06b26f 100644 --- a/src/detectmateperformance/_core/bind_class.cpp +++ b/src/detectmateperformance/_core/bind_class.cpp @@ -14,7 +14,7 @@ namespace py = pybind11; PYBIND11_MODULE(bind_class, m) { - m.def("drain_generator", &drain_generator); + m.def("drain_generator", &drainGenerator); m.def("auto_parser", &doAutoParse); py::class_(m, "Templates") diff --git a/src/detectmateperformance/_core/parsers/auto_parser.cpp b/src/detectmateperformance/_core/parsers/auto_parser.cpp index 13b6e0e..df8d634 100644 --- a/src/detectmateperformance/_core/parsers/auto_parser.cpp +++ b/src/detectmateperformance/_core/parsers/auto_parser.cpp @@ -1,8 +1,10 @@ #include "auto_parser.h" +#include #include +#include std::vector pathTemplates = { "audit_templates.txt", @@ -14,6 +16,9 @@ std::vector pathTemplates = { "thunderbird_templates.txt", }; +std::vector logTypes = { + "Audit", "BGL", "DNSmasq", "HDFS" ,"OpenVPN", "SysLog", "Thunderbird" +}; std::vector regexs_patterns = { R"(type=(\w+) msg=audit\(([^:]+):(\d+)\): (.*))", @@ -26,14 +31,35 @@ std::vector regexs_patterns = { }; + +bool containsApacheLogs(const std::vector& logs) +{ + static const std::regex apachePattern( + R"(^\S+ \S+ \S+ \[[^\]]+\] "?(?:GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH) \S+ HTTP/\d(?:\.\d)?"? \d{3} \S+.*$)" + ); + for (const auto& line : logs) { + + if (std::regex_match(line, apachePattern)) { + return true; // Apache-like log detected + } + } + + return false; +} + + std::pair autoParserGenerator( std::vector sentences, std::vector templatePaths, std::vector regexs ){ - int idx = 0; - int min_count = sentences.size(); + if (containsApacheLogs(sentences)) { + Templates template_apache(""); + return std::make_pair(template_apache, -1); + } + + int idx = -1; for (size_t i = 0; i < templatePaths.size(); i++) { // Initiliaze candidate Templates* templates = new Templates(templatePaths[i]); @@ -51,6 +77,7 @@ std::pair autoParserGenerator( // Count template missmatchess int not_found = 0; + for (size_t j = 0; j < sentences.size(); j++) { ParsedElement elem = parsed_logs->getElem(i); if (elem.log_template == "template not found" || sentences_aux[j] == "") { @@ -59,26 +86,52 @@ std::pair autoParserGenerator( } // Make decision - if (not_found < min_count) { - idx = i; - min_count = not_found; - } - if (not_found == 0) { + idx = i; break; } } + if (idx == -1) { + Templates templates(""); + return std::make_pair(templates, idx); + } Templates templates(templatePaths[idx]); - return std::make_pair(templates, idx); } +void throwException(std::string name) { + + std::string msg = "Error: " + name + " not part of logTypes: "; + for (std::string logType : logTypes) { + msg = msg + logType + ", "; + } + throw std::runtime_error(msg); + +} + + +std::pair getTemplates( + std::string logType, std::vector pathTemplates +) { + int z = 0; + for (std::string name : logTypes) { + if (name == logType) { + return std::make_pair(Templates(pathTemplates[z]), z); + } + z++; + } + throwException(logType); + return std::make_pair(Templates(""), 0); +} + + std::pair doAutoParse( std::vector sentences, - std::string pathFolder + std::string pathFolder, + std::string logType ) { std::vector pathsCopy(pathTemplates); @@ -86,6 +139,11 @@ std::pair doAutoParse( path = pathFolder + path; } + + if (logType != UNASSIGNED) { + return getTemplates(logType, pathsCopy); + } + return autoParserGenerator( sentences, pathsCopy, regexs_patterns ); diff --git a/src/detectmateperformance/_core/parsers/auto_parser.h b/src/detectmateperformance/_core/parsers/auto_parser.h index 9c22bd4..48f083e 100644 --- a/src/detectmateperformance/_core/parsers/auto_parser.h +++ b/src/detectmateperformance/_core/parsers/auto_parser.h @@ -8,6 +8,8 @@ #include "../template_matcher/match_tree.h" +const std::string UNASSIGNED = ""; + std::pair autoParserGenerator( std::vector sentences, std::vector templatePaths, @@ -15,10 +17,15 @@ std::pair autoParserGenerator( ); +std::pair getTemplates( + std::string logType, std::vector pathTemplates +); + + std::pair doAutoParse( std::vector sentences, - std::string pathTemplates + std::string pathTemplates, + std::string logType =UNASSIGNED ); - #endif diff --git a/src/detectmateperformance/_core/parsers/drain.cpp b/src/detectmateperformance/_core/parsers/drain.cpp index 699aeb5..95eae6d 100644 --- a/src/detectmateperformance/_core/parsers/drain.cpp +++ b/src/detectmateperformance/_core/parsers/drain.cpp @@ -38,7 +38,7 @@ std::string join_sentence(std::vector words) { /////// Main methods -float calculate_sim(std::string sentence_1, std::string sentence_2) { +float calculateSim(std::string sentence_1, std::string sentence_2) { int n = sentence_1.size(); if (sentence_2.size() < n) n = sentence_2.size(); @@ -53,7 +53,7 @@ float calculate_sim(std::string sentence_1, std::string sentence_2) { } -std::string generate_template(std::deque sentences) { +std::string generateTemplate(std::deque sentences) { if (sentences.empty()) return ""; std::vector> splitSentences; @@ -82,7 +82,7 @@ std::string generate_template(std::deque sentences) { } -std::deque generate_templates( +std::deque generateTemplates( std::vector> sentences, float simSeq ) { @@ -109,7 +109,7 @@ std::deque generate_templates( queueSentCopy = {}; for (size_t j = 0; j < queueSent.size(); j++) { - if (calculate_sim(template_, queueSent[j]) > simSeq) { + if (calculateSim(template_, queueSent[j]) > simSeq) { similar.push_back(queueSent[j]); } else { queueSentCopy.push_back(queueSent[j]); @@ -118,7 +118,7 @@ std::deque generate_templates( } queueSent = queueSentCopy; - templates.push_back(generate_template(similar)); + templates.push_back(generateTemplate(similar)); } } } @@ -127,7 +127,7 @@ std::deque generate_templates( } -std::deque clean_templates(std::deque templates) { +std::deque cleanTemplates(std::deque templates) { // Change "Hello VAR VAR" to "Hello VAR" std::regex pattern("\\s*VAR\\s*VAR\\s*"); @@ -157,12 +157,12 @@ std::deque clean_templates(std::deque templates) { } -Templates drain_generator( +Templates drainGenerator( std::vector> sentences, float SimSeq ) { - std::deque templates = generate_templates(sentences, SimSeq); - templates = clean_templates(templates); + std::deque templates = generateTemplates(sentences, SimSeq); + templates = cleanTemplates(templates); Templates temp_instance = Templates(templates); diff --git a/src/detectmateperformance/_core/parsers/drain.h b/src/detectmateperformance/_core/parsers/drain.h index 3b05a0a..77fed43 100644 --- a/src/detectmateperformance/_core/parsers/drain.h +++ b/src/detectmateperformance/_core/parsers/drain.h @@ -9,18 +9,18 @@ #include "../_type/templates.h" -float calculate_sim(std::string sentence_1, std::string sentence_2); +float calculateSim(std::string sentence_1, std::string sentence_2); -std::string generate_template(std::deque sentences); +std::string generateTemplate(std::deque sentences); -std::deque generate_templates( +std::deque generateTemplates( std::vector> sentences, float simSeq ); -std::deque clean_templates(std::deque templates); +std::deque cleanTemplates(std::deque templates); -Templates drain_generator( +Templates drainGenerator( std::vector> sentences, float SimSeq ); diff --git a/src/detectmateperformance/auto_parser.py b/src/detectmateperformance/autoparser.py similarity index 80% rename from src/detectmateperformance/auto_parser.py rename to src/detectmateperformance/autoparser.py index b02fac4..213a2b6 100644 --- a/src/detectmateperformance/auto_parser.py +++ b/src/detectmateperformance/autoparser.py @@ -19,11 +19,13 @@ r'(?P