diff --git a/include/actsim_agent/log_parser.hpp b/include/actsim_agent/log_parser.hpp index 1018ef2..0b4cb16 100644 --- a/include/actsim_agent/log_parser.hpp +++ b/include/actsim_agent/log_parser.hpp @@ -27,6 +27,7 @@ #define __LOG_PARSER__ #include +#include #include "agent_artifact.hpp" class LogParser { @@ -38,13 +39,31 @@ class LogParser { void parse_log(const std::string& line); void parse_error(const std::string& line); + void finalize(); + bool check_busy_deadlock(); private: + using log_it_t = std::vector::const_iterator; + + bool compare_logs(log_it_t& iterator, const std::string& line); + void check_token_count(const std::string& line); + void check_value_fault(const std::string& line); + void check_coding_fault(const std::string& line); + std::unique_ptr& artifact; std::shared_ptr reference; bool has_reference; + + log_it_t log_it; + log_it_t err_it; + + bool logs_diverge = false; + bool err_diverge = false; + + int dut_output_tokens_ = 0; + int output_token_difference_ = 0; }; #endif diff --git a/src/actsim_agent/log_parser.cpp b/src/actsim_agent/log_parser.cpp index 3df8eef..7ab4195 100644 --- a/src/actsim_agent/log_parser.cpp +++ b/src/actsim_agent/log_parser.cpp @@ -23,11 +23,18 @@ ************************************************************************** */ +#include +#include +#define DEBUG + +#include "util.h" #include "log_parser.hpp" LogParser::LogParser(std::unique_ptr& artifact, std::shared_ptr reference) : artifact(artifact) { this->reference = reference; this->has_reference = true; + this->log_it = reference->get_content().first.begin(); + this->err_it = reference->get_content().second.begin(); } LogParser::LogParser(std::unique_ptr& artifact) : artifact(artifact) { @@ -35,9 +42,169 @@ LogParser::LogParser(std::unique_ptr& artifact) : artifact( }; void LogParser::parse_log(const std::string& line) { + + // check for output tokens + check_token_count(line); + + // check for excl high constraint violations + check_coding_fault(line); + + // check for value fault + check_value_fault(line); + + // if there is no reference, just add the log line and return + if (!this->has_reference || logs_diverge) { + this->artifact->add_log_output(line); + return; + } + + // if logs are the same, there is nothing to do + if (compare_logs(log_it, line)) { + return; + } + + // check for glitch + + + // the logs have diverged + logs_diverge = true; this->artifact->add_log_output(line); + } void LogParser::parse_error(const std::string& line) { - this->artifact->add_err_output(line); + + // if there is no reference, just add the log line and return + if (!this->has_reference || err_diverge) { + this->artifact->add_err_output(line); + return; + } + + // actsim actually outputs everything on stdout + // Only the warnings in the beginning are on stderr. + + // // if logs are the same, there is nothing to do + // if (compare_logs(err_it, line)) { + // return; + // } + + // // the logs have diverged + // err_diverge = true; + // this->artifact->add_err_output(line); + +} + +void LogParser::finalize() { + + /* + So the only way to do this cleanly is to make sure that either + - the model has not sent all tokens yet -> deadlock + - or the model has sent everything but there is one missing -> token fault + + This has the consequence that we cannot inject close to the end of the test! + + This also means that we don't really know if a deadlock occurred without + a reference run to go off. + + Only that there was a potential token count difference. + */ + + if (has_reference) { + + // model has not sent all tokens yet + if ((dut_output_tokens_ + output_token_difference_) < reference->output_tokens) { + + // a deadlock must have occured + artifact->set_fault_deadlock(true); + DEBUG_PRINT("Deadlock detected during finalization (compared to reference)"); + DEBUG_PRINT("Reference had " + + std::to_string(reference->output_tokens) + + " tokens, task had " + + std::to_string(dut_output_tokens_) + + " + " + std::to_string(output_token_difference_)); + + // model has sent all tokens but DUT has not sent all of them + } else if (output_token_difference_ != 0) { + + // a token amount error has occurred + artifact->set_fault_token_count(true); + DEBUG_PRINT("Token count mismatch detected during finalization (compared to reference)"); + + } + + } else { + + // if token difference != 0: token count error + if (output_token_difference_ != 0) { + artifact->set_fault_token_count(true); + artifact->set_output_tokens(dut_output_tokens_); + DEBUG_PRINT("Token count mismatch detected during finalization."); + } + + } +} + +inline void LogParser::check_token_count(const std::string& line) { + + // difference counter should be back at 0 when log is finished + // -> both model and DUT have emitted the same number of tokens + + const std::string model_token = "Model response received"; + const std::string dut_token = "DUT response received"; + + if (line.find(model_token) != std::string::npos) { + ++output_token_difference_; + } + + if (line.find(dut_token) != std::string::npos) { + --output_token_difference_; + ++dut_output_tokens_; + } +} + +inline void LogParser::check_value_fault(const std::string& line) { + + // simply check if the standard test failed output + // is given by the scoreboard + + const std::string test_failed = "TEST FAILED"; + + if (line.find(test_failed) != std::string::npos) { + artifact->set_fault_value(true); + DEBUG_PRINT("Value error detected"); + } +} + +void LogParser::check_coding_fault(const std::string& line) { + + // check for actsim's excl-hi warning + + const std::string excl_hi_violated = "WARNING: excl-hi constraint in"; + + if (line.find(excl_hi_violated) != std::string::npos) { + artifact->set_fault_coding(true); + DEBUG_PRINT("Excl-hi constraint violated"); + } +} + +bool LogParser::check_busy_deadlock() { + + // we allow for more than 3x the events to happen compared + // to the reference run, then we assume a deadlock + + // if there is no reference, we have nothing to do + if (!has_reference) { + return false; + } + + if (artifact->get_size() > (reference->get_size() * 3)) { + return true; + DEBUG_PRINT("Busy deadlock detected"); + } + + return false; +} + +bool LogParser::compare_logs([[maybe_unused]]log_it_t& it, [[maybe_unused]]const std::string& line) { + return false; }