fabian/actsim-cluster-agent
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

implement remaining failure modes and optimize reference data pull

Browse source
This commit is contained in:
Fabian Posch 2025-01-08 16:53:16 +01:00
parent fe9eaeb8f6
commit 0a3a235575
5 changed files with 150 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
include/actsim_agent/log_parser.hpp
View file

@ -27,6 +27,7 @@
#define __LOG_PARSER__
#include <cluster/artifact.hpp>
#include <cstdint>
#include <vector>
#include "agent_artifact.hpp"
@ -44,23 +45,24 @@ class LogParser {
private:
using log_it_t = std::vector<std::string>::const_iterator;
using timing_it_t = std::vector<uint32_t>::const_iterator;
bool compare_logs(log_it_t& iterator, const std::string& line);
void check_token_count(const std::string& line);
void check_value_fault(const std::string& line);
void check_value_timing_fault(const std::string& line);
void check_coding_fault(const std::string& line);
void check_glitch(const std::string& line);
uint32_t extract_timestamp(const std::string& line);
void handle_output_token(const std::string& line);
std::unique_ptr<DBSimOutputArtifact>& artifact;
std::shared_ptr<pl::SimOutputArtifact> reference;
bool has_reference;
log_it_t log_it;
log_it_t err_it;
timing_it_t timing_it;
timing_it_t reference_ott_end;
bool logs_diverge = false;
bool err_diverge = false;
bool failure_mode = false;
int dut_output_tokens_ = 0;
int output_token_difference_ = 0;

60
src/actsim_agent/downloader.cpp
View file

@ -23,6 +23,8 @@
**************************************************************************
*/
#include <cstddef>
#include <cstdint>
#include <string>
#include <cluster/artifact.hpp>
#include <filesystem>
@ -30,7 +32,7 @@
#include <cstdlib>
#include <pqxx/pqxx>
#include <functional>
#include <chrono>
#include <vector>
#include "util.h"
#include "downloader.hpp"
@ -186,15 +188,8 @@ bool Downloader::fetch_tasks(size_t n) {
}
*source_config = row["source_config"].as<db::uuid_t>();
std::vector<std::string> commands;
auto arr = row["sim_commands"].as_array();
std::pair<pqxx::array_parser::juncture, std::string> elem;
do {
elem = arr.get_next();
if (elem.first == pqxx::array_parser::juncture::string_value) {
commands.push_back(elem.second);
}
} while (elem.first != pqxx::array_parser::juncture::done);
auto com_arr = row["sim_commands"].as_sql_array<std::string>();
std::vector<std::string> commands(com_arr.cbegin(), com_arr.cend());
*testcase = {
commands,
@ -379,36 +374,21 @@ std::shared_ptr<pl::SimOutputArtifact> Downloader::fetch_reference_run(const db:
DEBUG_PRINT("Loading reference run with ID " + db::to_string(id) + " from database.");
auto fetch_design_lambda = [id](
pqxx::work *txn, std::vector<std::string> *sim_log,
std::vector<std::string> *error_log,
pqxx::work *txn,
std::vector<uint32_t> *output_token_timings,
long *output_tokens,
long *log_size,
bool *found
) {
try {
auto res = txn->exec_params1("SELECT sim_log, error_log, output_tokens FROM sim_outputs WHERE sim_config = $1;", id);
auto res = txn->exec_params1("SELECT output_tokens, output_token_timings, log_size FROM sim_outputs WHERE sim_config = $1;", id);
// load sim_log into string vector
*sim_log = std::vector<std::string>();
auto arr_l = res["sim_log"].as_array();
std::pair<pqxx::array_parser::juncture, std::string> elem;
do {
elem = arr_l.get_next();
if (elem.first == pqxx::array_parser::juncture::string_value) {
(*sim_log).push_back(elem.second);
}
} while (elem.first != pqxx::array_parser::juncture::done);
// parse the error log into string vector
*error_log = std::vector<std::string>();
auto arr_e = res["error_log"].as_array();
do {
elem = arr_e.get_next();
if (elem.first == pqxx::array_parser::juncture::string_value) {
(*error_log).push_back(elem.second);
}
} while (elem.first != pqxx::array_parser::juncture::done);
// load the output token timings
auto arr_ott = res["output_token_timings"].as_sql_array<uint32_t>();
*output_token_timings = std::vector<uint32_t>(arr_ott.cbegin(), arr_ott.cend());
*output_tokens = res["output_tokens"].as<long>();
*log_size = res["log_size"].as<long>();
*found = true;
@ -420,18 +400,18 @@ std::shared_ptr<pl::SimOutputArtifact> Downloader::fetch_reference_run(const db:
std::function<void(
pqxx::work *txn,
std::vector<std::string> *sim_log,
std::vector<std::string> *error_log,
std::vector<uint32_t> *output_token_timings,
long *output_tokens,
long *log_size,
bool *found
)> fetch_design_func = fetch_design_lambda;
std::vector<std::string> sim_log;
std::vector<std::string> error_log;
std::vector<uint32_t> output_token_timings;
long output_tokens;
long log_size;
bool ref_run_found;
if (!this->conn->send_request(&fetch_design_func, &sim_log, &error_log, &output_tokens, &ref_run_found)) {
if (!this->conn->send_request(&fetch_design_func, &output_token_timings, &output_tokens, &log_size, &ref_run_found)) {
// if we lost connection, there's nothing else we can really do
std::cerr << "Error: Lost connection while trying fetch a design! Aborting!" << std::endl;
this->interface.stop();
@ -443,8 +423,10 @@ std::shared_ptr<pl::SimOutputArtifact> Downloader::fetch_reference_run(const db:
return nullptr;
}
auto reference = std::make_shared<pl::SimOutputArtifact>(sim_log, error_log);
auto reference = std::make_shared<pl::SimOutputArtifact>();
reference->set_output_token_timings(output_token_timings);
reference->set_output_tokens(output_tokens);
reference->set_size(log_size);
return reference;
}

139
src/actsim_agent/log_parser.cpp
View file

@ -23,18 +23,20 @@
**************************************************************************
*/
#include <iostream>
#include <string>
#define DEBUG
#include <regex>
#include "util.h"
#include "log_parser.hpp"
#include <cstdint>
#include <vector>
#include <string>
LogParser::LogParser(std::unique_ptr<DBSimOutputArtifact>& artifact, std::shared_ptr<pl::SimOutputArtifact> reference) : artifact(artifact) {
this->reference = reference;
this->has_reference = true;
this->log_it = reference->get_content().first.begin();
this->err_it = reference->get_content().second.begin();
// get the output token timing iterator
this->timing_it = reference->get_output_token_timings().begin();
this->reference_ott_end = reference->get_output_token_timings().end();
}
LogParser::LogParser(std::unique_ptr<DBSimOutputArtifact>& artifact) : artifact(artifact) {
@ -43,6 +45,8 @@ LogParser::LogParser(std::unique_ptr<DBSimOutputArtifact>& artifact) : artifact(
void LogParser::parse_log(const std::string& line) {
// DEBUG_PRINT("Parsing log line " + line);
// check for output tokens
check_token_count(line);
@ -50,47 +54,23 @@ void LogParser::parse_log(const std::string& line) {
check_coding_fault(line);
// check for value fault
check_value_fault(line);
// if there is no reference, just add the log line and return
if (!this->has_reference || logs_diverge) {
this->artifact->add_log_output(line);
return;
}
// if logs are the same, there is nothing to do
if (compare_logs(log_it, line)) {
return;
}
check_value_timing_fault(line);
// check for glitch
check_glitch(line);
// the logs have diverged
logs_diverge = true;
this->artifact->add_log_output(line);
}
void LogParser::parse_error(const std::string& line) {
// if there is no reference, just add the log line and return
if (!this->has_reference || err_diverge) {
this->artifact->add_err_output(line);
return;
}
// DEBUG_PRINT("Parsing error line " + line);
// actsim actually outputs everything on stdout
// Only the warnings in the beginning are on stderr.
// // if logs are the same, there is nothing to do
// if (compare_logs(err_it, line)) {
// return;
// }
// // the logs have diverged
// err_diverge = true;
// this->artifact->add_err_output(line);
this->artifact->add_err_output(line);
}
@ -116,6 +96,7 @@ void LogParser::finalize() {
// a deadlock must have occured
artifact->set_fault_deadlock(true);
failure_mode = true;
DEBUG_PRINT("Deadlock detected during finalization (compared to reference)");
DEBUG_PRINT("Reference had " +
std::to_string(reference->output_tokens) +
@ -128,10 +109,17 @@ void LogParser::finalize() {
// a token amount error has occurred
artifact->set_fault_token_count(true);
failure_mode = true;
DEBUG_PRINT("Token count mismatch detected during finalization (compared to reference)");
}
// if there is no failure condition,
// we don't need to save the log
if (!failure_mode) {
artifact->clear_logs();
}
} else {
// if token difference != 0: token count error
@ -140,7 +128,6 @@ void LogParser::finalize() {
artifact->set_output_tokens(dut_output_tokens_);
DEBUG_PRINT("Token count mismatch detected during finalization.");
}
}
}
@ -162,7 +149,7 @@ inline void LogParser::check_token_count(const std::string& line) {
}
}
inline void LogParser::check_value_fault(const std::string& line) {
inline void LogParser::check_value_timing_fault(const std::string& line) {
// simply check if the standard test failed output
// is given by the scoreboard
@ -171,11 +158,55 @@ inline void LogParser::check_value_fault(const std::string& line) {
if (line.find(test_failed) != std::string::npos) {
artifact->set_fault_value(true);
failure_mode = true;
DEBUG_PRINT("Value error detected");
handle_output_token(line);
return;
}
// if we passed the test we still need to check for timing issues
const std::string test_succeeded = "TEST SUCCESS";
if (line.find(test_succeeded) != std::string::npos) {
// DEBUG_PRINT("Successful output token detected");
handle_output_token(line);
return;
}
}
void LogParser::check_coding_fault(const std::string& line) {
inline void LogParser::handle_output_token(const std::string& line) {
// add the timing to the artifact
auto timing = extract_timestamp(line);
artifact->add_output_token_timing(timing);
// check if there is also a timing error
if (has_reference) {
// make sure there is still a token to compare to left
if (timing_it == reference_ott_end) {
// there is a mismatch in tokens
artifact->set_fault_token_count(true);
failure_mode = true;
DEBUG_PRINT("Tried to compare token timing but no reference token left.");
return;
}
// check if the timings align
if (timing != *timing_it) {
// timings don't line up!
artifact->set_fault_timing_deviation(true);
failure_mode = true;
DEBUG_PRINT("Token timing does not line up.");
}
// increment the iterator
++timing_it;
}
}
inline void LogParser::check_coding_fault(const std::string& line) {
// check for actsim's excl-hi warning
@ -183,6 +214,7 @@ void LogParser::check_coding_fault(const std::string& line) {
if (line.find(excl_hi_violated) != std::string::npos) {
artifact->set_fault_coding(true);
failure_mode = true;
DEBUG_PRINT("Excl-hi constraint violated");
}
}
@ -198,13 +230,40 @@ bool LogParser::check_busy_deadlock() {
}
if (artifact->get_size() > (reference->get_size() * 3)) {
failure_mode = true;
DEBUG_PRINT("Busy deadlock detected, reference size is " +
std::to_string(reference->get_size()) +
", ours is " + std::to_string(artifact->get_size())
);
return true;
DEBUG_PRINT("Busy deadlock detected");
}
return false;
}
bool LogParser::compare_logs([[maybe_unused]]log_it_t& it, [[maybe_unused]]const std::string& line) {
return false;
inline void LogParser::check_glitch(const std::string& line) {
// simply check if the standard glitch output is given
const std::string glitch_detected = "Glitch detected in channel";
if (line.find(glitch_detected) != std::string::npos) {
artifact->set_fault_glitch(true);
failure_mode = true;
DEBUG_PRINT("Glitch in interface detected");
}
}
inline uint32_t LogParser::extract_timestamp(const std::string& line) {
// regex match the timestamp format
std::regex pattern(R"(^\[\s*(\d+)\s*\])");
std::smatch match;
if (std::regex_search(line, match, pattern)) {
return std::stoi(match[1].str());
} else {
return 0;
}
}

19
src/actsim_agent/uploader.cpp
View file

@ -123,12 +123,21 @@ bool Uploader::upload_task(std::unique_ptr<OutputType> task) {
auto&& task_id = task->id;
auto&& sim_log = task->get_content().first;
auto&& sim_error = task->get_content().second;
auto&& output_token_timings = task->get_output_token_timings();
auto&& output_tokens = task->output_tokens;
auto log_size = sim_log.size() + sim_error.size();
const auto&& fault_flags = build_fault_flags(task);
// make sure any task that is uploaded isn't halted in the database
auto task_upload_lambda = [task_id, sim_log, sim_error, output_tokens, fault_flags](
auto task_upload_lambda = [ task_id,
sim_log,
sim_error,
output_tokens,
output_token_timings,
fault_flags,
log_size
](
pqxx::work *txn
) {
txn->exec_params0(
@ -136,13 +145,17 @@ bool Uploader::upload_task(std::unique_ptr<OutputType> task) {
" sim_log = $1, "
" error_log = $2, "
" output_tokens = $3, "
" fault_flags = $4, "
" output_token_timings = $4, "
" fault_flags = $5, "
" log_size = $6, "
" part_status = 'finished' "
"WHERE id = $5 AND part_status != 'halted';",
"WHERE id = $7 AND part_status != 'halted';",
sim_log,
sim_error,
output_tokens,
output_token_timings,
fault_flags,
log_size,
task_id
);
};

6
src/actsim_agent/worker.cpp
View file

@ -291,8 +291,12 @@ std::unique_ptr<OutputType> Worker::perform_task(std::unique_ptr<InputType>& tas
auto bin_str = std::string(std::getenv("ACT_HOME")) + "/bin/actsim";
char* bin = new char[bin_str.length() + 1];
std::strcpy(bin, bin_str.c_str());
std::string arg_str = "-m";
char* arg = new char[arg_str.length() + 1];
std::strcpy(arg, arg_str.c_str());
char* const argv[] = {bin, design_char, top_proc_char, (char*)0};
char* const argv[] = {bin, arg, design_char, top_proc_char, (char*)0};
// and call actsim
execv(argv[0], argv);