fabian/actsim-cluster-agent
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

final performant version

Browse source
This commit is contained in:
Fabian Posch 2025-02-02 16:33:52 +01:00
parent 5786b7bf22
commit 7dcadc1ca2
9 changed files with 176 additions and 237 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/actsim_agent/downloader.hpp
View file

@ -31,8 +31,6 @@
#include <cluster/db_client.hpp> #include <cluster/db_client.hpp>
#include "task_interface.hpp" #include "task_interface.hpp"
#define NOTHING_AVAILABLE_SLEEP_TIME 500ms
class Downloader { class Downloader {
public: public:

14
include/actsim_agent/task_interface.hpp
View file

@ -26,6 +26,7 @@
#ifndef __TASK_INTERFACE__ #ifndef __TASK_INTERFACE__
#define __TASK_INTERFACE__ #define __TASK_INTERFACE__
#include <cstddef>
#include <queue> #include <queue>
#include <unordered_map> #include <unordered_map>
#include <atomic> #include <atomic>
@ -33,6 +34,7 @@
#include <mutex> #include <mutex>
#include <vector> #include <vector>
#include "agent_artifact.hpp" #include "agent_artifact.hpp"
#include <chrono>
/** /**
* If you want to use this interface for different types, you only need to change it here. * If you want to use this interface for different types, you only need to change it here.
@ -41,6 +43,8 @@
using InputType = DBSimConfigArtifact; using InputType = DBSimConfigArtifact;
using OutputType = DBSimOutputArtifact; using OutputType = DBSimOutputArtifact;
#define NOTHING_AVAILABLE_SLEEP_TIME 500ms
class TaskInterface { class TaskInterface {
public: public:
@ -49,10 +53,11 @@ class TaskInterface {
~TaskInterface(); ~TaskInterface();
void wait_for_fresh(); void wait_for_fresh();
void wait_for_finished(); void wait_for_finished(size_t min_size);
void wait_for_buffer_consume(); void wait_for_buffer_consume();
void wait_for_cleanup_ready(); void wait_for_cleanup_ready();
void wait_for_download_halt(); void wait_for_download_halt();
void wait_for_available();
void notify_cleanup_ready(); void notify_cleanup_ready();
void notify_workers_program_halt(); void notify_workers_program_halt();
@ -62,7 +67,7 @@ class TaskInterface {
void push_fresh(std::vector<std::unique_ptr<InputType>>& tasks); void push_fresh(std::vector<std::unique_ptr<InputType>>& tasks);
std::unique_ptr<InputType> pop_fresh(bool& empty); std::unique_ptr<InputType> pop_fresh(bool& empty);
void push_finished(std::unique_ptr<OutputType> task); void push_finished(std::unique_ptr<OutputType> task);
std::unique_ptr<OutputType> pop_finished(bool& empty); std::vector<std::unique_ptr<OutputType>> pop_finished(bool& empty);
size_t get_buffer_space(); size_t get_buffer_space();
/* /*
@ -87,7 +92,7 @@ class TaskInterface {
void stop_immediately() { this->immediate_stop.store(true, std::memory_order_relaxed); }; void stop_immediately() { this->immediate_stop.store(true, std::memory_order_relaxed); };
bool fresh_queue_empty(); bool fresh_queue_empty();
bool finished_queue_empty(); bool finished_queue_empty(size_t min_size);
private: private:
@ -102,6 +107,9 @@ class TaskInterface {
std::unordered_map<db::uuid_t, std::pair<size_t, std::string>> designs; std::unordered_map<db::uuid_t, std::pair<size_t, std::string>> designs;
std::unordered_map<db::uuid_t, std::pair<size_t, std::shared_ptr<pl::SimOutputArtifact>>> references; std::unordered_map<db::uuid_t, std::pair<size_t, std::shared_ptr<pl::SimOutputArtifact>>> references;
bool worker_waiting = false;
bool downloader_waiting = false;
////// Mutexes ////// ////// Mutexes //////
// access to task queues // access to task queues

5
include/actsim_agent/uploader.hpp
View file

@ -32,6 +32,8 @@
#include <thread> #include <thread>
#include "task_interface.hpp" #include "task_interface.hpp"
std::string build_fault_flags(const std::unique_ptr<OutputType>& task);
class Uploader { class Uploader {
public: public:
@ -44,8 +46,7 @@ class Uploader {
private: private:
void thread_run(); void thread_run();
bool upload_task(std::unique_ptr<OutputType> task); bool upload_task(std::vector<std::unique_ptr<OutputType>>& tasks);
std::string build_fault_flags(const std::unique_ptr<OutputType>& task);
std::unique_ptr<std::thread> uploader_thread; std::unique_ptr<std::thread> uploader_thread;
std::unique_ptr<db::Connection> conn; std::unique_ptr<db::Connection> conn;

26
include/cluster/db_client.hpp
View file

@ -26,6 +26,7 @@
#define __DB_CLIENT_H__ #define __DB_CLIENT_H__
#include <cstddef>
#include <cstdint> #include <cstdint>
#include <iostream> #include <iostream>
#include <pqxx/pqxx> #include <pqxx/pqxx>
@ -69,6 +70,31 @@ class Connection {
bool prepare_statement(std::string name, std::string statement); bool prepare_statement(std::string name, std::string statement);
bool unprepare_statement(std::string name); bool unprepare_statement(std::string name);
pqxx::work open_transaction(bool& success) {
retry_send:
if (!ensure_connected()) {
cerr << "Could contact database. Broken database connection." << endl;
success = false;
}
try {
return pqxx::work(*c);
} catch (const pqxx::broken_connection& e) {
cerr << "Pipe broke during database operation... Retrying..." << endl;
cerr << e.what() << endl;
// using a goto here since it'll gracefully retry connection
goto retry_send;
}
success = true;
}
// Send request takes an arbitrary request method to the database and wraps error handling // Send request takes an arbitrary request method to the database and wraps error handling
// around it. It takes a function with an arbitrary set of arguments and the arguments to give // around it. It takes a function with an arbitrary set of arguments and the arguments to give
// that function and returns whatever the function returns. // that function and returns whatever the function returns.

1
src/actsim_agent/CMakeLists.txt
View file

@ -1,5 +1,4 @@
include_directories(../../include) include_directories(../../include)
include_directories(../../include/db_lib)
include_directories(../../include/actsim_agent) include_directories(../../include/actsim_agent)
include_directories($ENV{ACT_HOME}/include) include_directories($ENV{ACT_HOME}/include)
include_directories(../../deps/libpqxx/include) include_directories(../../deps/libpqxx/include)

228
src/actsim_agent/downloader.cpp
View file

@ -23,6 +23,7 @@
************************************************************************** **************************************************************************
*/ */
#include "../../include/cluster/db_client.hpp"
#include <cstddef> #include <cstddef>
#include <cstdint> #include <cstdint>
#include <iostream> #include <iostream>
@ -34,11 +35,11 @@
#include <cstdlib> #include <cstdlib>
#include <pqxx/pqxx> #include <pqxx/pqxx>
#include <functional> #include <functional>
#include <unordered_set>
#include <vector> #include <vector>
#include "actsim_agent.hpp" #include "actsim_agent.hpp"
#include "agent_artifact.hpp" #include "agent_artifact.hpp"
#include "cluster/db_types.hpp" #include "cluster/db_types.hpp"
#include "pqxx/internal/statement_parameters.hxx"
#include "pqxx/prepared_statement.hxx" #include "pqxx/prepared_statement.hxx"
#include "task_interface.hpp" #include "task_interface.hpp"
#include "util.h" #include "util.h"
@ -62,8 +63,6 @@ void Downloader::join() {
void Downloader::thread_run() { void Downloader::thread_run() {
using namespace std::chrono_literals;
// connect to the database // connect to the database
if (!this->conn->connect()) { if (!this->conn->connect()) {
std::cerr << "Error: Upload thread could not connect to the database!" << std::endl; std::cerr << "Error: Upload thread could not connect to the database!" << std::endl;
@ -72,6 +71,10 @@ void Downloader::thread_run() {
return; return;
} }
this->conn->prepare_statement("mark_tasks",
"UPDATE sim_configs SET part_status = 'finished' WHERE id = $1;"
);
while (this->interface.running()) { while (this->interface.running()) {
// wait until there is more space in the buffer to fill // wait until there is more space in the buffer to fill
@ -86,7 +89,7 @@ void Downloader::thread_run() {
if (!this->fetch_tasks(this->interface.get_buffer_space())) { if (!this->fetch_tasks(this->interface.get_buffer_space())) {
// we can sleep for a certain amount of time, nothing to do // we can sleep for a certain amount of time, nothing to do
// DEBUG_PRINT("Going to sleep. Checking for more tasks in a bit..."); // DEBUG_PRINT("Going to sleep. Checking for more tasks in a bit...");
std::this_thread::sleep_for(NOTHING_AVAILABLE_SLEEP_TIME); this->interface.wait_for_available();
} }
} }
@ -112,155 +115,46 @@ void Downloader::thread_run() {
bool Downloader::fetch_tasks(size_t n) { bool Downloader::fetch_tasks(size_t n) {
DEBUG_PRINT("Downloader fetching " + std::to_string(n) + " tasks..."); DEBUG_PRINT("Downloader fetching " + std::to_string(n) + " tasks...");
// fetch a new task from the database bool txn_opened;
auto fetch_task_lambda = [n]( auto txn = this->conn->open_transaction(txn_opened);
pqxx::work *txn, pqxx::result res;
bool *task_avail,
std::unordered_set<db::uuid_t> *references,
std::vector<std::unique_ptr<DBSimConfigArtifact>> *tasks
) {
// Fetch a new task by a couple rules: try {
// 1.) It has not been started yet or the output row doesn't exist res = txn.exec(
// 2.) Passes that are already in progress are preferred "SELECT * "
// 3.) New passes are started in the order they were added to the database "FROM open_tasks "
// 4.) Passes are only started if all their dependencies are fulfilled
auto res = txn->exec(
"SELECT "
" ap.design_file AS design, "
" ap.top_proc, "
" ap.outputs AS target_artifact, "
" ap.id AS source_pass, "
" sc.id AS source_config, "
" sc.sim_commands, "
" sc.has_reference AS reference, "
" so.id AS id "
"FROM "
" sim_configs sc "
"JOIN "
" actsim_passes ap ON ap.sim_configs = sc.artifact "
"LEFT JOIN "
" sim_outputs so ON sc.has_reference = so.sim_config "
"JOIN "
" jobs j ON ap.job = j.id "
"WHERE "
" (so.part_status IS NULL OR so.part_status = 'finished') "
" AND ap.id IN ( "
" SELECT ap_dep.id "
" FROM actsim_passes ap_dep "
" JOIN passes p ON ap_dep.id = p.id "
" WHERE NOT EXISTS ( "
" SELECT 1 "
" FROM passes dep "
" WHERE dep.id = ANY(p.depends_on) "
" AND dep.pass_status != 'finished' "
" ) "
" ) "
// " AND (sc.has_reference IS NULL OR "
// " EXISTS ("
// " SELECT 1 "
// " FROM sim_outputs out "
// " WHERE out.sim_config = sc.has_reference "
// " AND out.part_status = 'finished' "
// " ) "
// " ) "
// "ORDER BY "
// " ap.pass_status = 'in_progress' DESC, "
// " j.time_added ASC "
"LIMIT $1;", "LIMIT $1;",
n n
); );
// seems like there is nothing to do right now for (auto&& row : res) {
if (res.size() < 1) { txn.exec(pqxx::prepped("mark_tasks"), {row["source_config"].as<db::uuid_t>()});
*task_avail = false;
return;
} }
*task_avail = true; txn.commit();
} catch (const std::exception& e) {
// generate the output rows // if something happened during the transaction, we roll it back
std::string sql = R"( txn.abort();
INSERT INTO sim_outputs (artifact, source_pass, sim_config)
VALUES ($1, $2, $3)
ON CONFLICT (sim_config)
DO UPDATE SET part_status = 'in_progress'
RETURNING id;
)";
txn->conn().prepare("insert_output_rows", sql);
for (auto row : res) {
auto target_artifact = row["target_artifact"].as<db::uuid_t>();
auto source_pass = row["source_pass"].as<db::uuid_t>();
auto design = row["design"].as<db::uuid_t>();
db::uuid_t reference;
reference = 0;
if (!row["reference"].is_null()) {
reference = row["reference"].as<db::uuid_t>();
(*references).emplace(reference);
}
auto source_config = row["source_config"].as<db::uuid_t>();
auto com_arr = row["sim_commands"].as_sql_array<std::string>();
std::vector<std::string> commands(com_arr.cbegin(), com_arr.cend());
pl::testcase_t testcase = {
commands,
row["top_proc"].as<std::string>(),
false
};
// get the id
auto id = txn->exec(pqxx::prepped{"insert_output_rows"}, {target_artifact, source_pass, source_config})[0]["id"].as<db::uuid_t>();
auto task = std::make_unique<DBSimConfigArtifact>(id, source_pass, target_artifact, design, reference, source_config);
task->add_testcase(testcase);
(*tasks).emplace_back(std::move(task));
}
txn->conn().unprepare("insert_output_rows");
};
std::function<void(
pqxx::work *txn,
bool *task_avail,
std::unordered_set<db::uuid_t> *references,
std::vector<std::unique_ptr<DBSimConfigArtifact>> *tasks
)> fetch_task_func = fetch_task_lambda;
std::unordered_set<db::uuid_t> references;
bool task_avail;
std::vector<std::unique_ptr<DBSimConfigArtifact>> tasks;
if (!this->conn->send_request(
&fetch_task_func,
&task_avail,
&references,
&tasks
)) {
// if we lost connection, there's nothing else we can really do
std::cerr << "Error: Lost connection while trying to fetch more tasks! Aborting!" << std::endl; std::cerr << "Error: Lost connection while trying to fetch more tasks! Aborting!" << std::endl;
std::cerr << e.what() << std::endl;
this->interface.stop(); this->interface.stop();
this->interface.stop_immediately(); this->interface.stop_immediately();
}
// seems like there are no tasks to do right now; tell the calling function
// that there is still space in the buffer and we should just wait a while
if (!task_avail) {
DEBUG_PRINT("Buffer could not be filled - no more tasks available in the database!");
return false; return false;
} }
for (auto&& task : tasks) { // seems like there is nothing to do right now
if (res.size() < 1) {
return false;
}
auto design = task->design; for (auto row : res) {
auto target_artifact = row["target_artifact"].as<db::uuid_t>();
auto source_pass = row["source_pass"].as<db::uuid_t>();
auto design = row["design"].as<db::uuid_t>();
auto source_config = row["source_config"].as<db::uuid_t>();
// see if we already have the design locally; if not, load it // see if we already have the design locally; if not, load it
if (!this->interface.increment_design(design)) { if (!this->interface.increment_design(design)) {
@ -270,37 +164,51 @@ bool Downloader::fetch_tasks(size_t n) {
// if we could not load the design, reopen it in the database // if we could not load the design, reopen it in the database
if (!this->fetch_design(design, design_path)) { if (!this->fetch_design(design, design_path)) {
std::cerr << "Error: Could not load design for task " << task->id << ", reopening it." << std::endl; std::cerr << "Error: Could not load design for task " << source_config << ", reopening it." << std::endl;
this->reopen_task(task->id, true); this->reopen_task(source_config, true);
continue; continue;
} }
this->interface.store_design(design, design_path); this->interface.store_design(design, design_path);
} }
}
db::uuid_t reference;
reference = 0;
// if we have a reference for this run, we have to see if it is loaded if (!row["reference"].is_null()) {
for (auto&& reference : references) { reference = row["reference"].as<db::uuid_t>();
// see if we already have the reference run locally; if not, load it
if (!this->interface.increment_reference(reference)) {
// see if we already have the reference run locally; if not, load it DEBUG_PRINT("Fetching new reference run with ID " + db::to_string(reference));
if (!this->interface.increment_reference(reference)) { std::shared_ptr<pl::SimOutputArtifact> reference_run;
// if we could not load the reference run, reopen the task in the database
if ((reference_run = this->fetch_reference_run(reference)) == nullptr) {
std::cerr << "Error: Could not load reference run " << reference << "!" << std::endl;
// this->reopen_task(task->id, true);
continue;
}
DEBUG_PRINT("Fetching new reference run with ID " + db::to_string(reference)); this->interface.store_reference(reference, reference_run);
std::shared_ptr<pl::SimOutputArtifact> reference_run;
// if we could not load the reference run, reopen the task in the database
if ((reference_run = this->fetch_reference_run(reference)) == nullptr) {
std::cerr << "Error: Could not load reference run " << reference << "!" << std::endl;
// this->reopen_task(task->id, true);
continue;
} }
this->interface.store_reference(reference, reference_run);
} }
}
// push the task to the list of open tasks auto com_arr = row["sim_commands"].as_sql_array<std::string>();
this->interface.push_fresh(tasks); std::vector<std::string> commands(com_arr.cbegin(), com_arr.cend());
pl::testcase_t testcase = {
commands,
row["top_proc"].as<std::string>(),
false
};
auto task = std::make_unique<DBSimConfigArtifact>(db::uuid_t(), source_pass, target_artifact, design, reference, source_config);
task->add_testcase(testcase);
this->interface.push_fresh(std::move(task));
}
return true; return true;
} }
@ -379,14 +287,14 @@ std::shared_ptr<pl::SimOutputArtifact> Downloader::fetch_reference_run(const db:
bool *found bool *found
) { ) {
try { try {
auto res = txn->exec("SELECT output_tokens, output_token_timings, log_size FROM sim_outputs WHERE sim_config = $1;", id)[0]; auto res = txn->exec("SELECT output_tokens, output_token_timings FROM sim_outputs WHERE sim_config = $1;", id)[0];
// load the output token timings // load the output token timings
auto arr_ott = res["output_token_timings"].as_sql_array<uint32_t>(); auto arr_ott = res["output_token_timings"].as_sql_array<uint32_t>();
*output_token_timings = std::vector<uint32_t>(arr_ott.cbegin(), arr_ott.cend()); *output_token_timings = std::vector<uint32_t>(arr_ott.cbegin(), arr_ott.cend());
*output_tokens = res["output_tokens"].as<long>(); *output_tokens = res["output_tokens"].as<long>();
*log_size = res["log_size"].as<long>(); *log_size = 0;
*found = true; *found = true;
@ -443,7 +351,7 @@ void Downloader::reopen_task(const db::uuid_t& id, bool halt) {
// open up the status of this partial output in the database again // open up the status of this partial output in the database again
auto task_reopen_lambda = [id, status](pqxx::work *txn) { auto task_reopen_lambda = [id, status](pqxx::work *txn) {
txn->exec("UPDATE sim_outputs SET part_status = $2 WHERE id = $1 AND part_status = 'in_progress';", {id, status}); txn->exec("UPDATE sim_configs SET part_status = $2 WHERE id = $1 AND part_status = 'in_progress';", {id, status});
}; };
std::function<void(pqxx::work*)> task_reopen_func = task_reopen_lambda; std::function<void(pqxx::work*)> task_reopen_func = task_reopen_lambda;

51
src/actsim_agent/task_interface.cpp
View file

@ -24,7 +24,10 @@
*/ */
#include "util.h" #include "util.h"
#include <chrono>
#include <cstddef>
#include <cstdio> #include <cstdio>
#include <thread>
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "task_interface.hpp" #include "task_interface.hpp"
@ -75,9 +78,9 @@ bool TaskInterface::fresh_queue_empty() {
return this->fresh_queue.empty(); return this->fresh_queue.empty();
} }
bool TaskInterface::finished_queue_empty() { bool TaskInterface::finished_queue_empty(size_t min_size) {
std::lock_guard<std::mutex> lock(this->finished_queue_mutex); std::lock_guard<std::mutex> lock(this->finished_queue_mutex);
return this->finished_queue.empty(); return this->finished_queue.size() < min_size;
} }
std::unique_ptr<InputType> TaskInterface::pop_fresh(bool& empty) { std::unique_ptr<InputType> TaskInterface::pop_fresh(bool& empty) {
@ -105,17 +108,21 @@ std::unique_ptr<InputType> TaskInterface::pop_fresh(bool& empty) {
void TaskInterface::wait_for_fresh() { void TaskInterface::wait_for_fresh() {
std::unique_lock<std::mutex> lock (this->fresh_queue_empty_mutex); std::unique_lock<std::mutex> lock (this->fresh_queue_empty_mutex);
worker_waiting = true;
finished_queue_empty_condition.notify_all();
// we will be notified either when there is new data or the program has been stopped // we will be notified either when there is new data or the program has been stopped
this->fresh_queue_empty_condition.wait(lock, [&] { return !this->fresh_queue_empty() || !running(); }); this->fresh_queue_empty_condition.wait(lock, [&] { return !this->fresh_queue_empty() || !running(); });
DEBUG_PRINT("Worker released from block"); DEBUG_PRINT("Worker released from block");
worker_waiting = false;
} }
void TaskInterface::wait_for_finished() { void TaskInterface::wait_for_finished(size_t min_size) {
std::unique_lock<std::mutex> lock (this->finished_queue_empty_mutex); std::unique_lock<std::mutex> lock (this->finished_queue_empty_mutex);
// we will be notified either when there is new data or the program has been stopped // we will be notified either when there is new data or the program has been stopped
this->finished_queue_empty_condition.wait(lock, [&] { return !this->finished_queue_empty() || !running(); }); this->finished_queue_empty_condition.wait(lock, [&] { return !this->finished_queue_empty(min_size) || !running() || (worker_waiting && downloader_waiting);});
} }
void TaskInterface::wait_for_buffer_consume() { void TaskInterface::wait_for_buffer_consume() {
@ -125,6 +132,18 @@ void TaskInterface::wait_for_buffer_consume() {
this->fresh_queue_full_condition.wait(lock, [&] { return this->get_buffer_space() > 0 || !running(); }); this->fresh_queue_full_condition.wait(lock, [&] { return this->get_buffer_space() > 0 || !running(); });
} }
void TaskInterface::wait_for_available() {
downloader_waiting = true;
finished_queue_empty_condition.notify_all();
using namespace std::chrono_literals;
std::this_thread::sleep_for(NOTHING_AVAILABLE_SLEEP_TIME);
downloader_waiting = false;
}
void TaskInterface::push_finished(std::unique_ptr<OutputType> task) { void TaskInterface::push_finished(std::unique_ptr<OutputType> task) {
std::lock_guard<std::mutex> lock(this->finished_queue_mutex); std::lock_guard<std::mutex> lock(this->finished_queue_mutex);
this->finished_queue.push(std::move(task)); this->finished_queue.push(std::move(task));
@ -133,21 +152,27 @@ void TaskInterface::push_finished(std::unique_ptr<OutputType> task) {
this->finished_queue_empty_condition.notify_one(); this->finished_queue_empty_condition.notify_one();
} }
std::unique_ptr<OutputType> TaskInterface::pop_finished(bool& empty) { std::vector<std::unique_ptr<OutputType>> TaskInterface::pop_finished(bool& empty) {
std::vector<std::unique_ptr<OutputType>> tasks;
// we first need exclusive access to the queue // we first need exclusive access to the queue
std::lock_guard<std::mutex> lock (this->finished_queue_mutex); std::lock_guard<std::mutex> lock (this->finished_queue_mutex);
// we have to make sure the queue wasn't emptied since the last empty call was made // check if there is a minimum amount of tasks in the queue
// or at least inform the calling thread that the queue is currently empty if (this->finished_queue.empty()) {
std::unique_ptr<OutputType> task; empty = true;
return tasks;
if (!(empty = this->finished_queue.empty())) {
task = std::move(this->finished_queue.front());
this->finished_queue.pop();
} }
return task; empty = false;
while (!this->finished_queue.empty()) {
tasks.emplace_back(std::move(this->finished_queue.front()));
this->finished_queue.pop();
}
return tasks;
} }
bool TaskInterface::increment_design(const db::uuid_t& id) { bool TaskInterface::increment_design(const db::uuid_t& id) {

84
src/actsim_agent/uploader.cpp
View file

@ -23,6 +23,7 @@
************************************************************************** **************************************************************************
*/ */
#include <cstddef>
#include <iostream> #include <iostream>
#include <memory> #include <memory>
#include <pqxx/pqxx> #include <pqxx/pqxx>
@ -57,12 +58,15 @@ void Uploader::thread_run() {
this->interface.stop(); this->interface.stop();
return; return;
} }
size_t min_size = 1;
while (this->interface.running()) { while (this->interface.running()) {
// this blocks until either a new task is available for upload or the // this blocks until either a new task is available for upload or the
// program was halted // program was halted
this->interface.wait_for_finished(); this->interface.wait_for_finished(min_size);
DEBUG_PRINT("Uploader was woken up"); DEBUG_PRINT("Uploader was woken up");
@ -71,7 +75,7 @@ void Uploader::thread_run() {
// we're still good to go! get a task from the fresh queue // we're still good to go! get a task from the fresh queue
bool queue_empty; bool queue_empty;
auto task = this->interface.pop_finished(queue_empty); auto tasks = this->interface.pop_finished(queue_empty);
// we need to make sure the queue wasn't emptied between waiting and getting new data // we need to make sure the queue wasn't emptied between waiting and getting new data
if (queue_empty) continue; if (queue_empty) continue;
@ -79,7 +83,7 @@ void Uploader::thread_run() {
DEBUG_PRINT("Uploader dequeued new task"); DEBUG_PRINT("Uploader dequeued new task");
// everything is good, upload the given task // everything is good, upload the given task
bool success = this->upload_task(std::move(task)); bool success = this->upload_task(tasks);
// Uh oh, seems like we lost database connection! Close the program. // Uh oh, seems like we lost database connection! Close the program.
if (!success) { if (!success) {
@ -89,6 +93,7 @@ void Uploader::thread_run() {
} }
DEBUG_PRINT("Task successfully uploaded"); DEBUG_PRINT("Task successfully uploaded");
min_size = 200;
} }
DEBUG_PRINT("Uploader is starting the shutdown procedure, waiting for cleanup ready"); DEBUG_PRINT("Uploader is starting the shutdown procedure, waiting for cleanup ready");
@ -100,75 +105,44 @@ void Uploader::thread_run() {
DEBUG_PRINT("Uploader has received go for cleanup"); DEBUG_PRINT("Uploader has received go for cleanup");
// upload all the remaining tasks // upload all the remaining tasks
while (!this->interface.finished_queue_empty()) {
bool queue_empty; bool queue_empty;
auto task = this->interface.pop_finished(queue_empty); auto tasks = this->interface.pop_finished(queue_empty);
// in case there are ever multiple upload threads, // in case there are ever multiple upload threads,
// the same issues apply as before // the same issues apply as before
if (!queue_empty) { if (!queue_empty) {
DEBUG_PRINT("Uploading finished task"); DEBUG_PRINT("Uploading finished task");
if (!this->upload_task(std::move(task))) { if (!this->upload_task(tasks)) {
std::cerr << "Error: Lost database connection for uploading tasks during cleanup. Database integrity might be compromised." << std::endl; std::cerr << "Error: Lost database connection for uploading tasks during cleanup. Database integrity might be compromised." << std::endl;
}
} }
} }
DEBUG_PRINT("Uploader is done"); DEBUG_PRINT("Uploader is done");
} }
bool Uploader::upload_task(std::unique_ptr<OutputType> task) { bool Uploader::upload_task(std::vector<std::unique_ptr<OutputType>>& tasks) {
auto&& task_id = task->id; auto upload_results_lambda = [] (pqxx::work* txn, std::vector<std::unique_ptr<OutputType>>* tasks) {
auto&& sim_log = task->get_content().first;
auto&& sim_error = task->get_content().second;
auto&& output_token_timings = task->get_output_token_timings();
auto&& output_tokens = task->output_tokens;
auto log_size = sim_log.size() + sim_error.size();
const auto&& fault_flags = build_fault_flags(task); pqxx::stream_to stream = pqxx::stream_to::table(*txn, {"sim_outputs"}, {"artifact","source_pass", "sim_config", "output_tokens", "output_token_timings", "fault_flags", "part_status"});
// make sure any task that is uploaded isn't halted in the database for (auto&& task : (*tasks)) {
auto task_upload_lambda = [ task_id, stream.write_values(task->target_artifact, task->source_pass, task->source_config, task->output_tokens, task->get_output_token_timings(), build_fault_flags(task), "finished");
sim_log, }
sim_error,
output_tokens, stream.complete();
output_token_timings,
fault_flags,
log_size
](
pqxx::work *txn
) {
txn->exec(
"UPDATE sim_outputs SET "
" sim_log = $1, "
" error_log = $2, "
" output_tokens = $3, "
" output_token_timings = $4, "
" fault_flags = $5, "
" log_size = $6, "
" part_status = 'finished' "
"WHERE id = $7 AND part_status != 'halted';",
{sim_log,
sim_error,
output_tokens,
output_token_timings,
fault_flags,
log_size,
task_id}
);
}; };
std::function<void(pqxx::work*)> task_upload_func = task_upload_lambda; std::function<void(pqxx::work*, std::vector<std::unique_ptr<OutputType>>*)> task_upload_func = upload_results_lambda;
DEBUG_PRINT("Updating task " + db::to_string(task->id)); DEBUG_PRINT("Updating tasks");
return this->conn->send_request(&task_upload_func); return this->conn->send_request(&task_upload_func, &tasks);
} }
std::string Uploader::build_fault_flags(const std::unique_ptr<OutputType>& task) { std::string build_fault_flags(const std::unique_ptr<OutputType>& task) {
// bit mask for faults is // bit mask for faults is
// 0: timing // 0: timing

2
src/cluster/db_client.cpp
View file

@ -125,7 +125,7 @@ bool Connection::prepare_statement(std::string name, std::string statement) {
try { try {
this->c->prepare(name, statement); this->c->prepare(name, statement);
} catch (const exception &e) { } catch (const exception &e) {
cerr << "Error: Could not prepare statements!" << endl; cerr << "Error: Could not prepare statements! " << e.what() << endl;
return false; return false;
} }