From b12ec302a5641e13c7caa917944a210b8a7baa79 Mon Sep 17 00:00:00 2001 From: Tina_Azure <-> Date: Thu, 5 Oct 2023 17:19:16 +0200 Subject: [PATCH] regular execution purge expired submissions --- src/database.cpp | 34 +++++++++++++++++++ src/databaseStatementConstCollection.cpp | 43 ++++++++++++++++++------ src/default-cavecomm.conf | 2 +- src/regularTaskExecution.cpp | 25 ++++++++++++++ src/utilities.cpp | 1 + 5 files changed, 93 insertions(+), 12 deletions(-) diff --git a/src/database.cpp b/src/database.cpp index bfaee3e..4d7f104 100644 --- a/src/database.cpp +++ b/src/database.cpp @@ -791,6 +791,40 @@ namespace Database { work.commit(); } + /* + * Executes the prepared statement SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH + * Takes an open pqxx::connection the alias and the freelancer ID + * returns the path to the file + */ + pqxx::result executePreparedStatement_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH(pqxx::connection &connection, int expirationInHours) { + pqxx::work work(connection); + pqxx::result result = work.exec_prepared(PREPARED_STATEMENT_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH, expirationInHours); + work.commit(); + return result; + } + + /* + * Executes the prepared statement PURGE_FREELANCER_FILE_SUBMISSION + * Deletes all expired file submissions + * Takes an open pqxx::connection and the expiration in hours + */ + void executePreparedStatement_PURGE_FREELANCER_FILE_SUBMISSION(pqxx::connection &connection, int expirationInHours) { + pqxx::work work(connection); + work.exec_prepared(PREPARED_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION, expirationInHours); + work.commit(); + } + + /* + * Executes the prepared statement PURGE_FREELANCER_FILE_SUBMISSION_ALIAS + * Deletes all alias related to expired file submissions + * Takes an open pqxx::connection and the expiration in hours + */ + void executePreparedStatement_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS(pqxx::connection &connection, int expirationInHours) { + pqxx::work work(connection); + work.exec_prepared(PREPARED_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS, expirationInHours); + work.commit(); + } + /* * Executes the prepared statement INSERT_FREELANCER_FILE_SUBMISSION * Takes an open pqxx::connection and the freelancer email, the file name, the full path and the filesize in byte diff --git a/src/databaseStatementConstCollection.cpp b/src/databaseStatementConstCollection.cpp index 8b23ea8..39219f4 100644 --- a/src/databaseStatementConstCollection.cpp +++ b/src/databaseStatementConstCollection.cpp @@ -294,12 +294,6 @@ namespace DatabaseStatementConstCollection { const static std::string PREPARED_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES = "deleteFreelancerFileSubmissionAliasRoutes"; const static std::string SQL_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES = "delete from aliasroutes where routeparameter = 'FILESUBMISSION' and freelancerid = (select freelancers.id from freelancers where emailaddress = $1) and route = $2"; - /* - * Name and Statement for prepared statement to purge expired file submissions from the database - */ - const static std::string PREPARED_STATEMENT_PRUGE_FREELANCER_FILE_SUBMISSION = "purgeFreelancerFileSubmissions"; - const static std::string SQL_STATEMENT_PRUGE_FREELANCER_FILE_SUBMISSION = "delete from freelancersubmissions where (current_timestamp - uploaddate) > make_interval(hours => $1)"; - /* * Name and Statement for prepared statement to insert metadata of a submission. * 1=freelancer email, 2=filename, 3=fullpath, 4=filesize in byte @@ -307,6 +301,27 @@ namespace DatabaseStatementConstCollection { const static std::string PREPARED_STATEMENT_INSERT_FREELANCER_FILE_SUBMISSION = "insertFreelancerFileSubmission"; const static std::string SQL_STATEMENT_INSERT_FREELANCER_FILE_SUBMISSION = "insert into freelancersubmissions values ((select freelancers.id from freelancers where emailaddress = $1), $2, $3, $4, CURRENT_TIMESTAMP);"; + /* + * Name and Statement for prepared statement to get the full paths of all expired submissions + * 1=Expiration in hours + */ + const static std::string PREPARED_STATEMENT_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH = "selectFreelancerExpiredFileSubmissionPath"; + const static std::string SQL_STATEMENT_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH = "select fullpath from freelancersubmissions where (current_timestamp - uploaddate) > make_interval(hours => $1)"; + + /* + * Name and Statement for prepared statement to purge all expired submissions + * 1=Expiration in hours + */ + const static std::string PREPARED_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION = "purgeFreelancerExpiredFileSubmission"; + const static std::string SQL_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION = "delete from freelancersubmissions where (current_timestamp - uploaddate) > make_interval(hours => $1)"; + + /* + * Name and Statement for prepared statement to purge all alias related to expired submissions + * 1=Expiration in hours + */ + const static std::string PREPARED_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS = "purgeFreelancerExpiredFileSubmissionAlias"; + const static std::string SQL_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS = "delete from aliasroutes where route in (select freelancersubmissions.filename from freelancersubmissions where (current_timestamp - uploaddate) > make_interval(hours => $1) and routeparameter = 'FILESUBMISSION')"; + /* * IDs of prepared statements @@ -352,13 +367,15 @@ namespace DatabaseStatementConstCollection { const static int ID_SELECT_FREELANCER_FILE_SUBMISSION_USED_STORAGE = 38; const static int ID_SELECT_FREELANCER_FILE_SUBMISSION_PATH = 39; const static int ID_DELETE_FREELANCER_FILE_SUBMISSION = 40; - const static int ID_PRUGE_FREELANCER_FILE_SUBMISSION = 41; + const static int ID_PURGE_FREELANCER_FILE_SUBMISSION = 41; const static int ID_INSERT_FREELANCER_FILE_SUBMISSION = 42; const static int ID_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS_PATH = 43; const static int ID_INSERT_FREELANCER_FILE_SUBMISSION_ALIAS = 44; const static int ID_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS = 45; const static int ID_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS = 46; const static int ID_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES = 47; + const static int ID_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH = 48; + const static int ID_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS = 49; @@ -407,13 +424,15 @@ namespace DatabaseStatementConstCollection { {PREPARED_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_USED_STORAGE, SQL_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_USED_STORAGE}, {PREPARED_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_PATH, SQL_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_PATH}, {PREPARED_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION, SQL_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION}, - {PREPARED_STATEMENT_PRUGE_FREELANCER_FILE_SUBMISSION, SQL_STATEMENT_PRUGE_FREELANCER_FILE_SUBMISSION}, + {PREPARED_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION, SQL_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION}, {PREPARED_STATEMENT_INSERT_FREELANCER_FILE_SUBMISSION, SQL_STATEMENT_INSERT_FREELANCER_FILE_SUBMISSION}, {PREPARED_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS_PATH, SQL_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS_PATH}, {PREPARED_STATEMENT_INSERT_FREELANCER_FILE_SUBMISSION_ALIAS, SQL_STATEMENT_INSERT_FREELANCER_FILE_SUBMISSION_ALIAS}, {PREPARED_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS, SQL_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS}, {PREPARED_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS, SQL_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS}, - {PREPARED_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES, SQL_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES} + {PREPARED_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES, SQL_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES}, + {PREPARED_STATEMENT_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH, SQL_STATEMENT_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH}, + {PREPARED_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS, SQL_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS} }; /* * Easy access to prepared statement name via int @@ -460,13 +479,15 @@ namespace DatabaseStatementConstCollection { {ID_SELECT_FREELANCER_FILE_SUBMISSION_USED_STORAGE, PREPARED_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_USED_STORAGE}, {ID_SELECT_FREELANCER_FILE_SUBMISSION_PATH, PREPARED_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_PATH}, {ID_DELETE_FREELANCER_FILE_SUBMISSION, PREPARED_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION}, - {ID_PRUGE_FREELANCER_FILE_SUBMISSION, PREPARED_STATEMENT_PRUGE_FREELANCER_FILE_SUBMISSION}, + {ID_PURGE_FREELANCER_FILE_SUBMISSION, PREPARED_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION}, {ID_INSERT_FREELANCER_FILE_SUBMISSION, PREPARED_STATEMENT_INSERT_FREELANCER_FILE_SUBMISSION}, {ID_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS_PATH, PREPARED_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS_PATH}, {ID_INSERT_FREELANCER_FILE_SUBMISSION_ALIAS, PREPARED_STATEMENT_INSERT_FREELANCER_FILE_SUBMISSION_ALIAS}, {ID_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS, PREPARED_STATEMENT_SELECT_FREELANCER_FILE_SUBMISSION_ALIAS}, {ID_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS, PREPARED_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS}, - {ID_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES, PREPARED_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES} + {ID_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES, PREPARED_STATEMENT_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS_ROUTES}, + {ID_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH, PREPARED_STATEMENT_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH}, + {ID_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS, PREPARED_STATEMENT_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS} }; /* diff --git a/src/default-cavecomm.conf b/src/default-cavecomm.conf index 810d993..cfdfa37 100644 --- a/src/default-cavecomm.conf +++ b/src/default-cavecomm.conf @@ -61,7 +61,7 @@ bruteForceMitigationAttempts=5; submissionFileRootPath=./submissions; submissionMaxFileSizeMB=8; submissionMaxtotalStorageMB=16; -submissionMaxStorageDurationH=24; +submissionMaxStorageDurationH=720; submissionAllowedFiletypes=rar|zip|7z|arc|jpg|jpeg|png|mp4|webm; submissionMaxFileNameSize=255; submissionMaxFilePathSize=4096; diff --git a/src/regularTaskExecution.cpp b/src/regularTaskExecution.cpp index 7fc150a..ad54083 100644 --- a/src/regularTaskExecution.cpp +++ b/src/regularTaskExecution.cpp @@ -27,9 +27,32 @@ namespace RegularTaskExecution { Database::executeStatement_STATEMENT_PURGE_EXPIRED_FREELANCER_RESET_KEYS(connection); } + /* + * Deletes all file submissions and related alias that are expired + */ + void purgeExpiredFreelancerSubmissions(pqxx::connection &connection, const Utilities::config& configuration) { + pqxx::result resultFilePaths = Database::executePreparedStatement_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH(connection, configuration.submissionMaxStorageDurationH); + if (!resultFilePaths.empty()) { + for (int i = 0; i < resultFilePaths.size(); i++) { + if (!Utilities::deleteFile(resultFilePaths.at(i).at(0).c_str())) { + std::string errorMessage = "purgeExpiredFreelancerSubmissions unable to delete file: "; + errorMessage.append(resultFilePaths.at(i).at(0).c_str()); + Utilities::errorOut(errorMessage); + } + } + Database::executePreparedStatement_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS(connection, configuration.submissionMaxStorageDurationH); + Database::executePreparedStatement_PURGE_FREELANCER_FILE_SUBMISSION(connection, configuration.submissionMaxStorageDurationH); + } + } + void regularExecution(const Utilities::config& configuration) { std::chrono::seconds seconds(configuration.regularTaskExecutionIntervalSeconds); pqxx::connection databaseConnection(configuration.databaseConnectionString); + Database::prepareStatements(databaseConnection, { + ID_SELECT_FREELANCER_EXPIRED_FILE_SUBMISSION_PATH, + ID_PURGE_FREELANCER_FILE_SUBMISSION, + ID_PURGE_FREELANCER_FILE_SUBMISSION_ALIAS + }); size_t counter = 0; std::cout << "REGULAR EXECUTION START: " << std::endl; while (configuration.regularTaskExecution) { @@ -38,6 +61,8 @@ namespace RegularTaskExecution { purgeExpiredLoginLockouts(databaseConnection); if (configuration.regularTaskExecutionModules.at(Utilities::MODULE_NAME_FREELANCER_RESET_KEY_CLEANER)) purgeExpiredFreelancerResetKeys(databaseConnection); + if (configuration.regularTaskExecutionModules.at(Utilities::MODULE_NAME_SUBMISSION_STORAGE_CLEANER) && configuration.submissionMaxStorageDurationH > 0) + purgeExpiredFreelancerSubmissions(databaseConnection, configuration); counter++; std::this_thread::sleep_for(seconds); } diff --git a/src/utilities.cpp b/src/utilities.cpp index ad6305b..72227e6 100644 --- a/src/utilities.cpp +++ b/src/utilities.cpp @@ -798,6 +798,7 @@ namespace Utilities { Database::executePreparedStatement_DELETE_FREELANCER_FILE_SUBMISSION_ALIAS(databaseConnection, emailAddress, fileName, aliasName); } + /* * Generates an alias route to the freelancers profile * takes a freelancers name