t_a/cavepaintingsFork
1
Fork 0
forked from Cavemanon/cavepaintings
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added lock file usage to cron uploader to prevent concurrent runs.

Browse Source 
Changed extension manager to allow author to be a comma-separated list.
This commit is contained in:
Matthew Barbour
2019-06-15 11:01:13 -05:00
committed by matthew
parent e940d87c22
commit 0202597f88
3 changed files with 162 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
ext/cron_uploader/main.php
View File

@@ -1,42 +1,44 @@
<?php
/*
* Name: Cron Uploader
* Author: YaoiFox <admin@yaoifox.com>
* Authors: YaoiFox <admin@yaoifox.com>, Matthew Barbour <matthew@darkholme.net>
* Link: http://www.yaoifox.com/
* License: GPLv2
* Description: Uploads images automatically using Cron Jobs
* Documentation: Installation guide: activate this extension and navigate to www.yoursite.com/cron_upload
*/
class CronUploader extends Extension
{
// TODO: Checkbox option to only allow localhost + a list of additional IP adresses that can be set in /cron_upload
// TODO: Change logging to MySQL + display log at /cron_upload
// TODO: Move stuff to theme.php
/**
* Lists all log events this session
* @var string
*/
private $upload_info = "";
/**
* Lists all files & info required to upload.
* @var array
*/
private $image_queue = [];
/**
* Cron Uploader root directory
* @var string
*/
private $root_dir = "";
/**
* Key used to identify uploader
* @var string
*/
private $upload_key = "";
/**
* Checks if the cron upload page has been accessed
* and initializes the upload.
@@ -44,39 +46,50 @@ class CronUploader extends Extension
public function onPageRequest(PageRequestEvent $event)
{
global $config, $user;
if ($event->page_matches("cron_upload")) {
$this->upload_key = $config->get_string("cron_uploader_key", "");
// If the key is in the url, upload
if ($this->upload_key != "" && $event->get_arg(0) == $this->upload_key) {
// log in as admin
$this->process_upload(); // Start upload
$this->set_dir();
$lockfile = fopen($this->root_dir . "/.lock", "w");
try {
if (!flock($lockfile, LOCK_EX | LOCK_NB)) {
throw new Exception("Cron upload process is already running");
}
$this->process_upload(); // Start upload
} finally {
flock($lockfile, LOCK_UN);
fclose($lockfile);
}
} elseif ($user->is_admin()) {
$this->set_dir();
$this->display_documentation();
}
}
}
private function display_documentation()
{
global $page;
$this->set_dir(); // Determines path to cron_uploader_dir
$queue_dir = $this->root_dir . "/queue";
$uploaded_dir = $this->root_dir . "/uploaded";
$failed_dir = $this->root_dir . "/failed_to_upload";
$queue_dirinfo = $this->scan_dir($queue_dir);
$uploaded_dirinfo = $this->scan_dir($uploaded_dir);
$failed_dirinfo = $this->scan_dir($failed_dir);
$cron_url = make_http(make_link("/cron_upload/" . $this->upload_key));
$cron_cmd = "curl --silent $cron_url";
$log_path = $this->root_dir . "/uploads.log";
$info_html = "<b>Information</b>
<br>
<table style='width:470px;'>
@@ -105,7 +118,7 @@ class CronUploader extends Extension
<br>Cron Command: <input type='text' size='60' value='$cron_cmd'><br>
Create a cron job with the command above.<br/>
Read the documentation if you're not sure what to do.<br>";
$install_html = "
This cron uploader is fairly easy to use but has to be configured first.
<br />1. Install & activate this plugin.
@@ -130,8 +143,10 @@ class CronUploader extends Extension
<br />So when you want to manually upload an image, all you have to do is open the link once.
<br />This link can be found under 'Cron Command' in the board config, just remove the 'wget ' part and only the url remains.
<br />(<b>$cron_url</b>)";
$page->set_title("Cron Uploader");
$page->set_heading("Cron Uploader");
$block = new Block("Cron Uploader", $info_html, "main", 10);
$block_install = new Block("Installation Guide", $install_html, "main", 20);
$page->add_block($block);
@@ -143,35 +158,35 @@ class CronUploader extends Extension
global $config;
// Set default values
$this->upload_key = $config->get_string("cron_uploader_key", "");
if (strlen($this->upload_key)<=0) {
if (strlen($this->upload_key) <= 0) {
$this->upload_key = $this->generate_key();
$config->set_default_int('cron_uploader_count', 1);
$config->set_default_string('cron_uploader_key', $this->upload_key);
$this->set_dir();
}
}
public function onSetupBuilding(SetupBuildingEvent $event)
{
$this->set_dir();
$cron_url = make_http(make_link("/cron_upload/" . $this->upload_key));
$cron_cmd = "curl --silent $cron_url";
$documentation_link = make_http(make_link("cron_upload"));
$sb = new SetupBlock("Cron Uploader");
$sb->add_label("<b>Settings</b><br>");
$sb->add_int_option("cron_uploader_count", "How many to upload each time");
$sb->add_text_option("cron_uploader_dir", "<br>Set Cron Uploader root directory<br>");
$sb->add_label("<br>Cron Command: <input type='text' size='60' value='$cron_cmd'><br>
Create a cron job with the command above.<br/>
<a href='$documentation_link'>Read the documentation</a> if you're not sure what to do.");
$event->panel->add_block($sb);
}
/*
* Generates a unique key for the website to prevent unauthorized access.
*/
@@ -180,14 +195,14 @@ class CronUploader extends Extension
$length = 20;
$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$randomString = '';
for ($i = 0; $i < $length; $i ++) {
for ($i = 0; $i < $length; $i++) {
$randomString .= $characters [rand(0, strlen($characters) - 1)];
}
return $randomString;
}
/*
* Set the directory for the image queue. If no directory was given, set it to the default directory.
*/
@@ -195,15 +210,15 @@ class CronUploader extends Extension
{
global $config;
// Determine directory (none = default)
$dir = $config->get_string("cron_uploader_dir", "");
// Sets new default dir if not in config yet/anymore
if ($dir == "") {
$dir = data_path("cron_uploader");
$config->set_string('cron_uploader_dir', $dir);
}
// Make the directory if it doesn't exist yet
if (!is_dir($dir . "/queue/")) {
mkdir($dir . "/queue/", 0775, true);
@@ -214,31 +229,31 @@ class CronUploader extends Extension
if (!is_dir($dir . "/failed_to_upload/")) {
mkdir($dir . "/failed_to_upload/", 0775, true);
}
$this->root_dir = $dir;
return $dir;
}
/**
* Returns amount of files & total size of dir.
*/
public function scan_dir(string $path): array
{
$bytestotal=0;
$nbfiles=0;
$bytestotal = 0;
$nbfiles = 0;
$ite=new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS);
foreach (new RecursiveIteratorIterator($ite) as $filename=>$cur) {
$ite = new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS);
foreach (new RecursiveIteratorIterator($ite) as $filename => $cur) {
$filesize = $cur->getSize();
$bytestotal += $filesize;
$nbfiles++;
}
$size_mb = $bytestotal / 1048576; // to mb
$size_mb = number_format($size_mb, 2, '.', '');
return ['total_files'=>$nbfiles,'total_mb'=>$size_mb];
return ['total_files' => $nbfiles, 'total_mb' => $size_mb];
}
/**
* Uploads the image & handles everything
*/
@@ -246,24 +261,24 @@ class CronUploader extends Extension
{
global $config, $database;
set_time_limit(0);
//set_time_limit(0);
$output_subdir = date('Ymd-His', time())."/";
$this->set_dir();
$output_subdir = date('Ymd-His', time()) . "/";
$this->generate_image_queue();
// Gets amount of imgs to upload
if ($upload_count == 0) {
$upload_count = $config->get_int("cron_uploader_count", 1);
}
// Throw exception if there's nothing in the queue
if (count($this->image_queue) == 0) {
$this->add_upload_info("Your queue is empty so nothing could be uploaded.");
$this->handle_log();
return false;
}
// Randomize Images
//shuffle($this->image_queue);
@@ -274,15 +289,15 @@ class CronUploader extends Extension
$failedItems = [];
// Upload the file(s)
for ($i = 0; $i < $upload_count && sizeof($this->image_queue)>0; $i++) {
for ($i = 0; $i < $upload_count && sizeof($this->image_queue) > 0; $i++) {
$img = array_pop($this->image_queue);
try {
$database->beginTransaction();
$result = $this->add_image($img[0], $img[1], $img[2]);
$database->commit();
$this->move_uploaded($img[0], $img[1], $output_subdir, false);
if ($result==null) {
if ($result == null) {
$merged++;
} else {
$added++;
@@ -290,7 +305,7 @@ class CronUploader extends Extension
} catch (Exception $e) {
$failed++;
$this->move_uploaded($img[0], $img[1], $output_subdir, true);
$msgNumber = $this->add_upload_info("(".gettype($e).") ".$e->getMessage());
$msgNumber = $this->add_upload_info("(" . gettype($e) . ") " . $e->getMessage());
$msgNumber = $this->add_upload_info($e->getTraceAsString());
if (strpos($e->getMessage(), 'SQLSTATE') !== false) {
// Postgres invalidates the transaction if there is an SQL error,
@@ -310,40 +325,40 @@ class CronUploader extends Extension
$msgNumber = $this->add_upload_info("Items failed: $failed");
// Display & save upload log
$this->handle_log();
return true;
}
private function move_uploaded($path, $filename, $output_subdir, $corrupt = false)
{
global $config;
// Create
$newDir = $this->root_dir;
$relativeDir = dirname(substr($path, strlen($this->root_dir) + 7));
// Determine which dir to move to
if ($corrupt) {
// Move to corrupt dir
$newDir .= "/failed_to_upload/".$output_subdir.$relativeDir;
$newDir .= "/failed_to_upload/" . $output_subdir . $relativeDir;
$info = "ERROR: Image was not uploaded.";
} else {
$newDir .= "/uploaded/".$output_subdir.$relativeDir;
$newDir .= "/uploaded/" . $output_subdir . $relativeDir;
$info = "Image successfully uploaded. ";
}
$newDir = str_replace("//", "/", $newDir."/");
$newDir = str_replace("//", "/", $newDir . "/");
if (!is_dir($newDir)) {
mkdir($newDir, 0775, true);
}
// move file to correct dir
rename($path, $newDir.$filename);
rename($path, $newDir . $filename);
$this->add_upload_info($info . "Image \"$filename\" moved from queue to \"$newDir\".");
}
@@ -353,7 +368,7 @@ class CronUploader extends Extension
private function add_image(string $tmpname, string $filename, string $tags)
{
assert(file_exists($tmpname));
$pathinfo = pathinfo($filename);
$metadata = [];
$metadata ['filename'] = $pathinfo ['basename'];
@@ -364,7 +379,7 @@ class CronUploader extends Extension
$metadata ['source'] = null;
$event = new DataUploadEvent($tmpname, $metadata);
send_event($event);
// Generate info message
$infomsg = ""; // Will contain info message
if ($event->image_id == -1) {
@@ -377,18 +392,18 @@ class CronUploader extends Extension
$msgNumber = $this->add_upload_info($infomsg);
return $event->image_id;
}
private function generate_image_queue(): void
{
$base = $this->root_dir . "/queue";
if (! is_dir($base)) {
if (!is_dir($base)) {
$this->add_upload_info("Image Queue Directory could not be found at \"$base\".");
return;
}
$ite=new RecursiveDirectoryIterator($base, FilesystemIterator::SKIP_DOTS);
foreach (new RecursiveIteratorIterator($ite) as $fullpath=>$cur) {
$ite = new RecursiveDirectoryIterator($base, FilesystemIterator::SKIP_DOTS);
foreach (new RecursiveIteratorIterator($ite) as $fullpath => $cur) {
if (!is_link($fullpath) && !is_dir($fullpath)) {
$pathinfo = pathinfo($fullpath);
@@ -396,62 +411,62 @@ class CronUploader extends Extension
$tags = path_to_tags($relativePath);
$img = [
0 => $fullpath,
1 => $pathinfo ["basename"],
2 => $tags
0 => $fullpath,
1 => $pathinfo ["basename"],
2 => $tags
];
array_push($this->image_queue, $img);
}
}
}
/**
* Adds a message to the info being published at the end
*/
private function add_upload_info(string $text, int $addon = 0): int
{
$info = $this->upload_info;
$time = "[" .date('Y-m-d H:i:s'). "]";
$time = "[" . date('Y-m-d H:i:s') . "]";
// If addon function is not used
if ($addon == 0) {
$this->upload_info .= "$time $text\r\n";
$this->upload_info .= "$time $text\r\n";
// Returns the number of the current line
$currentLine = substr_count($this->upload_info, "\n") -1;
$currentLine = substr_count($this->upload_info, "\n") - 1;
return $currentLine;
}
// else if addon function is used, select the line & modify it
$lines = substr($info, "\n"); // Seperate the string to array in lines
$lines[$addon] = "$lines[$addon] $text"; // Add the content to the line
$this->upload_info = implode("\n", $lines); // Put string back together & update
return $addon; // Return line number
}
/**
* This is run at the end to display & save the log.
*/
private function handle_log()
{
global $page;
// Display message
$page->set_mode("data");
$page->set_type("text/plain");
$page->set_data($this->upload_info);
// Save log
$log_path = $this->root_dir . "/uploads.log";
if (file_exists($log_path)) {
$prev_content = file_get_contents($log_path);
} else {
$prev_content = "";
}
$content = $prev_content ."\r\n".$this->upload_info;
$content = $prev_content . "\r\n" . $this->upload_info;
file_put_contents($log_path, $content);
}
}