From 0202597f88b602d294117b32f06eacad9429f2b9 Mon Sep 17 00:00:00 2001 From: Matthew Barbour Date: Sat, 15 Jun 2019 11:01:13 -0500 Subject: [PATCH] Added lock file usage to cron uploader to prevent concurrent runs. Changed extension manager to allow author to be a comma-separated list. --- ext/cron_uploader/main.php | 187 ++++++++++++++++++++----------------- ext/ext_manager/main.php | 54 +++++++---- ext/ext_manager/theme.php | 41 ++++---- 3 files changed, 162 insertions(+), 120 deletions(-) diff --git a/ext/cron_uploader/main.php b/ext/cron_uploader/main.php index f4bd3c51..fe1bbfbf 100644 --- a/ext/cron_uploader/main.php +++ b/ext/cron_uploader/main.php @@ -1,42 +1,44 @@ + * Authors: YaoiFox , Matthew Barbour * Link: http://www.yaoifox.com/ * License: GPLv2 * Description: Uploads images automatically using Cron Jobs * Documentation: Installation guide: activate this extension and navigate to www.yoursite.com/cron_upload */ + class CronUploader extends Extension { // TODO: Checkbox option to only allow localhost + a list of additional IP adresses that can be set in /cron_upload // TODO: Change logging to MySQL + display log at /cron_upload // TODO: Move stuff to theme.php - + /** * Lists all log events this session * @var string */ private $upload_info = ""; - + /** * Lists all files & info required to upload. * @var array */ private $image_queue = []; - + /** * Cron Uploader root directory * @var string */ private $root_dir = ""; - + /** * Key used to identify uploader * @var string */ private $upload_key = ""; - + /** * Checks if the cron upload page has been accessed * and initializes the upload. @@ -44,39 +46,50 @@ class CronUploader extends Extension public function onPageRequest(PageRequestEvent $event) { global $config, $user; - + if ($event->page_matches("cron_upload")) { $this->upload_key = $config->get_string("cron_uploader_key", ""); - + // If the key is in the url, upload if ($this->upload_key != "" && $event->get_arg(0) == $this->upload_key) { // log in as admin - $this->process_upload(); // Start upload + $this->set_dir(); + + $lockfile = fopen($this->root_dir . "/.lock", "w"); + try { + if (!flock($lockfile, LOCK_EX | LOCK_NB)) { + throw new Exception("Cron upload process is already running"); + } + $this->process_upload(); // Start upload + } finally { + flock($lockfile, LOCK_UN); + fclose($lockfile); + } } elseif ($user->is_admin()) { $this->set_dir(); $this->display_documentation(); } } } - + private function display_documentation() { global $page; $this->set_dir(); // Determines path to cron_uploader_dir - - + + $queue_dir = $this->root_dir . "/queue"; $uploaded_dir = $this->root_dir . "/uploaded"; $failed_dir = $this->root_dir . "/failed_to_upload"; - + $queue_dirinfo = $this->scan_dir($queue_dir); $uploaded_dirinfo = $this->scan_dir($uploaded_dir); $failed_dirinfo = $this->scan_dir($failed_dir); - + $cron_url = make_http(make_link("/cron_upload/" . $this->upload_key)); $cron_cmd = "curl --silent $cron_url"; $log_path = $this->root_dir . "/uploads.log"; - + $info_html = "Information
@@ -105,7 +118,7 @@ class CronUploader extends Extension
Cron Command:
Create a cron job with the command above.
Read the documentation if you're not sure what to do.
"; - + $install_html = " This cron uploader is fairly easy to use but has to be configured first.
1. Install & activate this plugin. @@ -130,8 +143,10 @@ class CronUploader extends Extension
So when you want to manually upload an image, all you have to do is open the link once.
This link can be found under 'Cron Command' in the board config, just remove the 'wget ' part and only the url remains.
($cron_url)"; - - + + $page->set_title("Cron Uploader"); + $page->set_heading("Cron Uploader"); + $block = new Block("Cron Uploader", $info_html, "main", 10); $block_install = new Block("Installation Guide", $install_html, "main", 20); $page->add_block($block); @@ -143,35 +158,35 @@ class CronUploader extends Extension global $config; // Set default values $this->upload_key = $config->get_string("cron_uploader_key", ""); - if (strlen($this->upload_key)<=0) { + if (strlen($this->upload_key) <= 0) { $this->upload_key = $this->generate_key(); - + $config->set_default_int('cron_uploader_count', 1); $config->set_default_string('cron_uploader_key', $this->upload_key); $this->set_dir(); } } - + public function onSetupBuilding(SetupBuildingEvent $event) { $this->set_dir(); - + $cron_url = make_http(make_link("/cron_upload/" . $this->upload_key)); $cron_cmd = "curl --silent $cron_url"; $documentation_link = make_http(make_link("cron_upload")); - + $sb = new SetupBlock("Cron Uploader"); $sb->add_label("Settings
"); $sb->add_int_option("cron_uploader_count", "How many to upload each time"); $sb->add_text_option("cron_uploader_dir", "
Set Cron Uploader root directory
"); - + $sb->add_label("
Cron Command:
Create a cron job with the command above.
Read the documentation if you're not sure what to do."); $event->panel->add_block($sb); } - + /* * Generates a unique key for the website to prevent unauthorized access. */ @@ -180,14 +195,14 @@ class CronUploader extends Extension $length = 20; $characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; $randomString = ''; - - for ($i = 0; $i < $length; $i ++) { + + for ($i = 0; $i < $length; $i++) { $randomString .= $characters [rand(0, strlen($characters) - 1)]; } - + return $randomString; } - + /* * Set the directory for the image queue. If no directory was given, set it to the default directory. */ @@ -195,15 +210,15 @@ class CronUploader extends Extension { global $config; // Determine directory (none = default) - + $dir = $config->get_string("cron_uploader_dir", ""); - + // Sets new default dir if not in config yet/anymore if ($dir == "") { $dir = data_path("cron_uploader"); $config->set_string('cron_uploader_dir', $dir); } - + // Make the directory if it doesn't exist yet if (!is_dir($dir . "/queue/")) { mkdir($dir . "/queue/", 0775, true); @@ -214,31 +229,31 @@ class CronUploader extends Extension if (!is_dir($dir . "/failed_to_upload/")) { mkdir($dir . "/failed_to_upload/", 0775, true); } - + $this->root_dir = $dir; return $dir; } - + /** * Returns amount of files & total size of dir. */ public function scan_dir(string $path): array { - $bytestotal=0; - $nbfiles=0; + $bytestotal = 0; + $nbfiles = 0; - $ite=new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS); - foreach (new RecursiveIteratorIterator($ite) as $filename=>$cur) { + $ite = new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS); + foreach (new RecursiveIteratorIterator($ite) as $filename => $cur) { $filesize = $cur->getSize(); $bytestotal += $filesize; $nbfiles++; } - + $size_mb = $bytestotal / 1048576; // to mb $size_mb = number_format($size_mb, 2, '.', ''); - return ['total_files'=>$nbfiles,'total_mb'=>$size_mb]; + return ['total_files' => $nbfiles, 'total_mb' => $size_mb]; } - + /** * Uploads the image & handles everything */ @@ -246,24 +261,24 @@ class CronUploader extends Extension { global $config, $database; - set_time_limit(0); + //set_time_limit(0); - $output_subdir = date('Ymd-His', time())."/"; - $this->set_dir(); + + $output_subdir = date('Ymd-His', time()) . "/"; $this->generate_image_queue(); - + // Gets amount of imgs to upload if ($upload_count == 0) { $upload_count = $config->get_int("cron_uploader_count", 1); } - + // Throw exception if there's nothing in the queue if (count($this->image_queue) == 0) { $this->add_upload_info("Your queue is empty so nothing could be uploaded."); $this->handle_log(); return false; } - + // Randomize Images //shuffle($this->image_queue); @@ -274,15 +289,15 @@ class CronUploader extends Extension $failedItems = []; // Upload the file(s) - for ($i = 0; $i < $upload_count && sizeof($this->image_queue)>0; $i++) { + for ($i = 0; $i < $upload_count && sizeof($this->image_queue) > 0; $i++) { $img = array_pop($this->image_queue); - + try { $database->beginTransaction(); $result = $this->add_image($img[0], $img[1], $img[2]); $database->commit(); $this->move_uploaded($img[0], $img[1], $output_subdir, false); - if ($result==null) { + if ($result == null) { $merged++; } else { $added++; @@ -290,7 +305,7 @@ class CronUploader extends Extension } catch (Exception $e) { $failed++; $this->move_uploaded($img[0], $img[1], $output_subdir, true); - $msgNumber = $this->add_upload_info("(".gettype($e).") ".$e->getMessage()); + $msgNumber = $this->add_upload_info("(" . gettype($e) . ") " . $e->getMessage()); $msgNumber = $this->add_upload_info($e->getTraceAsString()); if (strpos($e->getMessage(), 'SQLSTATE') !== false) { // Postgres invalidates the transaction if there is an SQL error, @@ -310,40 +325,40 @@ class CronUploader extends Extension $msgNumber = $this->add_upload_info("Items failed: $failed"); - // Display & save upload log $this->handle_log(); - + return true; + } - + private function move_uploaded($path, $filename, $output_subdir, $corrupt = false) { global $config; - + // Create $newDir = $this->root_dir; - + $relativeDir = dirname(substr($path, strlen($this->root_dir) + 7)); // Determine which dir to move to if ($corrupt) { // Move to corrupt dir - $newDir .= "/failed_to_upload/".$output_subdir.$relativeDir; + $newDir .= "/failed_to_upload/" . $output_subdir . $relativeDir; $info = "ERROR: Image was not uploaded."; } else { - $newDir .= "/uploaded/".$output_subdir.$relativeDir; + $newDir .= "/uploaded/" . $output_subdir . $relativeDir; $info = "Image successfully uploaded. "; } - $newDir = str_replace("//", "/", $newDir."/"); + $newDir = str_replace("//", "/", $newDir . "/"); if (!is_dir($newDir)) { mkdir($newDir, 0775, true); } // move file to correct dir - rename($path, $newDir.$filename); - + rename($path, $newDir . $filename); + $this->add_upload_info($info . "Image \"$filename\" moved from queue to \"$newDir\"."); } @@ -353,7 +368,7 @@ class CronUploader extends Extension private function add_image(string $tmpname, string $filename, string $tags) { assert(file_exists($tmpname)); - + $pathinfo = pathinfo($filename); $metadata = []; $metadata ['filename'] = $pathinfo ['basename']; @@ -364,7 +379,7 @@ class CronUploader extends Extension $metadata ['source'] = null; $event = new DataUploadEvent($tmpname, $metadata); send_event($event); - + // Generate info message $infomsg = ""; // Will contain info message if ($event->image_id == -1) { @@ -377,18 +392,18 @@ class CronUploader extends Extension $msgNumber = $this->add_upload_info($infomsg); return $event->image_id; } - + private function generate_image_queue(): void { $base = $this->root_dir . "/queue"; - - if (! is_dir($base)) { + + if (!is_dir($base)) { $this->add_upload_info("Image Queue Directory could not be found at \"$base\"."); return; } - - $ite=new RecursiveDirectoryIterator($base, FilesystemIterator::SKIP_DOTS); - foreach (new RecursiveIteratorIterator($ite) as $fullpath=>$cur) { + + $ite = new RecursiveDirectoryIterator($base, FilesystemIterator::SKIP_DOTS); + foreach (new RecursiveIteratorIterator($ite) as $fullpath => $cur) { if (!is_link($fullpath) && !is_dir($fullpath)) { $pathinfo = pathinfo($fullpath); @@ -396,62 +411,62 @@ class CronUploader extends Extension $tags = path_to_tags($relativePath); $img = [ - 0 => $fullpath, - 1 => $pathinfo ["basename"], - 2 => $tags + 0 => $fullpath, + 1 => $pathinfo ["basename"], + 2 => $tags ]; array_push($this->image_queue, $img); } } } - + /** * Adds a message to the info being published at the end */ private function add_upload_info(string $text, int $addon = 0): int { $info = $this->upload_info; - $time = "[" .date('Y-m-d H:i:s'). "]"; - + $time = "[" . date('Y-m-d H:i:s') . "]"; + // If addon function is not used if ($addon == 0) { - $this->upload_info .= "$time $text\r\n"; - + $this->upload_info .= "$time $text\r\n"; + // Returns the number of the current line - $currentLine = substr_count($this->upload_info, "\n") -1; + $currentLine = substr_count($this->upload_info, "\n") - 1; return $currentLine; } - + // else if addon function is used, select the line & modify it $lines = substr($info, "\n"); // Seperate the string to array in lines $lines[$addon] = "$lines[$addon] $text"; // Add the content to the line $this->upload_info = implode("\n", $lines); // Put string back together & update - + return $addon; // Return line number } - + /** * This is run at the end to display & save the log. */ private function handle_log() { global $page; - + // Display message $page->set_mode("data"); $page->set_type("text/plain"); $page->set_data($this->upload_info); - + // Save log $log_path = $this->root_dir . "/uploads.log"; - + if (file_exists($log_path)) { $prev_content = file_get_contents($log_path); } else { $prev_content = ""; } - - $content = $prev_content ."\r\n".$this->upload_info; + + $content = $prev_content . "\r\n" . $this->upload_info; file_put_contents($log_path, $content); } } diff --git a/ext/ext_manager/main.php b/ext/ext_manager/main.php index f9ef6bba..dddefc70 100644 --- a/ext/ext_manager/main.php +++ b/ext/ext_manager/main.php @@ -22,8 +22,7 @@ class ExtensionInfo public $ext_name; public $name; public $link; - public $author; - public $email; + public $authors; public $description; public $documentation; public $version; @@ -39,8 +38,9 @@ class ExtensionInfo $this->ext_name = $matches[1]; $this->name = $this->ext_name; $this->enabled = $this->is_enabled($this->ext_name); + $this->authors = []; - for ($i=0; $i<$number_of_lines; $i++) { + for ($i = 0; $i < $number_of_lines; $i++) { $line = $lines[$i]; if (preg_match("/Name: (.*)/", $line, $matches)) { $this->name = $matches[1]; @@ -53,25 +53,31 @@ class ExtensionInfo } } elseif (preg_match("/Version: (.*)/", $line, $matches)) { $this->version = $matches[1]; - } elseif (preg_match("/Author: (.*) [<\(](.*@.*)[>\)]/", $line, $matches)) { - $this->author = $matches[1]; - $this->email = $matches[2]; - } elseif (preg_match("/Author: (.*)/", $line, $matches)) { - $this->author = $matches[1]; + } elseif (preg_match("/Authors?: (.*)/", $line, $matches)) { + $author_list = explode(',', $matches[1]); + foreach ($author_list as $author) { + if (preg_match("/(.*) [<\(](.*@.*)[>\)]/", $author, $matches)) { + $this->authors[] = new ExtensionAuthor($matches[1], $matches[2]); + } else { + $this->authors[] = new ExtensionAuthor($author, null); + } + } + + } elseif (preg_match("/(.*)Description: ?(.*)/", $line, $matches)) { $this->description = $matches[2]; - $start = $matches[1]." "; + $start = $matches[1] . " "; $start_len = strlen($start); - while (substr($lines[$i+1], 0, $start_len) == $start) { - $this->description .= " ".substr($lines[$i+1], $start_len); + while (substr($lines[$i + 1], 0, $start_len) == $start) { + $this->description .= " " . substr($lines[$i + 1], $start_len); $i++; } } elseif (preg_match("/(.*)Documentation: ?(.*)/", $line, $matches)) { $this->documentation = $matches[2]; - $start = $matches[1]." "; + $start = $matches[1] . " "; $start_len = strlen($start); - while (substr($lines[$i+1], 0, $start_len) == $start) { - $this->documentation .= " ".substr($lines[$i+1], $start_len); + while (substr($lines[$i + 1], 0, $start_len) == $start) { + $this->documentation .= " " . substr($lines[$i + 1], $start_len); $i++; } $this->documentation = str_replace('$site', make_http(get_base_href()), $this->documentation); @@ -96,6 +102,18 @@ class ExtensionInfo } } +class ExtensionAuthor +{ + public $name; + public $email; + + public function __construct(string $name, string $email) + { + $this->name = $name; + $this->email = $email; + } +} + class ExtManager extends Extension { public function onPageRequest(PageRequestEvent $event) @@ -166,7 +184,7 @@ class ExtManager extends Extension if ($all) { $exts = zglob("ext/*/main.php"); } else { - $exts = zglob("ext/{".ENABLED_EXTS."}/main.php"); + $exts = zglob("ext/{" . ENABLED_EXTS . "}/main.php"); } foreach ($exts as $main) { $extensions[] = new ExtensionInfo($main); @@ -200,9 +218,9 @@ class ExtManager extends Extension { file_put_contents( "data/config/extensions.conf.php", - '<'.'?php'."\n". - 'define("EXTRA_EXTS", "'.implode(",", $extras).'");'."\n". - '?'.">" + '<' . '?php' . "\n" . + 'define("EXTRA_EXTS", "' . implode(",", $extras) . '");' . "\n" . + '?' . ">" ); // when the list of active extensions changes, we can be diff --git a/ext/ext_manager/theme.php b/ext/ext_manager/theme.php index 3cc6d871..58bd79ab 100644 --- a/ext/ext_manager/theme.php +++ b/ext/ext_manager/theme.php @@ -9,7 +9,7 @@ class ExtManagerTheme extends Themelet { $h_en = $editable ? "" : ""; $html = " - ".make_form(make_link("ext_manager/set"))." + " . make_form(make_link("ext_manager/set")) . "
Enabled
@@ -26,17 +26,17 @@ class ExtManagerTheme extends Themelet continue; } - $h_name = html_escape(empty($extension->name) ? $extension->ext_name : $extension->name); + $h_name = html_escape(empty($extension->name) ? $extension->ext_name : $extension->name); $h_description = html_escape($extension->description); - $h_link = make_link("ext_doc/".url_escape($extension->ext_name)); - $h_enabled = ($extension->enabled === true ? " checked='checked'" : ($extension->enabled === false ? "" : " disabled checked='checked'")); - $h_enabled_box = $editable ? "" : ""; - $h_docs = ($extension->documentation ? "■" : ""); //TODO: A proper "docs" symbol would be preferred here. + $h_link = make_link("ext_doc/" . url_escape($extension->ext_name)); + $h_enabled = ($extension->enabled === true ? " checked='checked'" : ($extension->enabled === false ? "" : " disabled checked='checked'")); + $h_enabled_box = $editable ? "" : ""; + $h_docs = ($extension->documentation ? "■" : ""); //TODO: A proper "docs" symbol would be preferred here. $html .= " {$h_enabled_box} - + "; @@ -116,15 +116,24 @@ class ExtManagerTheme extends Themelet public function display_doc(Page $page, ExtensionInfo $info) { $author = ""; - if ($info->author) { - if ($info->email) { - $author = "
Author:email)."\">".html_escape($info->author).""; - } else { - $author = "
Author: ".html_escape($info->author); + if (count($info->authors) > 0) { + $author = "
Author"; + if (count($info->authors) > 1) { + $author .= "s"; } + $author .= ":"; + foreach ($info->authors as $auth) { + if (!empty($auth->email)) { + $author .= "email) . "\">" . html_escape($auth->name) . ""; + } else { + $author .= html_escape($auth->name); + } + } + } - $version = ($info->version) ? "
Version: ".html_escape($info->version) : ""; - $link = ($info->link) ? "
Home Page:link)."\">Link" : ""; + + $version = ($info->version) ? "
Version: " . html_escape($info->version) : ""; + $link = ($info->link) ? "
Home Page:link) . "\">Link" : ""; $doc = $info->documentation; $html = "
@@ -133,10 +142,10 @@ class ExtManagerTheme extends Themelet $link

$doc

-

Back to the list +

Back to the list

"; - $page->set_title("Documentation for ".html_escape($info->name)); + $page->set_title("Documentation for " . html_escape($info->name)); $page->set_heading(html_escape($info->name)); $page->add_block(new NavBlock()); $page->add_block(new Block("Documentation", $html));
{$h_docs} {$h_description}