From cb24ac69abf0a64542b9f8854daf7e773737417f Mon Sep 17 00:00:00 2001
From: Matthew Barbour <matthew@darkholme.net>
Date: Wed, 12 Jun 2019 17:50:00 -0500
Subject: [PATCH] Changes to cron upload: Added transaction handling so that
subsequent errors don't result in images that have already moved to the
uploaded folder from being wiped from the database. Changed output folders to
use subfolders based on the timestamp of the current run. This is to prevent
writing over files in the error folder that happen to have the same name and
path, effectively losing the data. Added additional error and information
logging, and a final count of imported/merged/failed.
---
ext/cron_uploader/main.php | 55 ++++++++++++++++++++++++++++++--------
1 file changed, 44 insertions(+), 11 deletions(-)
diff --git a/ext/cron_uploader/main.php b/ext/cron_uploader/main.php
index e33b1dba..65ad135a 100644
--- a/ext/cron_uploader/main.php
+++ b/ext/cron_uploader/main.php
@@ -244,8 +244,11 @@ class CronUploader extends Extension
*/
public function process_upload(int $upload_count = 0): bool
{
- global $config;
+ global $config, $database;
+
set_time_limit(0);
+
+ $output_subdir = date('Ymd-His', time())."/";
$this->set_dir();
$this->generate_image_queue();
@@ -262,24 +265,50 @@ class CronUploader extends Extension
}
// Randomize Images
- shuffle($this->image_queue);
+ //shuffle($this->image_queue);
+
+ $merged = 0;
+ $added = 0;
+ $failed = 0;
+
+ $failedItems = [];
// Upload the file(s)
for ($i = 0; $i < $upload_count && sizeof($this->image_queue)>0; $i++) {
$img = array_pop($this->image_queue);
try {
- $this->add_image($img[0], $img[1], $img[2]);
- $this->move_uploaded($img[0], $img[1], false);
+ $database->beginTransaction();
+ $result = $this->add_image($img[0], $img[1], $img[2]);
+ $database->commit();
+ $this->move_uploaded($img[0], $img[1], $output_subdir, false);
+ if($result==null) {
+ $merged++;
+ } else {
+ $added++;
+ }
} catch (Exception $e) {
- $this->move_uploaded($img[0], $img[1], true);
+ $failed++;
+ $this->move_uploaded($img[0], $img[1], $output_subdir, true);
+ $msgNumber = $this->add_upload_info("(".gettype($e).") ".$e->getMessage());
+ $msgNumber = $this->add_upload_info($e->getTraceAsString());
if (strpos($e->getMessage(), 'SQLSTATE') !== false) {
// Postgres invalidates the transaction if there is an SQL error,
// so all subsequence transactions will fail.
break;
}
+ try {
+ $database->rollback();
+ } catch (Exception $e) {}
}
}
+
+
+ $msgNumber = $this->add_upload_info("Items added: $added");
+ $msgNumber = $this->add_upload_info("Items merged: $merged");
+ $msgNumber = $this->add_upload_info("Items failed: $failed");
+
+
// Display & save upload log
$this->handle_log();
@@ -287,7 +316,7 @@ class CronUploader extends Extension
return true;
}
- private function move_uploaded($path, $filename, $corrupt = false)
+ private function move_uploaded($path, $filename, $output_subdir, $corrupt = false)
{
global $config;
@@ -299,11 +328,11 @@ class CronUploader extends Extension
// Determine which dir to move to
if ($corrupt) {
// Move to corrupt dir
- $newDir .= "/failed_to_upload/".$relativeDir;
+ $newDir .= "/failed_to_upload/".$output_subdir.$relativeDir;
$info = "ERROR: Image was not uploaded.";
}
else {
- $newDir .= "/uploaded/".$relativeDir;
+ $newDir .= "/uploaded/".$output_subdir.$relativeDir;
$info = "Image successfully uploaded. ";
}
$newDir = str_replace ( "//", "/", $newDir."/" );
@@ -327,7 +356,9 @@ class CronUploader extends Extension
$pathinfo = pathinfo($filename);
$metadata = [];
$metadata ['filename'] = $pathinfo ['basename'];
- $metadata ['extension'] = $pathinfo ['extension'];
+ if (array_key_exists('extension', $pathinfo)) {
+ $metadata ['extension'] = $pathinfo ['extension'];
+ }
$metadata ['tags'] = Tag::explode($tags);
$metadata ['source'] = null;
$event = new DataUploadEvent($tmpname, $metadata);
@@ -337,11 +368,13 @@ class CronUploader extends Extension
$infomsg = ""; // Will contain info message
if ($event->image_id == -1) {
throw new Exception("File type not recognised. Filename: {$filename}");
+ } else if ($event->image_id == null) {
+ $infomsg = "Image merged. Filename: {$filename}";
} else {
- $infomsg = "Image uploaded. ID: {$event->image_id} - Filename: {$filename} - Tags: {$tags}";
+ $infomsg = "Image uploaded. ID: {$event->image_id} - Filename: {$filename}";
}
$msgNumber = $this->add_upload_info($infomsg);
-
+ return $event->image_id;
}
private function generate_image_queue(): void