james/fembooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Changes to cron upload:

Browse Source 
Added transaction handling so that subsequent errors don't result in images that have already moved to the uploaded folder from being wiped from the database.
Changed output folders to use subfolders based on the timestamp of the current run. This is to prevent writing over files in the error folder that happen to have the same name and path, effectively losing the data.
Added additional error and information logging, and a final count of imported/merged/failed.
This commit is contained in:
Matthew Barbour 2019-06-12 17:50:00 -05:00
parent b27904a7e0
commit cb24ac69ab
1 changed files with 44 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
ext/cron_uploader/main.php
View File

@ -244,8 +244,11 @@ class CronUploader extends Extension
*/ */
public function process_upload(int $upload_count = 0): bool public function process_upload(int $upload_count = 0): bool
{ {
global $config; global $config, $database;
set_time_limit(0); set_time_limit(0);
$output_subdir = date('Ymd-His', time())."/";
$this->set_dir(); $this->set_dir();
$this->generate_image_queue(); $this->generate_image_queue();
@ -262,32 +265,58 @@ class CronUploader extends Extension
} }
// Randomize Images // Randomize Images
shuffle($this->image_queue); //shuffle($this->image_queue);
$merged = 0;
$added = 0;
$failed = 0;
$failedItems = [];
// Upload the file(s) // Upload the file(s)
for ($i = 0; $i < $upload_count && sizeof($this->image_queue)>0; $i++) { for ($i = 0; $i < $upload_count && sizeof($this->image_queue)>0; $i++) {
$img = array_pop($this->image_queue); $img = array_pop($this->image_queue);
try { try {
$this->add_image($img[0], $img[1], $img[2]); $database->beginTransaction();
$this->move_uploaded($img[0], $img[1], false); $result = $this->add_image($img[0], $img[1], $img[2]);
$database->commit();
$this->move_uploaded($img[0], $img[1], $output_subdir, false);
if($result==null) {
$merged++;
} else {
$added++;
}
} catch (Exception $e) { } catch (Exception $e) {
$this->move_uploaded($img[0], $img[1], true); $failed++;
$this->move_uploaded($img[0], $img[1], $output_subdir, true);
$msgNumber = $this->add_upload_info("(".gettype($e).") ".$e->getMessage());
$msgNumber = $this->add_upload_info($e->getTraceAsString());
if (strpos($e->getMessage(), 'SQLSTATE') !== false) { if (strpos($e->getMessage(), 'SQLSTATE') !== false) {
// Postgres invalidates the transaction if there is an SQL error, // Postgres invalidates the transaction if there is an SQL error,
// so all subsequence transactions will fail. // so all subsequence transactions will fail.
break; break;
} }
try {
$database->rollback();
} catch (Exception $e) {}
} }
} }
$msgNumber = $this->add_upload_info("Items added: $added");
$msgNumber = $this->add_upload_info("Items merged: $merged");
$msgNumber = $this->add_upload_info("Items failed: $failed");
// Display & save upload log // Display & save upload log
$this->handle_log(); $this->handle_log();
return true; return true;
} }
private function move_uploaded($path, $filename, $corrupt = false) private function move_uploaded($path, $filename, $output_subdir, $corrupt = false)
{ {
global $config; global $config;
@ -299,11 +328,11 @@ class CronUploader extends Extension
// Determine which dir to move to // Determine which dir to move to
if ($corrupt) { if ($corrupt) {
// Move to corrupt dir // Move to corrupt dir
$newDir .= "/failed_to_upload/".$relativeDir; $newDir .= "/failed_to_upload/".$output_subdir.$relativeDir;
$info = "ERROR: Image was not uploaded."; $info = "ERROR: Image was not uploaded.";
} }
else { else {
$newDir .= "/uploaded/".$relativeDir; $newDir .= "/uploaded/".$output_subdir.$relativeDir;
$info = "Image successfully uploaded. "; $info = "Image successfully uploaded. ";
} }
$newDir = str_replace ( "//", "/", $newDir."/" ); $newDir = str_replace ( "//", "/", $newDir."/" );
@ -327,7 +356,9 @@ class CronUploader extends Extension
$pathinfo = pathinfo($filename); $pathinfo = pathinfo($filename);
$metadata = []; $metadata = [];
$metadata ['filename'] = $pathinfo ['basename']; $metadata ['filename'] = $pathinfo ['basename'];
$metadata ['extension'] = $pathinfo ['extension']; if (array_key_exists('extension', $pathinfo)) {
$metadata ['extension'] = $pathinfo ['extension'];
}
$metadata ['tags'] = Tag::explode($tags); $metadata ['tags'] = Tag::explode($tags);
$metadata ['source'] = null; $metadata ['source'] = null;
$event = new DataUploadEvent($tmpname, $metadata); $event = new DataUploadEvent($tmpname, $metadata);
@ -337,11 +368,13 @@ class CronUploader extends Extension
$infomsg = ""; // Will contain info message $infomsg = ""; // Will contain info message
if ($event->image_id == -1) { if ($event->image_id == -1) {
throw new Exception("File type not recognised. Filename: {$filename}"); throw new Exception("File type not recognised. Filename: {$filename}");
} else if ($event->image_id == null) {
$infomsg = "Image merged. Filename: {$filename}";
} else { } else {
$infomsg = "Image uploaded. ID: {$event->image_id} - Filename: {$filename} - Tags: {$tags}"; $infomsg = "Image uploaded. ID: {$event->image_id} - Filename: {$filename}";
} }
$msgNumber = $this->add_upload_info($infomsg); $msgNumber = $this->add_upload_info($infomsg);
return $event->image_id;
} }
private function generate_image_queue(): void private function generate_image_queue(): void