New bulk import/export extension

core/imageboard/misc.php

@@ -39,7 +39,7 @@ function add_dir(string $base): array
  * @param string $tags
  * @throws UploadException
  */
-function add_image(string $tmpname, string $filename, string $tags): void
+function add_image(string $tmpname, string $filename, string $tags): int
 {
     assert(file_exists($tmpname));
 
@@ -52,7 +52,11 @@ function add_image(string $tmpname, string $filename, string $tags): void
 
     $metadata['tags'] = Tag::explode($tags);
     $metadata['source'] = null;
-    send_event(new DataUploadEvent($tmpname, $metadata));
+
+    $due = new DataUploadEvent($tmpname, $metadata);
+    send_event($due);
+
+    return $due->image_id;
 }
 
 /**

core/permissions.php

@@ -96,4 +96,7 @@ abstract class Permissions
     public const CRON_ADMIN = "cron_admin";
     public const APPROVE_IMAGE = "approve_image";
     public const APPROVE_COMMENT = "approve_comment";
+    
+    public const BULK_IMPORT = "bulk_import";
+    public const BULK_EXPORT = "bulk_export";
 }

core/userclass.php

@@ -197,6 +197,9 @@ new UserClass("admin", "base", [
 
     Permissions::APPROVE_IMAGE => true,
     Permissions::APPROVE_COMMENT => true,
+
+    Permissions::BULK_IMPORT =>true,
+    Permissions::BULK_EXPORT =>true,
 ]);
 
 @include_once "data/config/user-classes.conf.php";

ext/bulk_import_export/events.php

@@ -0,0 +1,24 @@
+<?php declare(strict_types=1);
+
+class BulkExportEvent extends Event
+{
+    public $image;
+    public $fields = [];
+
+    public function __construct(Image $image)
+    {
+        $this->image = $image;
+    }
+}
+
+
+class BulkImportEvent extends Event
+{
+    public $image;
+
+    public function __construct(Image $image, $fields)
+    {
+        $this->image = $image;
+        $this->fields = $fields;
+    }
+}

ext/bulk_import_export/info.php

@@ -0,0 +1,15 @@
+<?php declare(strict_types=1);
+
+include_once "events.php";
+
+class BulkImportExportInfo extends ExtensionInfo
+{
+    public const KEY = "bulk_import_export";
+
+    public $key = self::KEY;
+    public $name = "Bulk Import/Export";
+    public $authors = ["Matthew Barbour"=>"matthew@darkholme.net"];
+    public $license = self::LICENSE_WTFPL;
+    public $description = "Allows bulk exporting/importing of images and associated data.";
+    public $dependencies = [BulkActionsInfo::KEY];
+}

ext/bulk_import_export/main.php

@@ -0,0 +1,154 @@
+<?php declare(strict_types=1);
+
+
+class BulkImportExport extends DataHandlerExtension
+{
+    const EXPORT_ACTION_NAME = "bulk_export";
+    const EXPORT_INFO_FILE_NAME = "export.json";
+    protected $SUPPORTED_MIME = [MIME_TYPE_ZIP];
+
+
+    public function onDataUpload(DataUploadEvent $event)
+    {
+        global $user;
+
+        if ($this->supported_ext($event->type) &&
+            $user->can(Permissions::BULK_IMPORT)) {
+            $zip = new ZipArchive;
+
+            if ($zip->open($event->tmpname) === true) {
+                $info = $zip->getStream(self::EXPORT_INFO_FILE_NAME);
+                $json_data = [];
+                if ($info !== false) {
+                    try {
+                        $json_string = stream_get_contents($info);
+                        $json_data = json_decode($json_string);
+                    } finally {
+                        fclose($info);
+                    }
+                } else {
+                    throw new SCoreException("Could not get " . self::EXPORT_INFO_FILE_NAME . " from archive");
+                }
+                $total = 0;
+
+                while (!empty($json_data)) {
+                    $item = array_pop($json_data);
+                    $image = Image::by_hash($item->hash);
+                    if ($image!=null) {
+                        continue;
+                    }
+
+                    $tmpfile = tempnam("/tmp", "shimmie_bulk_import");
+                    $stream = $zip->getStream($item->hash);
+                    if ($zip === false) {
+                        log_error("BulkImportExport", "Could not import " . $item->hash . ": File not in zip", "Could not import " . $item->hash . ": File not in zip");
+                        continue;
+                    }
+
+                    try {
+                        file_put_contents($tmpfile, $stream);
+
+                        $id = add_image($tmpfile, $item->filename, Tag::implode($item->tags));
+
+                        if ($id==-1) {
+                            throw new SCoreException("Unable to import file $item->hash");
+                        }
+
+                        $image = Image::by_id($id);
+
+                        if ($image==null) {
+                            throw new SCoreException("Unable to import file $item->hash");
+                        }
+
+                        if ($item->source!=null) {
+                            $image->set_source($item->source);
+                        }
+                        send_event(new BulkImportEvent($image, $item));
+
+                        $total++;
+                    } catch (Exception $ex) {
+                        throw new SCoreException("Could not import " . $item->hash . ": " . $ex->getMessage());
+                        //log_error("BulkImportExport", "Could not import " . $item->hash . ": " . $ex->getMessage(), "Could not import " . $item->hash . ": " . $ex->getMessage());
+                        //continue;
+                    } finally {
+                        if (is_file($tmpfile)) {
+                            unlink($tmpfile);
+                        }
+                    }
+                }
+                $event->image_id = -2; // default -1 = upload wasn't handled
+
+                log_info("BulkImportExport", "Imported " . $total . " items", "Imported " . $total . " items");
+            } else {
+                throw new SCoreException("Could not open zip archive");
+            }
+        }
+    }
+
+
+
+    public function onBulkActionBlockBuilding(BulkActionBlockBuildingEvent $event)
+    {
+        global $user, $config;
+
+        if ($user->can(Permissions::BULK_EXPORT)) {
+            $event->add_action(self::EXPORT_ACTION_NAME, "Export");
+        }
+    }
+
+    public function onBulkAction(BulkActionEvent $event)
+    {
+        global $user, $page;
+
+        if ($user->can(Permissions::BULK_EXPORT) &&
+            ($event->action == self::EXPORT_ACTION_NAME)) {
+            $zip_filename = data_path($user->name . '-' . date('YmdHis') . '.zip');
+            $zip = new ZipArchive;
+
+            $json_data = [];
+
+            if ($zip->open($zip_filename, ZIPARCHIVE::CREATE | ZIPARCHIVE::OVERWRITE) === true) {
+                foreach ($event->items as $image) {
+                    $img_loc = warehouse_path(Image::IMAGE_DIR, $image->hash, false);
+
+                    $export_event = new BulkExportEvent($image);
+                    send_event($export_event);
+                    $data = $export_event->fields;
+                    $data["hash"] = $image->hash;
+                    $data["tags"] = $image->get_tag_array();
+                    $data["filename"] = $image->filename;
+                    $data["source"] = $image->source;
+
+                    array_push($json_data, $data);
+
+                    $zip->addFile($img_loc, $image->hash);
+                }
+
+                $json_data = json_encode($json_data, JSON_PRETTY_PRINT);
+                $zip->addFromString(self::EXPORT_INFO_FILE_NAME, $json_data);
+
+                $zip->close();
+
+                $page->set_mode(PageMode::FILE);
+                $page->set_file($zip_filename, true);
+                $page->set_filename(basename($zip_filename));
+
+                $event->redirect = false;
+            }
+        }
+    }
+    // we don't actually do anything, just accept one upload and spawn several
+    protected function media_check_properties(MediaCheckPropertiesEvent $event): void
+    {
+    }
+
+    protected function check_contents(string $tmpname): bool
+    {
+        return false;
+    }
+
+    protected function create_thumb(string $hash, string $type): bool
+    {
+        return false;
+    }
+}

ext/post_titles/main.php

@@ -73,7 +73,16 @@ class PostTitles extends Extension
         $event->panel->add_block($sb);
     }
 
-
+    public function onBulkExport(BulkExportEvent $event)
+    {
+        $event->fields["title"] = $event->image->title;
+    }
+    public function onBulkImport(BulkImportEvent $event)
+    {
+        if (property_exists($event->fields, "title") && $event->fields->title!=null) {
+            $this->set_title($event->image->id, $event->fields->title);
+        }
+    }
 
     private function set_title(int $image_id, string $title)
     {

ext/rating/main.php

@@ -170,6 +170,19 @@ class Ratings extends Extension
         }
     }
 
+    public function onBulkExport(BulkExportEvent $event)
+    {
+        $event->fields["rating"] = $event->image->rating;
+    }
+    public function onBulkImport(BulkImportEvent $event)
+    {
+        if (property_exists($event->fields, "rating")
+            && $event->fields->rating != null
+            && Ratings::rating_is_valid($event->fields->rating)) {
+            $this->set_rating($event->image->id, $event->fields->rating, "");
+        }
+    }
+
     public function onRatingSet(RatingSetEvent $event)
     {
         if (empty($event->image->rating)) {