Added ability to use generators with database queries.

Adapted bulk actions to use generators.

core/database.php

@@ -249,6 +249,17 @@ class Database
         return $data;
     }
 
+    /**
+     * Execute an SQL query and return a iterable object for use with generators.
+     */
+    public function get_all_iterable(string $query, array $args=[]): PDOStatement
+    {
+        $_start = microtime(true);
+        $data = $this->execute($query, $args);
+        $this->count_time("get_all_iterable", $_start);
+        return $data;
+    }
+
     /**
      * Execute an SQL query and return a single row.
      */
@@ -276,7 +287,20 @@ class Database
     }
 
     /**
-     * Execute an SQL query and return the the first row => the second row.
+     * Execute an SQL query and return the first column of each row as a single iterable object.
+     */
+    public function get_col_iterable(string $query, array $args=[]): Generator
+    {
+        $_start = microtime(true);
+        $stmt = $this->execute($query, $args);
+        $this->count_time("get_col_iterable", $_start);
+        foreach ($stmt as $row) {
+            yield $row[0];
+        }
+    }
+
+    /**
+     * Execute an SQL query and return the the first column => the second column.
      */
     public function get_pairs(string $query, array $args=[]): array
     {

core/imageboard/image.php

@@ -10,7 +10,6 @@
  */
 class Image
 {
-    public const DATA_DIR = "data";
     public const IMAGE_DIR = "images";
     public const THUMBNAIL_DIR = "thumbs";
 
@@ -104,6 +103,43 @@ class Image
         }
     }
 
+
+    private static function find_images_internal(int $start = 0, ?int $limit = null, array $tags=[]): iterable
+    {
+        global $database, $user, $config;
+
+        if ($start < 0) {
+            $start = 0;
+        }
+        if ($limit!=null && $limit < 1) {
+            $limit = 1;
+        }
+
+        if (SPEED_HAX) {
+            if (!$user->can("big_search") and count($tags) > 3) {
+                throw new SCoreException("Anonymous users may only search for up to 3 tags at a time");
+            }
+        }
+
+        list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags);
+
+        $result = Image::get_accelerated_result($tag_conditions, $img_conditions, $start, $limit);
+        if (!$result) {
+            $querylet = Image::build_search_querylet($tag_conditions, $img_conditions);
+            $querylet->append(new Querylet(" ORDER BY ".(Image::$order_sql ?: "images.".$config->get_string("index_order"))));
+            if($limit!=null) {
+                $querylet->append(new Querylet(" LIMIT :limit ", ["limit" => $limit]));
+            }
+            $querylet->append(new Querylet(" OFFSET :offset ", ["offset"=>$start]));
+            #var_dump($querylet->sql); var_dump($querylet->variables);
+            $result = $database->get_all_iterable($querylet->sql, $querylet->variables);
+        }
+
+        Image::$order_sql = null;
+
+        return $result;
+    }
+
     /**
      * Search for an array of images
      *
@@ -112,82 +148,24 @@ class Image
      */
     public static function find_images(int $start, int $limit, array $tags=[]): array
     {
-        global $database, $user, $config;
+        $result = self::find_images_internal($start, $limit, $tags);
 
         $images = [];
-
+        foreach ($result as $row) {
-        if ($start < 0) {
-            $start = 0;
-        }
-        if ($limit < 1) {
-            $limit = 1;
-        }
-
-        if (SPEED_HAX) {
-            if (!$user->can("big_search") and count($tags) > 3) {
-                throw new SCoreException("Anonymous users may only search for up to 3 tags at a time");
-            }
-        }
-
-        list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags);
-
-        $result = Image::get_accelerated_result($tag_conditions, $img_conditions, $start, $limit);
-        if (!$result) {
-            $querylet = Image::build_search_querylet($tag_conditions, $img_conditions);
-            $querylet->append(new Querylet(" ORDER BY ".(Image::$order_sql ?: "images.".$config->get_string("index_order"))));
-            $querylet->append(new Querylet(" LIMIT :limit OFFSET :offset", ["limit"=>$limit, "offset"=>$start]));
-            #var_dump($querylet->sql); var_dump($querylet->variables);
-            $result = $database->execute($querylet->sql, $querylet->variables);
-        }
-
-        while ($row = $result->fetch()) {
             $images[] = new Image($row);
         }
-        Image::$order_sql = null;
         return $images;
     }
 
     /**
-     * Search for an array of image IDs
+     * Search for an array of images, returning a iterable object of Image
-     *
-     * #param string[] $tags
-     * #return int[]
      */
-    public static function find_image_ids(int $start, int $limit, array $tags=[]): array
+    public static function find_images_iterable(int $start = 0, ?int $limit = null, array $tags=[]): Generator
     {
-        global $database, $user, $config;
+        $result = self::find_images_internal($start, $limit, $tags);
-
+        foreach ($result as $row) {
-        $images = [];
+            yield new Image($row);
-
-        if ($start < 0) {
-            $start = 0;
         }
-        if ($limit < 1) {
-            $limit = 1;
-        }
-
-        if (SPEED_HAX) {
-            if (!$user->can("big_search") and count($tags) > 3) {
-                throw new SCoreException("Anonymous users may only search for up to 3 tags at a time");
-            }
-        }
-
-        list($tag_conditions, $img_conditions) = self::terms_to_conditions($tags);
-
-        $result = Image::get_accelerated_result($tag_conditions, $img_conditions, $start, $limit);
-        if (!$result) {
-            $querylet = Image::build_search_querylet($tag_conditions, $img_conditions);
-            $querylet->append(new Querylet(" ORDER BY ".(Image::$order_sql ?: "images.".$config->get_string("index_order"))));
-            $querylet->append(new Querylet(" LIMIT :limit OFFSET :offset", ["limit"=>$limit, "offset"=>$start]));
-            #var_dump($querylet->sql); var_dump($querylet->variables);
-            $result = $database->execute($querylet->sql, $querylet->variables);
-        }
-
-        while ($row = $result->fetch()) {
-            $images[] = $row["id"];
-        }
-        Image::$order_sql = null;
-        return $images;
     }
 
     /*
@@ -220,7 +198,7 @@ class Image
         return null;
     }
 
-    public static function get_accelerated_result(array $tag_conditions, array $img_conditions, int $offset, int $limit): ?PDOStatement
+    public static function get_accelerated_result(array $tag_conditions, array $img_conditions, int $offset, ?int $limit): ?PDOStatement
     {
         if (!SEARCH_ACCEL || !empty($img_conditions)) {
             return null;

core/imageboard/misc.php

@@ -501,3 +501,8 @@ function is_animated_gif(String $image_filename)
     }
     return ($is_anim_gif == 0);
 }
+
+function image_to_id(Image $image): int
+{
+    return $image->id;
+}

ext/bulk_actions/main.php

@@ -51,7 +51,7 @@ class BulkActionEvent extends Event
     /** @var PageRequestEvent  */
     public $page_request;
 
-    public function __construct(String $action, PageRequestEvent $pageRequestEvent, array $items)
+    public function __construct(String $action, PageRequestEvent $pageRequestEvent, Generator  $items)
     {
         $this->action = $action;
         $this->page_request = $pageRequestEvent;
@@ -154,36 +154,20 @@ class BulkActions extends Extension
 
             $action = $_POST['bulk_action'];
 
-            $items = [];
+            $items = null;
             if (isset($_POST['bulk_selected_ids']) && $_POST['bulk_selected_ids'] != "") {
                 $data = json_decode($_POST['bulk_selected_ids']);
-                if (is_array($data)) {
+                if (is_array($data)&&!empty($data)) {
-                    foreach ($data as $id) {
+                    $items = $this->yield_items($data);
-                        if (is_numeric($id)) {
-                            array_push($items, int_escape($id));
-                        }
-                    }
                 }
             } elseif (isset($_POST['bulk_query']) && $_POST['bulk_query'] != "") {
                 $query = $_POST['bulk_query'];
                 if ($query != null && $query != "") {
-                    $n = 0;
+                    $items = $this->yield_search_results($query);
-                    $tags = Tag::explode($query);
-                    while (true) {
-                        $results = Image::find_image_ids($n, 100, $tags);
-                        if (count($results) == 0) {
-                            break;
-                        }
-
-                        reset($results); // rewind to first element in array.
-                        $items = array_merge($items, $results);
-                        $n += count($results);
-                    }
                 }
             }
 
-            if (sizeof($items) > 0) {
+            if (is_iterable($items)) {
-                reset($items); // rewind to first element in array.
                 $newEvent = new BulkActionEvent($action, $event, $items);
                 send_event($newEvent);
             }
@@ -197,21 +181,34 @@ class BulkActions extends Extension
         }
     }
 
+    private function yield_items(array $data): Generator
+    {
+        foreach ($data as $id) {
+            if (is_numeric($id)) {
+                $image = Image::by_id($id);
+                if($image!=null) {
+                    yield $image;
+                }
+            }
+        }
+    }
+
+    private function yield_search_results(string $query): Generator
+    {
+        $tags = Tag::explode($query);
+        return Image::find_images_iterable(0, null, $tags);
+    }
+
     private function sort_blocks($a, $b)
     {
         return $a["position"] - $b["position"];
     }
     
-    private function delete_items(array $items): int
+    private function delete_items(iterable $items): int
     {
         $total = 0;
-        foreach ($items as $id) {
+        foreach ($items as $image) {
             try {
-                $image = Image::by_id($id);
-                if ($image==null) {
-                    continue;
-                }
-
                 if (class_exists("ImageBan") && isset($_POST['bulk_ban_reason'])) {
                     $reason = $_POST['bulk_ban_reason'];
                     if ($reason) {
@@ -221,13 +218,13 @@ class BulkActions extends Extension
                 send_event(new ImageDeletionEvent($image));
                 $total++;
             } catch (Exception $e) {
-                flash_message("Error while removing $id: " . $e->getMessage(), "error");
+                flash_message("Error while removing {$image->id}: " . $e->getMessage(), "error");
             }
         }
         return $total;
     }
 
-    private function tag_items(array $items, string $tags, bool $replace): int
+    private function tag_items(iterable $items, string $tags, bool $replace): int
     {
         $tags = Tag::explode($tags);
 
@@ -243,28 +240,21 @@ class BulkActions extends Extension
 
         $total = 0;
         if ($replace) {
-            foreach ($items as $id) {
+            foreach ($items as $image) {
-                $image = Image::by_id($id);
-                if ($image==null) {
-                    continue;
-                }
-
                 send_event(new TagSetEvent($image, $tags));
                 $total++;
             }
         } else {
-            foreach ($items as $id) {
+            foreach ($items as $image) {
-                $image = Image::by_id($id);
+                $img_tags = array_map("strtolower",$image->get_tag_array());
-                if ($image==null) {
-                    continue;
-                }
 
-                $img_tags = [];
                 if (!empty($neg_tag_array)) {
-                    $img_tags = array_merge($pos_tag_array, $image->get_tag_array());
+                    $neg_tag_array = array_map("strtolower",$neg_tag_array);
+
+                    $img_tags = array_merge($pos_tag_array, $img_tags);
                     $img_tags = array_diff($img_tags, $neg_tag_array);
                 } else {
-                    $img_tags = array_merge($tags, $image->get_tag_array());
+                    $img_tags = array_merge($tags, $img_tags);
                 }
                 send_event(new TagSetEvent($image, $img_tags));
                 $total++;
@@ -274,23 +264,17 @@ class BulkActions extends Extension
         return $total;
     }
 
-    private function set_source(array $items, String $source): int
+    private function set_source(iterable $items, String $source): int
     {
         $total = 0;
-        foreach ($items as $id) {
+        foreach ($items as $image) {
             try {
-                $image = Image::by_id($id);
-                if ($image==null) {
-                    continue;
-                }
-
                 send_event(new SourceSetEvent($image, $source));
                 $total++;
             } catch (Exception $e) {
-                flash_message("Error while setting source for $id: " . $e->getMessage(), "error");
+                flash_message("Error while setting source for {$image->id}: " . $e->getMessage(), "error");
             }
         }
-
         return $total;
     }
 }

ext/rating/main.php

@@ -186,12 +186,7 @@ class Ratings extends Extension
                 if ($user->can("bulk_edit_image_rating")) {
                     $rating = $_POST['bulk_rating'];
                     $total = 0;
-                    foreach ($event->items as $id) {
+                    foreach ($event->items as $image) {
-                        $image = Image::by_id($id);
-                        if ($image==null) {
-                            continue;
-                        }
-
                         send_event(new RatingSetEvent($image, $rating));
                         $total++;
                     }

ext/regen_thumb/main.php

@@ -87,12 +87,7 @@ class RegenThumb extends Extension
                     }
     
                     $total = 0;
-                    foreach ($event->items as $id) {
+                    foreach ($event->items as $image) {
-                        $image = Image::by_id($id);
-                        if ($image==null) {
-                            continue;
-                        }
-
                         if ($this->regenerate_thumbnail($image, $force)) {
                             $total++;
                         }

ext/transcode/main.php

@@ -246,14 +246,10 @@ class TranscodeImage extends Extension
                 if ($user->is_admin()) {
                     $format = $_POST['transcode_format'];
                     $total = 0;
-                    foreach ($event->items as $id) {
+                    foreach ($event->items as $image) {
                         try {
                             $database->beginTransaction();
-                            $image = Image::by_id($id);
+
-                            if ($image==null) {
-                                continue;
-                            }
-                            
                             $this->transcode_and_replace_image($image, $format);
                             // If a subsequent transcode fails, the database need to have everything about the previous transcodes recorded already,
                             // otherwise the image entries will be stuck pointing to missing image files

ext/trash/main.php

@@ -136,8 +136,8 @@ class Trash extends Extension
             case "bulk_trash_restore":
                 if ($user->can("view_trash")) {
                     $total = 0;
-                    foreach ($event->items as $id) {
+                    foreach ($event->items as $image) {
-                        self::set_trash($id, false);
+                        self::set_trash($image->id, false);
                         $total++;
                     }
                     flash_message("Restored $total items from trash");