Sending queue refactoring WIP [MAILPOET-903]

lib/API/JSON/v1/Newsletters.php

@@ -105,8 +105,8 @@ class Newsletters extends APIEndpoint {
         $newsletter->schedule = Scheduler::processPostNotificationSchedule($newsletter);
         $next_run_date = Scheduler::getNextRunDate($newsletter->schedule);
         // find previously scheduled jobs and reschedule them using the new "next run" date
-        SendingQueue::where('newsletter_id', $newsletter->id)
+        SendingQueue::findTaskByNewsletterId($newsletter->id)
-          ->where('status', SendingQueue::STATUS_SCHEDULED)
+          ->where('tasks.status', SendingQueue::STATUS_SCHEDULED)
           ->findResultSet()
           ->set('scheduled_at', $next_run_date)
           ->save();
@@ -166,7 +166,7 @@ class Newsletters extends APIEndpoint {
     // if there are past due notifications, reschedule them for the next send date
     if($newsletter->type === Newsletter::TYPE_NOTIFICATION && $status === Newsletter::STATUS_ACTIVE) {
       $next_run_date = Scheduler::getNextRunDate($newsletter->schedule);
-      $newsletter->queue()
+      $newsletter->queue()->task()
         ->whereLte('scheduled_at', Carbon::createFromTimestamp(current_time('timestamp')))
         ->where('status', SendingQueue::STATUS_SCHEDULED)
         ->findResultSet()

lib/API/JSON/v1/SendingQueue.php

@@ -8,9 +8,9 @@ use MailPoet\Config\AccessControl;
 use MailPoet\Mailer\Mailer;
 use MailPoet\Models\Newsletter;
 use MailPoet\Models\SendingQueue as SendingQueueModel;
-use MailPoet\Models\Subscriber;
 use MailPoet\Newsletter\Scheduler\Scheduler;
 use MailPoet\Segments\SubscribersFinder;
+use MailPoet\Tasks\Sending as SendingTask;
 use MailPoet\Util\Helpers;
 
 if(!defined('ABSPATH')) exit;
@@ -46,8 +46,8 @@ class SendingQueue extends APIEndpoint {
     }
 
     // add newsletter to the sending queue
-    $queue = SendingQueueModel::whereNull('status')
+    $queue = SendingQueueModel::findTaskByNewsletterId($newsletter->id)
-    ->where('newsletter_id', $newsletter->id)
+      ->whereNull('tasks.status')
       ->findOne();
 
     if(!empty($queue)) {
@@ -55,11 +55,13 @@ class SendingQueue extends APIEndpoint {
         APIError::NOT_FOUND => __('This newsletter is already being sent.', 'mailpoet')
       ));
     }
-    $queue = SendingQueueModel::where('newsletter_id', $newsletter->id)
+    $task = SendingQueueModel::findTaskByNewsletterId($newsletter->id)
-      ->where('status', SendingQueueModel::STATUS_SCHEDULED)
+      ->where('tasks.status', SendingQueueModel::STATUS_SCHEDULED)
       ->findOne();
-    if(!$queue) {
+    if($task) {
-      $queue = SendingQueueModel::create();
+      $queue = SendingTask::createFromTask($task);
+    } else {
+      $queue = SendingTask::create();
       $queue->newsletter_id = $newsletter->id;
     }
 
@@ -72,8 +74,7 @@ class SendingQueue extends APIEndpoint {
       $queue->scheduled_at = Scheduler::formatDatetimeString(
         $newsletter->scheduledAt
       );
-      $queue->subscribers = null;
+      $queue->removeAllSubscribers();
-      $queue->count_total = $queue->count_to_process = 0;
     } else {
       $segments = $newsletter->segments()->findArray();
       $finder = new SubscribersFinder();
@@ -86,12 +87,7 @@ class SendingQueue extends APIEndpoint {
       }
       $queue->status = null;
       $queue->scheduled_at = null;
-      $queue->subscribers = serialize(
+      $queue->setSubscribers($subscribers);
-        array(
-          'to_process' => $subscribers
-        )
-      );
-      $queue->count_total = $queue->count_to_process = count($subscribers);
 
       // set newsletter status
       $newsletter->setStatus(Newsletter::STATUS_SENDING);

lib/Config/Migrator.php

@@ -137,18 +137,14 @@ class Migrator {
   function sendingQueues() {
     $attributes = array(
       'id int(11) unsigned NOT NULL AUTO_INCREMENT,',
-      'type varchar(90) NULL DEFAULT NULL,',
+      'task_id int(11) unsigned NOT NULL,',
       'newsletter_id int(11) unsigned NOT NULL,',
       'newsletter_rendered_body longtext,',
       'newsletter_rendered_subject varchar(250) NULL DEFAULT NULL,',
       'subscribers longtext,',
-      'status varchar(12) NULL DEFAULT NULL,',
-      'priority mediumint(9) NOT NULL DEFAULT 0,',
       'count_total int(11) unsigned NOT NULL DEFAULT 0,',
       'count_processed int(11) unsigned NOT NULL DEFAULT 0,',
       'count_to_process int(11) unsigned NOT NULL DEFAULT 0,',
-      'scheduled_at TIMESTAMP NULL,',
-      'processed_at TIMESTAMP NULL,',
       'created_at TIMESTAMP NULL,',
       'updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,',
       'deleted_at TIMESTAMP NULL,',

lib/Cron/Daemon.php

@@ -1,6 +1,7 @@
 <?php
 namespace MailPoet\Cron;
 use MailPoet\Cron\Workers\Scheduler as SchedulerWorker;
+use MailPoet\Cron\Workers\SendingQueue\Migration as MigrationWorker;
 use MailPoet\Cron\Workers\SendingQueue\SendingQueue as SendingQueueWorker;
 use MailPoet\Cron\Workers\Bounce as BounceWorker;
 use MailPoet\Cron\Workers\KeyCheck\PremiumKeyCheck as PremiumKeyCheckWorker;
@@ -49,6 +50,7 @@ class Daemon {
     $daemon['token'] = $this->token;
     CronHelper::saveDaemon($daemon);
     try {
+      $this->executeMigrationWorker();
       $this->executeScheduleWorker();
       $this->executeQueueWorker();
       $this->executeSendingServiceKeyCheckWorker();
@@ -100,6 +102,11 @@ class Daemon {
     return $bounce->process();
   }
 
+  function executeMigrationWorker() {
+    $migration = new MigrationWorker($this->timer);
+    return $migration->process();
+  }
+
   function callSelf() {
     CronHelper::accessDaemon($this->token);
     return $this->terminateRequest();

lib/Cron/Triggers/WordPress.php

@@ -3,6 +3,7 @@ namespace MailPoet\Cron\Triggers;
 
 use MailPoet\Cron\CronHelper;
 use MailPoet\Cron\Workers\Scheduler as SchedulerWorker;
+use MailPoet\Cron\Workers\SendingQueue\Migration as MigrationWorker;
 use MailPoet\Cron\Workers\SendingQueue\SendingQueue as SendingQueueWorker;
 use MailPoet\Cron\Workers\Bounce as BounceWorker;
 use MailPoet\Cron\Workers\KeyCheck\PremiumKeyCheck as PremiumKeyCheckWorker;
@@ -20,6 +21,9 @@ class WordPress {
   }
 
   static function checkExecutionRequirements() {
+    // migration
+    $migration_due_tasks = MigrationWorker::getAllDueTasks();
+    $migration_future_tasks = MigrationWorker::getFutureTasks();
     // sending queue
     $scheduled_queues = SchedulerWorker::getScheduledQueues();
     $running_queues = SendingQueueWorker::getRunningQueues();
@@ -42,8 +46,15 @@ class WordPress {
     $bounce_sync_active = ($mp_sending_enabled && ($bounce_due_tasks || !$bounce_future_tasks));
     $sending_service_key_check_active = ($mp_sending_enabled && ($msskeycheck_due_tasks || !$msskeycheck_future_tasks));
     $premium_key_check_active = ($premium_key_specified && ($premium_keycheck_due_tasks || !$premium_keycheck_future_tasks));
+    $migration_active = $migration_due_tasks || !$migration_future_tasks;
 
-    return ($sending_queue_active || $bounce_sync_active || $sending_service_key_check_active || $premium_key_check_active);
+    return (
+      $migration_active
+      || $sending_queue_active
+      || $bounce_sync_active
+      || $sending_service_key_check_active
+      || $premium_key_check_active
+    );
   }
 
   static function cleanup() {

lib/Cron/Workers/Scheduler.php

@@ -4,10 +4,10 @@ namespace MailPoet\Cron\Workers;
 use Carbon\Carbon;
 use MailPoet\Cron\CronHelper;
 use MailPoet\Models\Newsletter;
-use MailPoet\Models\SendingQueue;
 use MailPoet\Models\Subscriber;
 use MailPoet\Models\SubscriberSegment;
 use MailPoet\Segments\SubscribersFinder;
+use MailPoet\Tasks\Sending as SendingTask;
 use MailPoet\Util\Helpers;
 use MailPoet\Newsletter\Scheduler\Scheduler as NewsletterScheduler;
 
@@ -46,12 +46,12 @@ class Scheduler {
   }
 
   function processWelcomeNewsletter($newsletter, $queue) {
-    $subscriber = unserialize($queue->subscribers);
+    $subscribers = $queue->getSubscribers();
-    if(empty($subscriber['to_process'][0])) {
+    if(empty($subscribers[0])) {
       $queue->delete();
       return false;
     }
-    $subscriber_id = (int)$subscriber['to_process'][0];
+    $subscriber_id = (int)$subscribers[0];
     if($newsletter->event === 'segment') {
       if($this->verifyMailpoetSubscriber($subscriber_id, $newsletter, $queue) === false) {
         return false;
@@ -91,12 +91,7 @@ class Scheduler {
 
     // queue newsletter for delivery
     $queue->newsletter_id = $notification_history->id;
-    $queue->subscribers = serialize(
+    $queue->setSubscribers($subscribers);
-      array(
-        'to_process' => $subscribers
-      )
-    );
-    $queue->count_total = $queue->count_to_process = count($subscribers);
     $queue->status = null;
     $queue->save();
     // update notification status
@@ -110,12 +105,7 @@ class Scheduler {
     $subscribers = $finder->getSubscribersByList($segments);
     $subscribers = Helpers::flattenArray($subscribers);
     // update current queue
-    $queue->subscribers = serialize(
+    $queue->setSubscribers($subscribers);
-      array(
-        'to_process' => $subscribers
-      )
-    );
-    $queue->count_total = $queue->count_to_process = count($subscribers);
     $queue->status = null;
     $queue->save();
     // update newsletter status
@@ -189,9 +179,6 @@ class Scheduler {
   }
 
   static function getScheduledQueues() {
-    return SendingQueue::where('status', SendingQueue::STATUS_SCHEDULED)
+    return SendingTask::getScheduledQueues();
-      ->whereLte('scheduled_at', Carbon::createFromTimestamp(current_time('timestamp')))
-      ->whereNull('type')
-      ->findMany();
   }
 }

lib/Cron/Workers/SendingQueue/Migration.php

@@ -0,0 +1,237 @@
+<?php
+namespace MailPoet\Cron\Workers\SendingQueue;
+
+use Carbon\Carbon;
+use MailPoet\Cron\CronHelper;
+use MailPoet\Cron\Workers\SimpleWorker;
+use MailPoet\Mailer\MailerLog;
+use MailPoet\Models\ScheduledTask;
+use MailPoet\Models\ScheduledTaskSubscriber;
+use MailPoet\Models\SendingQueue;
+
+if(!defined('ABSPATH')) exit;
+
+class Migration extends SimpleWorker {
+  const TASK_TYPE = 'migration';
+  const BATCH_SIZE = 20;
+
+  function checkProcessingRequirements() {
+    // if migration was completed, don't run it again
+    $completed_tasks = self::getCompletedTasks();
+    return empty($completed_tasks);
+  }
+
+  function prepareTask(ScheduledTask $task) {
+    $unmigrated_queues_count = $this->getUnmigratedQueues()->count();
+    $unmigrated_queue_subscribers = $this->getTaskIdsForUnmigratedSubscribers();
+
+    if($unmigrated_queues_count == 0
+      && count($unmigrated_queue_subscribers) == 0
+    ) {
+      // nothing to migrate
+      $this->complete($task);
+      $this->resumeSending();
+      return false;
+    }
+
+    // pause sending while the migration is in process
+    $this->pauseSending();
+
+    return parent::prepareTask($task);
+  }
+
+  private function pauseSending() {
+    $mailer_log = MailerLog::getMailerLog();
+    if(MailerLog::isSendingPaused($mailer_log)) {
+      // sending is already paused
+      return false;
+    }
+    $mailer_log = MailerLog::setError(
+      $mailer_log,
+      'migration',
+      __('Your sending queue data is being migrated to allow better performance, sending is paused while the migration is in progress and will resume automatically upon completion. This may take a few minutes.')
+    );
+    return MailerLog::pauseSending($mailer_log);
+  }
+
+  private function resumeSending() {
+    $mailer_log = MailerLog::getMailerLog();
+    if(!MailerLog::isSendingPaused($mailer_log)) {
+      // sending is not paused
+      return false;
+    }
+    $error = MailerLog::getError($mailer_log);
+    // only resume sending if it was paused by migration
+    if(isset($error['operation']) && $error['operation'] === 'migration') {
+      return MailerLog::resumeSending();
+    }
+  }
+
+  function processTask(ScheduledTask $task) {
+    $this->migrateSendingQueues();
+    $this->migrateSubscribers();
+
+    $this->complete($task);
+    $this->resumeSending();
+
+    return true;
+  }
+
+  function getUnmigratedQueues() {
+    return SendingQueue::where('task_id', 0)
+      ->whereNull('type');
+  }
+
+  function getTaskIdsForUnmigratedSubscribers() {
+    global $wpdb;
+    $query = sprintf(
+      'SELECT queues.`task_id` FROM %1$s queues INNER JOIN %2$s tasks ON queues.`task_id` = tasks.`id` ' .
+      'WHERE tasks.`type` = "sending" AND (tasks.`status` IS NULL OR tasks.`status` = "paused") ' .
+      'AND queues.`subscribers` != "" AND queues.`subscribers` != "N;"' .
+      'AND queues.`count_total` > (SELECT COUNT(*) FROM %3$s subs WHERE subs.`task_id` = queues.`task_id`)',
+      MP_SENDING_QUEUES_TABLE,
+      MP_SCHEDULED_TASKS_TABLE,
+      MP_SCHEDULED_TASK_SUBSCRIBERS_TABLE
+    );
+    return $wpdb->get_col($query);
+  }
+
+  /*
+   * Migrate all sending queues without converting subscriber data
+   */
+  function migrateSendingQueues() {
+    global $wpdb;
+
+    $queues = $this->getUnmigratedQueues()
+      ->select('id')
+      ->findArray();
+
+    $column_list = array(
+      'status',
+      'priority',
+      'scheduled_at',
+      'processed_at',
+      'created_at',
+      'updated_at',
+      'deleted_at'
+    );
+
+    if(!empty($queues)) {
+      foreach(array_chunk($queues, self::BATCH_SIZE) as $queue_batch) {
+        // abort if execution limit is reached
+        CronHelper::enforceExecutionLimit($this->timer);
+
+        foreach($queue_batch as $queue) {
+          // create a new scheduled task of type "sending"
+          $wpdb->query(sprintf(
+            'INSERT IGNORE INTO %1$s (`type`, %2$s) ' .
+            'SELECT "sending", %2$s FROM %3$s WHERE `id` = %4$s',
+            MP_SCHEDULED_TASKS_TABLE,
+            '`' . join('`, `', $column_list) . '`',
+            MP_SENDING_QUEUES_TABLE,
+            $queue['id']
+          ));
+          // link the queue with the task via task_id
+          $new_task_id = $wpdb->insert_id;
+          $wpdb->query(sprintf(
+            'UPDATE %1$s SET `task_id` = %2$s WHERE `id` = %3$s',
+            MP_SENDING_QUEUES_TABLE,
+            $new_task_id,
+            $queue['id']
+          ));
+        }
+      }
+    }
+
+    return true;
+  }
+
+  /*
+   * Migrate subscribers for in-progress sending tasks from the `subscribers` field to a separate table
+   */
+  function migrateSubscribers() {
+    global $wpdb;
+
+    // find in-progress queues that have serialized subscribers
+    $task_ids = $this->getTaskIdsForUnmigratedSubscribers();
+
+    // check if subscribers for each one were already migrated
+    if(!empty($task_ids)) {
+      $task_ids = $wpdb->get_col(sprintf(
+        'SELECT queues.`task_id` FROM %1$s queues WHERE queues.`task_id` IN(' . join(',', array_map('intval', $task_ids)) . ') ' .
+        'AND queues.`count_total` > (SELECT COUNT(*) FROM %2$s subs WHERE subs.`task_id` = queues.`task_id`)',
+        MP_SENDING_QUEUES_TABLE,
+        MP_SCHEDULED_TASK_SUBSCRIBERS_TABLE
+      ));
+    }
+
+    if(!empty($task_ids)) {
+      foreach($task_ids as $task_id) {
+        // abort if execution limit is reached
+        CronHelper::enforceExecutionLimit($this->timer);
+
+        $this->migrateTaskSubscribers($task_id);
+      }
+    }
+
+    return true;
+  }
+
+  function migrateTaskSubscribers($task_id) {
+    global $wpdb;
+
+    $migrated_unprocessed_count = ScheduledTaskSubscriber::getUnprocessedCount($task_id);
+    $migrated_processed_count = ScheduledTaskSubscriber::getProcessedCount($task_id);
+
+    $subscribers = $wpdb->get_var(sprintf(
+      'SELECT `subscribers` FROM %1$s WHERE `task_id` = %2$d ' .
+      'AND (`count_processed` > %3$d OR `count_to_process` > %4$d)',
+      MP_SENDING_QUEUES_TABLE,
+      $task_id,
+      $migrated_unprocessed_count,
+      $migrated_processed_count
+    ));
+
+    // sanity check
+    if(empty($subscribers)) {
+      return false;
+    }
+
+    $subscribers = unserialize($subscribers);
+
+    if(!empty($subscribers['to_process'])) {
+      $subscribers_to_migrate = array_slice($subscribers['to_process'], $migrated_unprocessed_count);
+      foreach($subscribers_to_migrate as $sub_id) {
+        // abort if execution limit is reached
+        CronHelper::enforceExecutionLimit($this->timer);
+
+        ScheduledTaskSubscriber::createOrUpdate(array(
+          'task_id' => $task_id,
+          'subscriber_id' => $sub_id,
+          'processed' => ScheduledTaskSubscriber::STATUS_UNPROCESSED
+        ));
+      }
+    }
+
+    if(!empty($subscribers['processed'])) {
+      $subscribers_to_migrate = array_slice($subscribers['processed'], $migrated_processed_count);
+      foreach($subscribers_to_migrate as $sub_id) {
+        // abort if execution limit is reached
+        CronHelper::enforceExecutionLimit($this->timer);
+
+        ScheduledTaskSubscriber::createOrUpdate(array(
+          'task_id' => $task_id,
+          'subscriber_id' => $sub_id,
+          'processed' => ScheduledTaskSubscriber::STATUS_PROCESSED
+        ));
+      }
+    }
+
+    return true;
+  }
+
+  static function getNextRunDate() {
+    // run migration immediately
+    return Carbon::now();
+  }
+}

lib/Cron/Workers/SendingQueue/SendingQueue.php

@@ -6,10 +6,12 @@ use MailPoet\Cron\Workers\SendingQueue\Tasks\Links;
 use MailPoet\Cron\Workers\SendingQueue\Tasks\Mailer as MailerTask;
 use MailPoet\Cron\Workers\SendingQueue\Tasks\Newsletter as NewsletterTask;
 use MailPoet\Mailer\MailerLog;
-use MailPoet\Models\SendingQueue as SendingQueueModel;
+use MailPoet\Models\ScheduledTask as ScheduledTaskModel;
 use MailPoet\Models\StatisticsNewsletters as StatisticsNewslettersModel;
 use MailPoet\Models\Subscriber as SubscriberModel;
 use MailPoet\Segments\SubscribersFinder;
+use MailPoet\Tasks\Sending as SendingTask;
+use MailPoet\Tasks\Subscribers\BatchIterator;
 use MailPoet\WP\Hooks as WPHooks;
 
 if(!defined('ABSPATH')) exit;
@@ -45,11 +47,7 @@ class SendingQueue {
       // get newsletter segments
       $newsletter_segments_ids = $this->newsletter_task->getNewsletterSegments($newsletter);
       // get subscribers
-      $queue->subscribers = $queue->getSubscribers();
+      $subscriber_batches = new BatchIterator($queue->task_id, $this->batch_size);
-      $subscriber_batches = array_chunk(
-        $queue->subscribers['to_process'],
-        $this->batch_size
-      );
       foreach($subscriber_batches as $subscribers_to_process_ids) {
         if(!empty($newsletter_segments_ids[0])) {
           // Check that subscribers are in segments
@@ -72,7 +70,7 @@ class SendingQueue {
             $found_subscribers_ids
           );
           $queue->removeSubscribers($subscribers_to_remove);
-          if(!count($queue->subscribers['to_process'])) {
+          if(!$queue->count_to_process) {
             $this->newsletter_task->markNewsletterAsSent($newsletter, $queue);
             continue;
           }
@@ -82,7 +80,7 @@ class SendingQueue {
           $newsletter,
           $found_subscribers
         );
-        if($queue->status === SendingQueueModel::STATUS_COMPLETED) {
+        if($queue->status === ScheduledTaskModel::STATUS_COMPLETED) {
           $this->newsletter_task->markNewsletterAsSent($newsletter, $queue);
         }
         $this->enforceSendingAndExecutionLimits();
@@ -179,7 +177,7 @@ class SendingQueue {
     // update the sent count
     $this->mailer_task->updateSentCount();
     // enforce execution limits if queue is still being processed
-    if($queue->status !== SendingQueueModel::STATUS_COMPLETED) {
+    if($queue->status !== ScheduledTaskModel::STATUS_COMPLETED) {
       $this->enforceSendingAndExecutionLimits();
     }
     return $queue;
@@ -193,11 +191,6 @@ class SendingQueue {
   }
 
   static function getRunningQueues() {
-    return SendingQueueModel::orderByAsc('priority')
+    return SendingTask::getRunningQueues();
-      ->orderByAsc('created_at')
-      ->whereNull('deleted_at')
-      ->whereNull('status')
-      ->whereNull('type')
-      ->findMany();
   }
 }

lib/Cron/Workers/SimpleWorker.php

@@ -62,7 +62,7 @@ abstract class SimpleWorker {
     $task->type = static::TASK_TYPE;
     $task->status = ScheduledTask::STATUS_SCHEDULED;
     $task->priority = ScheduledTask::PRIORITY_LOW;
-    $task->scheduled_at = self::getNextRunDate();
+    $task->scheduled_at = static::getNextRunDate();
     $task->save();
     return $task;
   }
@@ -139,4 +139,11 @@ abstract class SimpleWorker {
   static function getFutureTasks() {
     return self::getScheduledTasks(true);
   }
+
+  static function getCompletedTasks() {
+    return ScheduledTask::where('type', static::TASK_TYPE)
+      ->whereNull('deleted_at')
+      ->where('status', ScheduledTask::STATUS_COMPLETED)
+      ->findMany();
+  }
 }

lib/Mailer/MailerLog.php

@@ -79,6 +79,17 @@ class MailerLog {
     $mailer_log = self::getMailerLog();
     (int)$mailer_log['retry_attempt']++;
     $mailer_log['retry_at'] = time() + self::RETRY_INTERVAL;
+    $mailer_log = self::setError($mailer_log, $operation, $error_message, $error_code);
+    self::updateMailerLog($mailer_log);
+    if($pause_sending) {
+      self::pauseSending($mailer_log);
+    } else {
+      self::updateMailerLog($mailer_log);
+    }
+    return self::enforceExecutionRequirements();
+  }
+
+  static function setError($mailer_log, $operation, $error_message, $error_code = null) {
     $mailer_log['error'] = array(
       'operation' => $operation,
       'error_message' => $error_message
@@ -86,12 +97,12 @@ class MailerLog {
     if($error_code) {
       $mailer_log['error']['error_code'] = $error_code;
     }
-    if($pause_sending) {
+    return $mailer_log;
-      self::pauseSending($mailer_log);
-    } else {
-      self::updateMailerLog($mailer_log);
   }
-    return self::enforceExecutionRequirements();
+
+  static function getError($mailer_log = false) {
+    $mailer_log = self::getMailerLog($mailer_log);
+    return isset($mailer_log['error']) ? $mailer_log['error'] : null;
   }
 
   static function incrementSentCount() {

lib/Models/Newsletter.php

@@ -2,6 +2,7 @@
 namespace MailPoet\Models;
 use Carbon\Carbon;
 use MailPoet\Newsletter\Renderer\Renderer;
+use MailPoet\Tasks\Sending as SendingTask;
 use MailPoet\Util\Helpers;
 use MailPoet\Util\Security;
 use MailPoet\WP\Emoji;
@@ -107,12 +108,24 @@ class Newsletter extends Model {
         'SET `deleted_at` = NOW() ' .
         'WHERE `parent_id` = ' . $this->id
       );
+      ScheduledTask::rawExecute(
+        'UPDATE `' . ScheduledTask::$_table . '` t ' .
+        'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
+        'SET t.`deleted_at` = NOW() ' .
+        'WHERE q.`newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), array($this->id))) . ')'
+      );
       SendingQueue::rawExecute(
         'UPDATE `' . SendingQueue::$_table . '` ' .
         'SET `deleted_at` = NOW() ' .
         'WHERE `newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), array($this->id))) . ')'
       );
     } else {
+      ScheduledTask::rawExecute(
+        'UPDATE `' . ScheduledTask::$_table . '` t ' .
+        'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
+        'SET t.`deleted_at` = NOW() ' .
+        'WHERE q.`newsletter_id` = ' . $this->id
+      );
       SendingQueue::rawExecute(
         'UPDATE `' . SendingQueue::$_table . '` ' .
         'SET `deleted_at` = NOW() ' .
@@ -133,12 +146,24 @@ class Newsletter extends Model {
           'SET `deleted_at` = NOW() ' .
           'WHERE `parent_id` IN (' . join(',', Helpers::flattenArray($ids)) . ')'
         );
+        ScheduledTask::rawExecute(
+          'UPDATE `' . ScheduledTask::$_table . '` t ' .
+          'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
+          'SET t.`deleted_at` = NOW() ' .
+          'WHERE q.`newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), $ids)) . ')'
+        );
         SendingQueue::rawExecute(
           'UPDATE `' . SendingQueue::$_table . '` ' .
           'SET `deleted_at` = NOW() ' .
           'WHERE `newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), $ids)) . ')'
         );
       } else {
+        ScheduledTask::rawExecute(
+          'UPDATE `' . ScheduledTask::$_table . '` t ' .
+          'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
+          'SET t.`deleted_at` = NOW() ' .
+          'WHERE q.`newsletter_id` IN (' . join(',', Helpers::flattenArray($ids)) . ')'
+        );
         SendingQueue::rawExecute(
           'UPDATE `' . SendingQueue::$_table . '` ' .
           'SET `deleted_at` = NOW() ' .
@@ -156,9 +181,11 @@ class Newsletter extends Model {
     if($children) {
       $children = Helpers::flattenArray($children);
       $this->children()->deleteMany();
+      SendingQueue::getTasks()->whereIn('queues.newsletter_id', array_merge($children, array($this->id)))->deleteMany();
       SendingQueue::whereIn('newsletter_id', array_merge($children, array($this->id)))->deleteMany();
       NewsletterSegment::whereIn('newsletter_id', array_merge($children, array($this->id)))->deleteMany();
     } else {
+      SendingQueue::getTasks()->where('queues.newsletter_id', $this->id)->deleteMany();
       $this->queue()->deleteMany();
       $this->segmentRelations()->deleteMany();
     }
@@ -173,9 +200,11 @@ class Newsletter extends Model {
       if($children) {
         $children = Helpers::flattenArray($children);
         Newsletter::whereIn('parent_id', $ids)->deleteMany();
+        SendingQueue::getTasks()->whereIn('queues.newsletter_id', array_merge($children, $ids))->deleteMany();
         SendingQueue::whereIn('newsletter_id', array_merge($children, $ids))->deleteMany();
         NewsletterSegment::whereIn('newsletter_id', array_merge($children, $ids))->deleteMany();
       } else {
+        SendingQueue::getTasks()->whereIn('queues.newsletter_id', $ids)->deleteMany();
         SendingQueue::whereIn('newsletter_id', $ids)->deleteMany();
         NewsletterSegment::whereIn('newsletter_id', $ids)->deleteMany();
       }
@@ -193,12 +222,24 @@ class Newsletter extends Model {
         'SET `deleted_at` = null ' .
         'WHERE `parent_id` = ' . $this->id
       );
+      ScheduledTask::rawExecute(
+        'UPDATE `' . ScheduledTask::$_table . '` t ' .
+        'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
+        'SET t.`deleted_at` = null ' .
+        'WHERE q.`newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), array($this->id))) . ')'
+      );
       SendingQueue::rawExecute(
         'UPDATE `' . SendingQueue::$_table . '` ' .
         'SET `deleted_at` = null ' .
         'WHERE `newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), array($this->id))) . ')'
       );
     } else {
+      ScheduledTask::rawExecute(
+        'UPDATE `' . ScheduledTask::$_table . '` t ' .
+        'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
+        'SET t.`deleted_at` = null ' .
+        'WHERE q.`newsletter_id` = ' . $this->id
+      );
       SendingQueue::rawExecute(
         'UPDATE `' . SendingQueue::$_table . '` ' .
         'SET `deleted_at` = null ' .
@@ -223,12 +264,24 @@ class Newsletter extends Model {
           ->findResultSet()
           ->set('deleted_at', null)
           ->save();
+        SendingQueue::getTasks()
+          ->whereIn('queues.newsletter_id', Helpers::flattenArray($children))
+          ->whereNotNull('deleted_at')
+          ->findResultSet()
+          ->set('deleted_at', null)
+          ->save();
         SendingQueue::whereIn('newsletter_id', Helpers::flattenArray($children))
           ->whereNotNull('deleted_at')
           ->findResultSet()
           ->set('deleted_at', null)
           ->save();
       } else {
+        SendingQueue::getTasks()
+          ->whereIn('queues.newsletter_id', $ids)
+          ->whereNotNull('deleted_at')
+          ->findResultSet()
+          ->set('deleted_at', null)
+          ->save();
         SendingQueue::whereIn('newsletter_id', $ids)
           ->whereNotNull('deleted_at')
           ->findResultSet()
@@ -408,23 +461,11 @@ class Newsletter extends Model {
   }
 
   function getQueue($columns = '*') {
-    return SendingQueue::select($columns)
+    return SendingTask::getByNewsletterId($this->id);
-      ->where('newsletter_id', $this->id)
-      ->orderByDesc('updated_at')
-      ->findOne();
   }
 
   function withSendingQueue() {
-    $queue = $this->getQueue(array(
+    $queue = $this->getQueue();
-      'id',
-      'newsletter_id',
-      'newsletter_rendered_subject',
-      'status',
-      'count_processed',
-      'count_total',
-      'scheduled_at',
-      'created_at'
-    ));
     if($queue === false) {
       $this->queue = false;
     } else {
@@ -445,9 +486,9 @@ class Newsletter extends Model {
 
   function withTotalSent() {
     // total of subscribers who received the email
-    $this->total_sent = (int)SendingQueue::where('newsletter_id', $this->id)
+    $this->total_sent = (int)SendingQueue::findTaskByNewsletterId($this->id)
-      ->where('status', SendingQueue::STATUS_COMPLETED)
+      ->where('tasks.status', SendingQueue::STATUS_COMPLETED)
-      ->sum('count_processed');
+      ->sum('queues.count_processed');
     return $this;
   }
 
@@ -480,8 +521,9 @@ class Newsletter extends Model {
       } else {
         $row = $statisticsExpr
           ->join(MP_SENDING_QUEUES_TABLE, array("queue_id", "=", "qt.id"), "qt")
+          ->join(MP_SCHEDULED_TASKS_TABLE, array("qt.task_Id", "=", "tasks.id"), "tasks")
           ->where(array(
-            "qt.status" => SendingQueue::STATUS_COMPLETED,
+            "tasks.status" => SendingQueue::STATUS_COMPLETED,
             "stat.newsletter_id" => $this->id,
           ))->findOne();
       }
@@ -511,8 +553,13 @@ class Newsletter extends Model {
         'queues.newsletter_id = newsletters.id',
         'queues'
       )
-      ->where('queues.status', SendingQueue::STATUS_COMPLETED)
+      ->join(
-      ->whereGte('queues.processed_at', Carbon::now()->subMonths(3))
+        MP_SCHEDULED_TASKS_TABLE,
+        'queues.task_id = tasks.id',
+        'tasks'
+      )
+      ->where('tasks.status', SendingQueue::STATUS_COMPLETED)
+      ->whereGte('tasks.processed_at', Carbon::now()->subMonths(3))
       ->count();
 
 
@@ -862,10 +909,15 @@ class Newsletter extends Model {
         'queues.newsletter_id = newsletters.id',
         'queues'
       )
-      ->where('queues.status', SendingQueue::STATUS_COMPLETED)
+      ->join(
+        MP_SCHEDULED_TASKS_TABLE,
+        'queues.task_id = tasks.id',
+        'tasks'
+      )
+      ->where('tasks.status', SendingQueue::STATUS_COMPLETED)
       ->whereNull('newsletters.deleted_at')
-      ->select('queues.processed_at')
+      ->select('tasks.processed_at')
-      ->orderByDesc('queues.processed_at');
+      ->orderByDesc('tasks.processed_at');
 
     if(!empty($segment_ids)) {
       $orm->join(

lib/Models/ScheduledTask.php

@@ -7,10 +7,32 @@ class ScheduledTask extends Model {
   public static $_table = MP_SCHEDULED_TASKS_TABLE;
   const STATUS_COMPLETED = 'completed';
   const STATUS_SCHEDULED = 'scheduled';
+  const STATUS_PAUSED = 'paused';
   const PRIORITY_HIGH = 1;
   const PRIORITY_MEDIUM = 5;
   const PRIORITY_LOW = 10;
 
+  function subscribers() {
+    return $this->hasManyThrough(
+      __NAMESPACE__.'\Subscriber',
+      __NAMESPACE__.'\ScheduledTaskSubscriber',
+      'task_id',
+      'subscriber_id'
+    );
+  }
+
+  function pause() {
+    $this->set('status', self::STATUS_PAUSED);
+    $this->save();
+    return ($this->getErrors() === false && $this->id() > 0);
+  }
+
+  function resume() {
+    $this->setExpr('status', 'NULL');
+    $this->save();
+    return ($this->getErrors() === false && $this->id() > 0);
+  }
+
   function complete() {
     $this->processed_at = current_time('mysql');
     $this->set('status', self::STATUS_COMPLETED);

lib/Models/ScheduledTaskSubscriber.php

@@ -10,6 +10,10 @@ class ScheduledTaskSubscriber extends Model {
   public static $_table = MP_SCHEDULED_TASK_SUBSCRIBERS_TABLE;
   public static $_id_column = array('task_id', 'subscriber_id');
 
+  function task() {
+    return $this->hasOne(__NAMESPACE__ . '\ScheduledTask', 'id', 'task_id');
+  }
+
   static function createOrUpdate($data = array()) {
     if(!is_array($data) || empty($data['task_id']) || empty($data['subscriber_id'])) {
       return;
@@ -21,6 +25,11 @@ class ScheduledTaskSubscriber extends Model {
     ));
   }
 
+  static function setSubscribers($task_id, array $subscriber_ids) {
+    static::clearSubscribers($task_id);
+    return static::addSubscribers($task_id, $subscriber_ids);
+  }
+
   static function addSubscribers($task_id, array $subscriber_ids) {
     foreach($subscriber_ids as $subscriber_id) {
       self::createOrUpdate(array(
@@ -30,6 +39,10 @@ class ScheduledTaskSubscriber extends Model {
     }
   }
 
+  static function clearSubscribers($task_id) {
+    return self::where('task_id', $task_id)->deleteMany();
+  }
+
   static function getUnprocessedCount($task_id) {
     return self::getCount($task_id, self::STATUS_UNPROCESSED);
   }

lib/Models/SendingQueue.php

@@ -3,6 +3,7 @@ namespace MailPoet\Models;
 
 use MailPoet\Util\Helpers;
 use MailPoet\WP\Emoji;
+use MailPoet\Tasks\Subscribers as TaskSubscribers;
 
 if(!defined('ABSPATH')) exit;
 
@@ -23,6 +24,10 @@ class SendingQueue extends Model {
     ));
   }
 
+  function task() {
+    return $this->hasOne(__NAMESPACE__ . '\ScheduledTask', 'id', 'task_id');
+  }
+
   function newsletter() {
     return $this->has_one(__NAMESPACE__ . '\Newsletter', 'id', 'newsletter_id');
   }
@@ -31,9 +36,7 @@ class SendingQueue extends Model {
     if($this->count_processed === $this->count_total) {
       return false;
     } else {
-      $this->set('status', self::STATUS_PAUSED);
+      return $this->task()->findOne()->pause();
-      $this->save();
-      return ($this->getErrors() === false && $this->id() > 0);
     }
   }
 
@@ -41,23 +44,16 @@ class SendingQueue extends Model {
     if($this->count_processed === $this->count_total) {
       return $this->complete();
     } else {
-      $this->setExpr('status', 'NULL');
+      return $this->task()->findOne()->resume();
-      $this->save();
-      return ($this->getErrors() === false && $this->id() > 0);
     }
   }
 
   function complete() {
-    $this->set('status', self::STATUS_COMPLETED);
+    return $this->task()->findOne()->complete();
-    $this->save();
-    return ($this->getErrors() === false && $this->id() > 0);
   }
 
   function save() {
     $this->newsletter_rendered_body = $this->getNewsletterRenderedBody();
-    if(!is_serialized($this->subscribers) && !is_null($this->subscribers)) {
-      $this->set('subscribers', serialize($this->subscribers));
-    }
     if(!Helpers::isJson($this->newsletter_rendered_body) && !is_null($this->newsletter_rendered_body)) {
       $this->set(
         'newsletter_rendered_body',
@@ -69,22 +65,10 @@ class SendingQueue extends Model {
       $this->priority = self::PRIORITY_MEDIUM;
     }
     parent::save();
-    $this->subscribers = $this->getSubscribers();
     $this->newsletter_rendered_body = $this->getNewsletterRenderedBody();
     return $this;
   }
 
-  function getSubscribers() {
-    if(!is_serialized($this->subscribers)) {
-      return $this->subscribers;
-    }
-    $subscribers = unserialize($this->subscribers);
-    if(empty($subscribers['processed'])) {
-      $subscribers['processed'] = array();
-    }
-    return $subscribers;
-  }
-
   function getNewsletterRenderedBody($type = false) {
     $rendered_newsletter = $this->decodeRenderedNewsletterBodyObject($this->newsletter_rendered_body);
     return ($type && !empty($rendered_newsletter[$type])) ?
@@ -115,57 +99,16 @@ class SendingQueue extends Model {
   }
 
   function isSubscriberProcessed($subscriber_id) {
-    $subscribers = $this->getSubscribers();
+    return (new TaskSubscribers($this->task()->findOne()))
-    return in_array($subscriber_id, $subscribers['processed']);
+      ->isSubscriberProcessed($subscriber_id);
   }
 
   function asArray() {
     $model = parent::asArray();
-    $model['subscribers'] = $this->getSubscribers();
     $model['newsletter_rendered_body'] = $this->getNewsletterRenderedBody();
     return $model;
   }
 
-  function removeSubscribers($subscribers_to_remove) {
-    $subscribers = $this->getSubscribers();
-    $subscribers['to_process'] = array_values(
-      array_diff(
-        $subscribers['to_process'],
-        $subscribers_to_remove
-      )
-    );
-    $this->subscribers = $subscribers;
-    $this->updateCount();
-  }
-
-  function updateProcessedSubscribers($processed_subscribers) {
-    $subscribers = $this->getSubscribers();
-    $subscribers['processed'] = array_merge(
-      $subscribers['processed'],
-      $processed_subscribers
-    );
-    $subscribers['to_process'] = array_values(
-      array_diff(
-        $subscribers['to_process'],
-        $processed_subscribers
-      )
-    );
-    $this->subscribers = $subscribers;
-    return $this->updateCount()->getErrors() === false;
-  }
-
-  function updateCount() {
-    $this->subscribers = $this->getSubscribers();
-    $this->count_processed = count($this->subscribers['processed']);
-    $this->count_to_process = count($this->subscribers['to_process']);
-    $this->count_total = $this->count_processed + $this->count_to_process;
-    if(!$this->count_to_process) {
-      $this->processed_at = current_time('mysql');
-      $this->status = self::STATUS_COMPLETED;
-    }
-    return $this->save();
-  }
-
   private function decodeRenderedNewsletterBodyObject($rendered_body) {
     if(is_serialized($rendered_body)) {
       return $this->decodeEmojisInBody(unserialize($rendered_body));
@@ -175,4 +118,36 @@ class SendingQueue extends Model {
     }
     return $rendered_body;
   }
+
+  static function getTasks() {
+    return ScheduledTask::table_alias('tasks')
+    ->join(
+      MP_SENDING_QUEUES_TABLE,
+      'tasks.id = queues.task_id',
+      'queues'
+    );
+  }
+
+  static function joinWithTasks() {
+    return static::table_alias('queues')
+    ->join(
+      MP_SCHEDULED_TASKS_TABLE,
+      'tasks.id = queues.task_id',
+      'tasks'
+    );
+  }
+
+  static function joinWithSubscribers() {
+    return static::joinWithTasks()
+    ->join(
+      MP_SCHEDULED_TASK_SUBSCRIBERS_TABLE,
+      'tasks.id = subscribers.task_id',
+      'subscribers'
+    );
+  }
+
+  static function findTaskByNewsletterId($newsletter_id) {
+    return static::joinWithTasks()
+    ->where('queues.newsletter_id', $newsletter_id);
+  }
 }

lib/Newsletter/Scheduler/Scheduler.php

@@ -7,6 +7,7 @@ use MailPoet\Models\NewsletterOption;
 use MailPoet\Models\NewsletterOptionField;
 use MailPoet\Models\NewsletterPost;
 use MailPoet\Models\SendingQueue;
+use MailPoet\Tasks\Sending as SendingTask;
 
 class Scheduler {
   const SECONDS_IN_HOUR = 3600;
@@ -73,18 +74,14 @@ class Scheduler {
   }
 
   static function createWelcomeNotificationQueue($newsletter, $subscriber_id) {
-    $previously_scheduled_notification = SendingQueue::where('newsletter_id', $newsletter->id)
+    $previously_scheduled_notification = SendingQueue::joinWithSubscribers()
-      ->whereLike('subscribers', '%' . serialize(array($subscriber_id)) . '%')
+      ->where('queues.newsletter_id', $newsletter->id)
+      ->where('subscribers.subscriber_id', $subscriber_id)
       ->findOne();
     if(!empty($previously_scheduled_notification)) return;
-    $queue = SendingQueue::create();
+    $queue = SendingTask::create();
     $queue->newsletter_id = $newsletter->id;
-    $queue->subscribers = serialize(
+    $queue->setSubscribers(array($subscriber_id));
-      array(
-        'to_process' => array($subscriber_id)
-      )
-    );
-    $queue->count_total = $queue->count_to_process = 1;
     $after_time_type = $newsletter->afterTimeType;
     $after_time_number = $newsletter->afterTimeNumber;
     $scheduled_at = null;
@@ -116,7 +113,7 @@ class Scheduler {
       ->where('scheduled_at', $next_run_date)
       ->findOne();
     if($existing_queue) return;
-    $queue = SendingQueue::create();
+    $queue = SendingTask::create();
     $queue->newsletter_id = $newsletter->id;
     $queue->status = SendingQueue::STATUS_SCHEDULED;
     $queue->scheduled_at = $next_run_date;

lib/Tasks/Sending.php

@@ -0,0 +1,198 @@
+<?php
+namespace MailPoet\Tasks;
+
+use Carbon\Carbon;
+use MailPoet\Models\ScheduledTask;
+use MailPoet\Models\ScheduledTaskSubscriber;
+use MailPoet\Models\SendingQueue;
+use MailPoet\Util\Helpers;
+
+if(!defined('ABSPATH')) exit;
+
+/**
+ * A facade class containing all necessary models to work with a sending queue
+ */
+class Sending {
+  const TASK_TYPE = 'sending';
+
+  private $task;
+  private $queue;
+  private $task_subscribers;
+
+  private $queue_fields = array(
+    'id',
+    'task_id',
+    'newsletter_id',
+    'newsletter_rendered_subject',
+    'newsletter_rendered_body',
+    'count_total',
+    'count_processed',
+    'count_to_process'
+  );
+
+  public function __construct(ScheduledTask $task, SendingQueue $queue) {
+    $this->task = $task;
+    $this->queue = $queue;
+    $this->task_subscribers = new Subscribers($task);
+  }
+
+  static function create(ScheduledTask $task = null, SendingQueue $queue = null) {
+    if(is_null($task) && is_null($queue)) {
+      $task = ScheduledTask::create();
+      $task->type = self::TASK_TYPE;
+      $task->save();
+
+      $queue = SendingQueue::create();
+      $queue->newsletter_id = 0;
+      $queue->task_id = $task->id;
+      $queue->save();
+    }
+
+    return new self($task, $queue);
+  }
+
+  static function createFromTask(ScheduledTask $task) {
+    $queue = SendingQueue::where('task_id', $task->id)->findOne();
+    if(!$queue) {
+      return false;
+    }
+
+    return self::create($task, $queue);
+  }
+
+  static function createFromQueue(SendingQueue $queue) {
+    $task = $queue->task()->findOne();
+    if(!$task) {
+      return false;
+    }
+
+    return self::create($task, $queue);
+  }
+
+  static function getByNewsletterId($newsletter_id) {
+    $queue = SendingQueue::where('newsletter_id', $newsletter_id)
+      ->orderByDesc('updated_at')
+      ->findOne();
+    if(!$queue) {
+      return false;
+    }
+
+    return self::createFromQueue($queue);
+  }
+
+  public function asArray() {
+    $queue = array_intersect_key(
+      $this->queue->asArray(),
+      array_flip($this->queue_fields)
+    );
+    $task = $this->task->asArray();
+    return array_merge($task, $queue);
+  }
+
+  public function save() {
+    $this->task->save();
+    $this->queue->save();
+    return $this;
+  }
+
+  public function delete() {
+    $this->task_subscribers->removeAllSubscribers();
+    $this->task->delete();
+    $this->queue->delete();
+  }
+
+  public function queue() {
+    return $this->queue;
+  }
+
+  public function task() {
+    return $this->task;
+  }
+
+  public function taskSubscribers() {
+    return $this->task_subscribers;
+  }
+
+  public function getSubscribers() {
+    $subscribers = $this->task_subscribers->getSubscribers()->findArray();
+    return Helpers::arrayColumn($subscribers, 'subscriber_id');
+  }
+
+  public function setSubscribers(array $subscriber_ids) {
+    $this->task_subscribers->setSubscribers($subscriber_ids);
+    $this->updateCount();
+  }
+
+  public function removeSubscribers(array $subscriber_ids) {
+    $this->task_subscribers->removeSubscribers($subscriber_ids);
+    $this->updateCount();
+  }
+
+  public function removeAllSubscribers() {
+    $this->task_subscribers->removeAllSubscribers();
+    $this->updateCount();
+  }
+
+  public function updateProcessedSubscribers(array $processed_subscribers) {
+    $this->task_subscribers->updateProcessedSubscribers($processed_subscribers);
+    $this->updateCount();
+  }
+
+  function updateCount() {
+    $this->queue->count_processed = ScheduledTaskSubscriber::getProcessedCount($this->task->id);
+    $this->queue->count_to_process = ScheduledTaskSubscriber::getUnprocessedCount($this->task->id);
+    $this->queue->count_total = $this->queue->count_processed + $this->queue->count_to_process;
+    return $this->queue->save();
+  }
+
+  public function __get($prop) {
+    if($this->isQueueProperty($prop)) {
+      return $this->queue->$prop;
+    } else {
+      return $this->task->$prop;
+    }
+  }
+
+  public function __set($prop, $value) {
+    if($this->isQueueProperty($prop)) {
+      $this->queue->$prop = $value;
+    } else {
+      $this->task->$prop = $value;
+    }
+  }
+
+  public function __call($name, $args) {
+    $obj = method_exists($this->queue, $name) ? $this->queue : $this->task;
+    return call_user_func_array(array($obj, $name), $args);
+  }
+
+  private function isQueueProperty($prop) {
+    return in_array($prop, $this->queue_fields);
+  }
+
+  static function getScheduledQueues() {
+    $tasks = ScheduledTask::where('status', ScheduledTask::STATUS_SCHEDULED)
+      ->whereLte('scheduled_at', Carbon::createFromTimestamp(current_time('timestamp')))
+      ->where('type', 'sending')
+      ->findMany();
+    $result = array();
+    foreach($tasks as $task) {
+      $result[] = static::createFromTask($task);
+    }
+    return $result;
+  }
+
+  static function getRunningQueues() {
+    $tasks = ScheduledTask::orderByAsc('priority')
+      ->orderByAsc('created_at')
+      ->whereNull('deleted_at')
+      ->whereNull('status')
+      ->where('type', 'sending')
+      ->findMany();
+    $result = array();
+    foreach($tasks as $task) {
+      $result[] = static::createFromTask($task);
+    }
+    return $result;
+  }
+}

lib/Tasks/Subscribers.php

@@ -13,6 +13,10 @@ class Subscribers {
     $this->task = $task;
   }
 
+  function setSubscribers(array $subscriber_ids) {
+    ScheduledTaskSubscriber::setSubscribers($this->task->id, $subscriber_ids);
+  }
+
   function getSubscribers() {
     return ScheduledTaskSubscriber::where('task_id', $this->task->id);
   }
@@ -20,17 +24,24 @@ class Subscribers {
   function isSubscriberProcessed($subscriber_id) {
     $subscriber = $this->getSubscribers()
       ->where('subscriber_id', $subscriber_id)
+      ->where('processed', ScheduledTaskSubscriber::STATUS_PROCESSED)
       ->findOne();
     return !empty($subscriber);
   }
 
-  function removeSubscribers($subscribers_to_remove) {
+  function removeSubscribers(array $subscribers_to_remove) {
     $this->getSubscribers()
       ->whereIn('subscriber_id', $subscribers_to_remove)
       ->deleteMany();
     $this->checkCompleted();
   }
 
+  function removeAllSubscribers() {
+    $this->getSubscribers()
+      ->deleteMany();
+    $this->checkCompleted();
+  }
+
   function updateProcessedSubscribers(array $processed_subscribers) {
     $this->getSubscribers()
       ->whereIn('subscriber_id', $processed_subscribers)
@@ -40,8 +51,8 @@ class Subscribers {
     $this->checkCompleted();
   }
 
-  private function checkCompleted() {
+  private function checkCompleted($count = null) {
-    if(!ScheduledTaskSubscriber::getUnprocessedCount($this->task->id)) {
+    if(!$count && !ScheduledTaskSubscriber::getUnprocessedCount($this->task->id)) {
       $this->task->complete();
     }
   }