Cavemanon/piratepoet
7
Fork 0
Code Pull Requests Actions Releases 16 Activity

Sending queue refactoring WIP [MAILPOET-903]

Browse Source
This commit is contained in:
stoletniy
2017-12-21 23:50:43 +03:00
parent bf8b0c81df
commit c0c57f6b67
17 changed files with 684 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/API/JSON/v1/Newsletters.php
View File

@ -105,8 +105,8 @@ class Newsletters extends APIEndpoint {
$newsletter->schedule = Scheduler::processPostNotificationSchedule($newsletter);
$next_run_date = Scheduler::getNextRunDate($newsletter->schedule);
// find previously scheduled jobs and reschedule them using the new "next run" date
SendingQueue::where('newsletter_id', $newsletter->id)
->where('status', SendingQueue::STATUS_SCHEDULED)
SendingQueue::findTaskByNewsletterId($newsletter->id)
->where('tasks.status', SendingQueue::STATUS_SCHEDULED)
->findResultSet()
->set('scheduled_at', $next_run_date)
->save();
@ -166,7 +166,7 @@ class Newsletters extends APIEndpoint {
// if there are past due notifications, reschedule them for the next send date
if($newsletter->type === Newsletter::TYPE_NOTIFICATION && $status === Newsletter::STATUS_ACTIVE) {
$next_run_date = Scheduler::getNextRunDate($newsletter->schedule);
$newsletter->queue()
$newsletter->queue()->task()
->whereLte('scheduled_at', Carbon::createFromTimestamp(current_time('timestamp')))
->where('status', SendingQueue::STATUS_SCHEDULED)
->findResultSet()

28
lib/API/JSON/v1/SendingQueue.php
View File

@ -8,9 +8,9 @@ use MailPoet\Config\AccessControl;
use MailPoet\Mailer\Mailer;
use MailPoet\Models\Newsletter;
use MailPoet\Models\SendingQueue as SendingQueueModel;
use MailPoet\Models\Subscriber;
use MailPoet\Newsletter\Scheduler\Scheduler;
use MailPoet\Segments\SubscribersFinder;
use MailPoet\Tasks\Sending as SendingTask;
use MailPoet\Util\Helpers;
if(!defined('ABSPATH')) exit;
@ -46,20 +46,22 @@ class SendingQueue extends APIEndpoint {
}
// add newsletter to the sending queue
$queue = SendingQueueModel::whereNull('status')
->where('newsletter_id', $newsletter->id)
->findOne();
$queue = SendingQueueModel::findTaskByNewsletterId($newsletter->id)
->whereNull('tasks.status')
->findOne();
if(!empty($queue)) {
return $this->errorResponse(array(
APIError::NOT_FOUND => __('This newsletter is already being sent.', 'mailpoet')
));
}
$queue = SendingQueueModel::where('newsletter_id', $newsletter->id)
->where('status', SendingQueueModel::STATUS_SCHEDULED)
$task = SendingQueueModel::findTaskByNewsletterId($newsletter->id)
->where('tasks.status', SendingQueueModel::STATUS_SCHEDULED)
->findOne();
if(!$queue) {
$queue = SendingQueueModel::create();
if($task) {
$queue = SendingTask::createFromTask($task);
} else {
$queue = SendingTask::create();
$queue->newsletter_id = $newsletter->id;
}
@ -72,8 +74,7 @@ class SendingQueue extends APIEndpoint {
$queue->scheduled_at = Scheduler::formatDatetimeString(
$newsletter->scheduledAt
);
$queue->subscribers = null;
$queue->count_total = $queue->count_to_process = 0;
$queue->removeAllSubscribers();
} else {
$segments = $newsletter->segments()->findArray();
$finder = new SubscribersFinder();
@ -86,12 +87,7 @@ class SendingQueue extends APIEndpoint {
}
$queue->status = null;
$queue->scheduled_at = null;
$queue->subscribers = serialize(
array(
'to_process' => $subscribers
)
);
$queue->count_total = $queue->count_to_process = count($subscribers);
$queue->setSubscribers($subscribers);
// set newsletter status
$newsletter->setStatus(Newsletter::STATUS_SENDING);

6
lib/Config/Migrator.php
View File

@ -137,18 +137,14 @@ class Migrator {
function sendingQueues() {
$attributes = array(
'id int(11) unsigned NOT NULL AUTO_INCREMENT,',
'type varchar(90) NULL DEFAULT NULL,',
'task_id int(11) unsigned NOT NULL,',
'newsletter_id int(11) unsigned NOT NULL,',
'newsletter_rendered_body longtext,',
'newsletter_rendered_subject varchar(250) NULL DEFAULT NULL,',
'subscribers longtext,',
'status varchar(12) NULL DEFAULT NULL,',
'priority mediumint(9) NOT NULL DEFAULT 0,',
'count_total int(11) unsigned NOT NULL DEFAULT 0,',
'count_processed int(11) unsigned NOT NULL DEFAULT 0,',
'count_to_process int(11) unsigned NOT NULL DEFAULT 0,',
'scheduled_at TIMESTAMP NULL,',
'processed_at TIMESTAMP NULL,',
'created_at TIMESTAMP NULL,',
'updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,',
'deleted_at TIMESTAMP NULL,',

7
lib/Cron/Daemon.php
View File

@ -1,6 +1,7 @@
<?php
namespace MailPoet\Cron;
use MailPoet\Cron\Workers\Scheduler as SchedulerWorker;
use MailPoet\Cron\Workers\SendingQueue\Migration as MigrationWorker;
use MailPoet\Cron\Workers\SendingQueue\SendingQueue as SendingQueueWorker;
use MailPoet\Cron\Workers\Bounce as BounceWorker;
use MailPoet\Cron\Workers\KeyCheck\PremiumKeyCheck as PremiumKeyCheckWorker;
@ -49,6 +50,7 @@ class Daemon {
$daemon['token'] = $this->token;
CronHelper::saveDaemon($daemon);
try {
$this->executeMigrationWorker();
$this->executeScheduleWorker();
$this->executeQueueWorker();
$this->executeSendingServiceKeyCheckWorker();
@ -100,6 +102,11 @@ class Daemon {
return $bounce->process();
}
function executeMigrationWorker() {
$migration = new MigrationWorker($this->timer);
return $migration->process();
}
function callSelf() {
CronHelper::accessDaemon($this->token);
return $this->terminateRequest();

13
lib/Cron/Triggers/WordPress.php
View File

@ -3,6 +3,7 @@ namespace MailPoet\Cron\Triggers;
use MailPoet\Cron\CronHelper;
use MailPoet\Cron\Workers\Scheduler as SchedulerWorker;
use MailPoet\Cron\Workers\SendingQueue\Migration as MigrationWorker;
use MailPoet\Cron\Workers\SendingQueue\SendingQueue as SendingQueueWorker;
use MailPoet\Cron\Workers\Bounce as BounceWorker;
use MailPoet\Cron\Workers\KeyCheck\PremiumKeyCheck as PremiumKeyCheckWorker;
@ -20,6 +21,9 @@ class WordPress {
}
static function checkExecutionRequirements() {
// migration
$migration_due_tasks = MigrationWorker::getAllDueTasks();
$migration_future_tasks = MigrationWorker::getFutureTasks();
// sending queue
$scheduled_queues = SchedulerWorker::getScheduledQueues();
$running_queues = SendingQueueWorker::getRunningQueues();
@ -42,8 +46,15 @@ class WordPress {
$bounce_sync_active = ($mp_sending_enabled && ($bounce_due_tasks || !$bounce_future_tasks));
$sending_service_key_check_active = ($mp_sending_enabled && ($msskeycheck_due_tasks || !$msskeycheck_future_tasks));
$premium_key_check_active = ($premium_key_specified && ($premium_keycheck_due_tasks || !$premium_keycheck_future_tasks));
$migration_active = $migration_due_tasks || !$migration_future_tasks;
return ($sending_queue_active || $bounce_sync_active || $sending_service_key_check_active || $premium_key_check_active);
return (
$migration_active
|| $sending_queue_active
|| $bounce_sync_active
|| $sending_service_key_check_active
|| $premium_key_check_active
);
}
static function cleanup() {

27
lib/Cron/Workers/Scheduler.php
View File

@ -4,10 +4,10 @@ namespace MailPoet\Cron\Workers;
use Carbon\Carbon;
use MailPoet\Cron\CronHelper;
use MailPoet\Models\Newsletter;
use MailPoet\Models\SendingQueue;
use MailPoet\Models\Subscriber;
use MailPoet\Models\SubscriberSegment;
use MailPoet\Segments\SubscribersFinder;
use MailPoet\Tasks\Sending as SendingTask;
use MailPoet\Util\Helpers;
use MailPoet\Newsletter\Scheduler\Scheduler as NewsletterScheduler;
@ -46,12 +46,12 @@ class Scheduler {
}
function processWelcomeNewsletter($newsletter, $queue) {
$subscriber = unserialize($queue->subscribers);
if(empty($subscriber['to_process'][0])) {
$subscribers = $queue->getSubscribers();
if(empty($subscribers[0])) {
$queue->delete();
return false;
}
$subscriber_id = (int)$subscriber['to_process'][0];
$subscriber_id = (int)$subscribers[0];
if($newsletter->event === 'segment') {
if($this->verifyMailpoetSubscriber($subscriber_id, $newsletter, $queue) === false) {
return false;
@ -91,12 +91,7 @@ class Scheduler {
// queue newsletter for delivery
$queue->newsletter_id = $notification_history->id;
$queue->subscribers = serialize(
array(
'to_process' => $subscribers
)
);
$queue->count_total = $queue->count_to_process = count($subscribers);
$queue->setSubscribers($subscribers);
$queue->status = null;
$queue->save();
// update notification status
@ -110,12 +105,7 @@ class Scheduler {
$subscribers = $finder->getSubscribersByList($segments);
$subscribers = Helpers::flattenArray($subscribers);
// update current queue
$queue->subscribers = serialize(
array(
'to_process' => $subscribers
)
);
$queue->count_total = $queue->count_to_process = count($subscribers);
$queue->setSubscribers($subscribers);
$queue->status = null;
$queue->save();
// update newsletter status
@ -189,9 +179,6 @@ class Scheduler {
}
static function getScheduledQueues() {
return SendingQueue::where('status', SendingQueue::STATUS_SCHEDULED)
->whereLte('scheduled_at', Carbon::createFromTimestamp(current_time('timestamp')))
->whereNull('type')
->findMany();
return SendingTask::getScheduledQueues();
}
}

237
lib/Cron/Workers/SendingQueue/Migration.php Normal file
View File

@ -0,0 +1,237 @@
<?php
namespace MailPoet\Cron\Workers\SendingQueue;
use Carbon\Carbon;
use MailPoet\Cron\CronHelper;
use MailPoet\Cron\Workers\SimpleWorker;
use MailPoet\Mailer\MailerLog;
use MailPoet\Models\ScheduledTask;
use MailPoet\Models\ScheduledTaskSubscriber;
use MailPoet\Models\SendingQueue;
if(!defined('ABSPATH')) exit;
class Migration extends SimpleWorker {
const TASK_TYPE = 'migration';
const BATCH_SIZE = 20;
function checkProcessingRequirements() {
// if migration was completed, don't run it again
$completed_tasks = self::getCompletedTasks();
return empty($completed_tasks);
}
function prepareTask(ScheduledTask $task) {
$unmigrated_queues_count = $this->getUnmigratedQueues()->count();
$unmigrated_queue_subscribers = $this->getTaskIdsForUnmigratedSubscribers();
if($unmigrated_queues_count == 0
&& count($unmigrated_queue_subscribers) == 0
) {
// nothing to migrate
$this->complete($task);
$this->resumeSending();
return false;
}
// pause sending while the migration is in process
$this->pauseSending();
return parent::prepareTask($task);
}
private function pauseSending() {
$mailer_log = MailerLog::getMailerLog();
if(MailerLog::isSendingPaused($mailer_log)) {
// sending is already paused
return false;
}
$mailer_log = MailerLog::setError(
$mailer_log,
'migration',
__('Your sending queue data is being migrated to allow better performance, sending is paused while the migration is in progress and will resume automatically upon completion. This may take a few minutes.')
);
return MailerLog::pauseSending($mailer_log);
}
private function resumeSending() {
$mailer_log = MailerLog::getMailerLog();
if(!MailerLog::isSendingPaused($mailer_log)) {
// sending is not paused
return false;
}
$error = MailerLog::getError($mailer_log);
// only resume sending if it was paused by migration
if(isset($error['operation']) && $error['operation'] === 'migration') {
return MailerLog::resumeSending();
}
}
function processTask(ScheduledTask $task) {
$this->migrateSendingQueues();
$this->migrateSubscribers();
$this->complete($task);
$this->resumeSending();
return true;
}
function getUnmigratedQueues() {
return SendingQueue::where('task_id', 0)
->whereNull('type');
}
function getTaskIdsForUnmigratedSubscribers() {
global $wpdb;
$query = sprintf(
'SELECT queues.`task_id` FROM %1$s queues INNER JOIN %2$s tasks ON queues.`task_id` = tasks.`id` ' .
'WHERE tasks.`type` = "sending" AND (tasks.`status` IS NULL OR tasks.`status` = "paused") ' .
'AND queues.`subscribers` != "" AND queues.`subscribers` != "N;"' .
'AND queues.`count_total` > (SELECT COUNT(*) FROM %3$s subs WHERE subs.`task_id` = queues.`task_id`)',
MP_SENDING_QUEUES_TABLE,
MP_SCHEDULED_TASKS_TABLE,
MP_SCHEDULED_TASK_SUBSCRIBERS_TABLE
);
return $wpdb->get_col($query);
}
/*
* Migrate all sending queues without converting subscriber data
*/
function migrateSendingQueues() {
global $wpdb;
$queues = $this->getUnmigratedQueues()
->select('id')
->findArray();
$column_list = array(
'status',
'priority',
'scheduled_at',
'processed_at',
'created_at',
'updated_at',
'deleted_at'
);
if(!empty($queues)) {
foreach(array_chunk($queues, self::BATCH_SIZE) as $queue_batch) {
// abort if execution limit is reached
CronHelper::enforceExecutionLimit($this->timer);
foreach($queue_batch as $queue) {
// create a new scheduled task of type "sending"
$wpdb->query(sprintf(
'INSERT IGNORE INTO %1$s (`type`, %2$s) ' .
'SELECT "sending", %2$s FROM %3$s WHERE `id` = %4$s',
MP_SCHEDULED_TASKS_TABLE,
'`' . join('`, `', $column_list) . '`',
MP_SENDING_QUEUES_TABLE,
$queue['id']
));
// link the queue with the task via task_id
$new_task_id = $wpdb->insert_id;
$wpdb->query(sprintf(
'UPDATE %1$s SET `task_id` = %2$s WHERE `id` = %3$s',
MP_SENDING_QUEUES_TABLE,
$new_task_id,
$queue['id']
));
}
}
}
return true;
}
/*
* Migrate subscribers for in-progress sending tasks from the `subscribers` field to a separate table
*/
function migrateSubscribers() {
global $wpdb;
// find in-progress queues that have serialized subscribers
$task_ids = $this->getTaskIdsForUnmigratedSubscribers();
// check if subscribers for each one were already migrated
if(!empty($task_ids)) {
$task_ids = $wpdb->get_col(sprintf(
'SELECT queues.`task_id` FROM %1$s queues WHERE queues.`task_id` IN(' . join(',', array_map('intval', $task_ids)) . ') ' .
'AND queues.`count_total` > (SELECT COUNT(*) FROM %2$s subs WHERE subs.`task_id` = queues.`task_id`)',
MP_SENDING_QUEUES_TABLE,
MP_SCHEDULED_TASK_SUBSCRIBERS_TABLE
));
}
if(!empty($task_ids)) {
foreach($task_ids as $task_id) {
// abort if execution limit is reached
CronHelper::enforceExecutionLimit($this->timer);
$this->migrateTaskSubscribers($task_id);
}
}
return true;
}
function migrateTaskSubscribers($task_id) {
global $wpdb;
$migrated_unprocessed_count = ScheduledTaskSubscriber::getUnprocessedCount($task_id);
$migrated_processed_count = ScheduledTaskSubscriber::getProcessedCount($task_id);
$subscribers = $wpdb->get_var(sprintf(
'SELECT `subscribers` FROM %1$s WHERE `task_id` = %2$d ' .
'AND (`count_processed` > %3$d OR `count_to_process` > %4$d)',
MP_SENDING_QUEUES_TABLE,
$task_id,
$migrated_unprocessed_count,
$migrated_processed_count
));
// sanity check
if(empty($subscribers)) {
return false;
}
$subscribers = unserialize($subscribers);
if(!empty($subscribers['to_process'])) {
$subscribers_to_migrate = array_slice($subscribers['to_process'], $migrated_unprocessed_count);
foreach($subscribers_to_migrate as $sub_id) {
// abort if execution limit is reached
CronHelper::enforceExecutionLimit($this->timer);
ScheduledTaskSubscriber::createOrUpdate(array(
'task_id' => $task_id,
'subscriber_id' => $sub_id,
'processed' => ScheduledTaskSubscriber::STATUS_UNPROCESSED
));
}
}
if(!empty($subscribers['processed'])) {
$subscribers_to_migrate = array_slice($subscribers['processed'], $migrated_processed_count);
foreach($subscribers_to_migrate as $sub_id) {
// abort if execution limit is reached
CronHelper::enforceExecutionLimit($this->timer);
ScheduledTaskSubscriber::createOrUpdate(array(
'task_id' => $task_id,
'subscriber_id' => $sub_id,
'processed' => ScheduledTaskSubscriber::STATUS_PROCESSED
));
}
}
return true;
}
static function getNextRunDate() {
// run migration immediately
return Carbon::now();
}
}

23
lib/Cron/Workers/SendingQueue/SendingQueue.php
View File

@ -6,10 +6,12 @@ use MailPoet\Cron\Workers\SendingQueue\Tasks\Links;
use MailPoet\Cron\Workers\SendingQueue\Tasks\Mailer as MailerTask;
use MailPoet\Cron\Workers\SendingQueue\Tasks\Newsletter as NewsletterTask;
use MailPoet\Mailer\MailerLog;
use MailPoet\Models\SendingQueue as SendingQueueModel;
use MailPoet\Models\ScheduledTask as ScheduledTaskModel;
use MailPoet\Models\StatisticsNewsletters as StatisticsNewslettersModel;
use MailPoet\Models\Subscriber as SubscriberModel;
use MailPoet\Segments\SubscribersFinder;
use MailPoet\Tasks\Sending as SendingTask;
use MailPoet\Tasks\Subscribers\BatchIterator;
use MailPoet\WP\Hooks as WPHooks;
if(!defined('ABSPATH')) exit;
@ -45,11 +47,7 @@ class SendingQueue {
// get newsletter segments
$newsletter_segments_ids = $this->newsletter_task->getNewsletterSegments($newsletter);
// get subscribers
$queue->subscribers = $queue->getSubscribers();
$subscriber_batches = array_chunk(
$queue->subscribers['to_process'],
$this->batch_size
);
$subscriber_batches = new BatchIterator($queue->task_id, $this->batch_size);
foreach($subscriber_batches as $subscribers_to_process_ids) {
if(!empty($newsletter_segments_ids[0])) {
// Check that subscribers are in segments
@ -72,7 +70,7 @@ class SendingQueue {
$found_subscribers_ids
);
$queue->removeSubscribers($subscribers_to_remove);
if(!count($queue->subscribers['to_process'])) {
if(!$queue->count_to_process) {
$this->newsletter_task->markNewsletterAsSent($newsletter, $queue);
continue;
}
@ -82,7 +80,7 @@ class SendingQueue {
$newsletter,
$found_subscribers
);
if($queue->status === SendingQueueModel::STATUS_COMPLETED) {
if($queue->status === ScheduledTaskModel::STATUS_COMPLETED) {
$this->newsletter_task->markNewsletterAsSent($newsletter, $queue);
}
$this->enforceSendingAndExecutionLimits();
@ -179,7 +177,7 @@ class SendingQueue {
// update the sent count
$this->mailer_task->updateSentCount();
// enforce execution limits if queue is still being processed
if($queue->status !== SendingQueueModel::STATUS_COMPLETED) {
if($queue->status !== ScheduledTaskModel::STATUS_COMPLETED) {
$this->enforceSendingAndExecutionLimits();
}
return $queue;
@ -193,11 +191,6 @@ class SendingQueue {
}
static function getRunningQueues() {
return SendingQueueModel::orderByAsc('priority')
->orderByAsc('created_at')
->whereNull('deleted_at')
->whereNull('status')
->whereNull('type')
->findMany();
return SendingTask::getRunningQueues();
}
}

9
lib/Cron/Workers/SimpleWorker.php
View File

@ -62,7 +62,7 @@ abstract class SimpleWorker {
$task->type = static::TASK_TYPE;
$task->status = ScheduledTask::STATUS_SCHEDULED;
$task->priority = ScheduledTask::PRIORITY_LOW;
$task->scheduled_at = self::getNextRunDate();
$task->scheduled_at = static::getNextRunDate();
$task->save();
return $task;
}
@ -139,4 +139,11 @@ abstract class SimpleWorker {
static function getFutureTasks() {
return self::getScheduledTasks(true);
}
static function getCompletedTasks() {
return ScheduledTask::where('type', static::TASK_TYPE)
->whereNull('deleted_at')
->where('status', ScheduledTask::STATUS_COMPLETED)
->findMany();
}
}

23
lib/Mailer/MailerLog.php
View File

@ -79,6 +79,17 @@ class MailerLog {
$mailer_log = self::getMailerLog();
(int)$mailer_log['retry_attempt']++;
$mailer_log['retry_at'] = time() + self::RETRY_INTERVAL;
$mailer_log = self::setError($mailer_log, $operation, $error_message, $error_code);
self::updateMailerLog($mailer_log);
if($pause_sending) {
self::pauseSending($mailer_log);
} else {
self::updateMailerLog($mailer_log);
}
return self::enforceExecutionRequirements();
}
static function setError($mailer_log, $operation, $error_message, $error_code = null) {
$mailer_log['error'] = array(
'operation' => $operation,
'error_message' => $error_message
@ -86,12 +97,12 @@ class MailerLog {
if($error_code) {
$mailer_log['error']['error_code'] = $error_code;
}
if($pause_sending) {
self::pauseSending($mailer_log);
} else {
self::updateMailerLog($mailer_log);
}
return self::enforceExecutionRequirements();
return $mailer_log;
}
static function getError($mailer_log = false) {
$mailer_log = self::getMailerLog($mailer_log);
return isset($mailer_log['error']) ? $mailer_log['error'] : null;
}
static function incrementSentCount() {

98
lib/Models/Newsletter.php
View File

@ -2,6 +2,7 @@
namespace MailPoet\Models;
use Carbon\Carbon;
use MailPoet\Newsletter\Renderer\Renderer;
use MailPoet\Tasks\Sending as SendingTask;
use MailPoet\Util\Helpers;
use MailPoet\Util\Security;
use MailPoet\WP\Emoji;
@ -107,12 +108,24 @@ class Newsletter extends Model {
'SET `deleted_at` = NOW() ' .
'WHERE `parent_id` = ' . $this->id
);
ScheduledTask::rawExecute(
'UPDATE `' . ScheduledTask::$_table . '` t ' .
'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
'SET t.`deleted_at` = NOW() ' .
'WHERE q.`newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), array($this->id))) . ')'
);
SendingQueue::rawExecute(
'UPDATE `' . SendingQueue::$_table . '` ' .
'SET `deleted_at` = NOW() ' .
'WHERE `newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), array($this->id))) . ')'
);
} else {
ScheduledTask::rawExecute(
'UPDATE `' . ScheduledTask::$_table . '` t ' .
'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
'SET t.`deleted_at` = NOW() ' .
'WHERE q.`newsletter_id` = ' . $this->id
);
SendingQueue::rawExecute(
'UPDATE `' . SendingQueue::$_table . '` ' .
'SET `deleted_at` = NOW() ' .
@ -133,12 +146,24 @@ class Newsletter extends Model {
'SET `deleted_at` = NOW() ' .
'WHERE `parent_id` IN (' . join(',', Helpers::flattenArray($ids)) . ')'
);
ScheduledTask::rawExecute(
'UPDATE `' . ScheduledTask::$_table . '` t ' .
'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
'SET t.`deleted_at` = NOW() ' .
'WHERE q.`newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), $ids)) . ')'
);
SendingQueue::rawExecute(
'UPDATE `' . SendingQueue::$_table . '` ' .
'SET `deleted_at` = NOW() ' .
'WHERE `newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), $ids)) . ')'
);
} else {
ScheduledTask::rawExecute(
'UPDATE `' . ScheduledTask::$_table . '` t ' .
'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
'SET t.`deleted_at` = NOW() ' .
'WHERE q.`newsletter_id` IN (' . join(',', Helpers::flattenArray($ids)) . ')'
);
SendingQueue::rawExecute(
'UPDATE `' . SendingQueue::$_table . '` ' .
'SET `deleted_at` = NOW() ' .
@ -156,9 +181,11 @@ class Newsletter extends Model {
if($children) {
$children = Helpers::flattenArray($children);
$this->children()->deleteMany();
SendingQueue::getTasks()->whereIn('queues.newsletter_id', array_merge($children, array($this->id)))->deleteMany();
SendingQueue::whereIn('newsletter_id', array_merge($children, array($this->id)))->deleteMany();
NewsletterSegment::whereIn('newsletter_id', array_merge($children, array($this->id)))->deleteMany();
} else {
SendingQueue::getTasks()->where('queues.newsletter_id', $this->id)->deleteMany();
$this->queue()->deleteMany();
$this->segmentRelations()->deleteMany();
}
@ -173,9 +200,11 @@ class Newsletter extends Model {
if($children) {
$children = Helpers::flattenArray($children);
Newsletter::whereIn('parent_id', $ids)->deleteMany();
SendingQueue::getTasks()->whereIn('queues.newsletter_id', array_merge($children, $ids))->deleteMany();
SendingQueue::whereIn('newsletter_id', array_merge($children, $ids))->deleteMany();
NewsletterSegment::whereIn('newsletter_id', array_merge($children, $ids))->deleteMany();
} else {
SendingQueue::getTasks()->whereIn('queues.newsletter_id', $ids)->deleteMany();
SendingQueue::whereIn('newsletter_id', $ids)->deleteMany();
NewsletterSegment::whereIn('newsletter_id', $ids)->deleteMany();
}
@ -193,12 +222,24 @@ class Newsletter extends Model {
'SET `deleted_at` = null ' .
'WHERE `parent_id` = ' . $this->id
);
ScheduledTask::rawExecute(
'UPDATE `' . ScheduledTask::$_table . '` t ' .
'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
'SET t.`deleted_at` = null ' .
'WHERE q.`newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), array($this->id))) . ')'
);
SendingQueue::rawExecute(
'UPDATE `' . SendingQueue::$_table . '` ' .
'SET `deleted_at` = null ' .
'WHERE `newsletter_id` IN (' . join(',', array_merge(Helpers::flattenArray($children), array($this->id))) . ')'
);
} else {
ScheduledTask::rawExecute(
'UPDATE `' . ScheduledTask::$_table . '` t ' .
'JOIN `' . SendingQueue::$_table . '` q ON t.`id` = q.`task_id` ' .
'SET t.`deleted_at` = null ' .
'WHERE q.`newsletter_id` = ' . $this->id
);
SendingQueue::rawExecute(
'UPDATE `' . SendingQueue::$_table . '` ' .
'SET `deleted_at` = null ' .
@ -223,12 +264,24 @@ class Newsletter extends Model {
->findResultSet()
->set('deleted_at', null)
->save();
SendingQueue::getTasks()
->whereIn('queues.newsletter_id', Helpers::flattenArray($children))
->whereNotNull('deleted_at')
->findResultSet()
->set('deleted_at', null)
->save();
SendingQueue::whereIn('newsletter_id', Helpers::flattenArray($children))
->whereNotNull('deleted_at')
->findResultSet()
->set('deleted_at', null)
->save();
} else {
SendingQueue::getTasks()
->whereIn('queues.newsletter_id', $ids)
->whereNotNull('deleted_at')
->findResultSet()
->set('deleted_at', null)
->save();
SendingQueue::whereIn('newsletter_id', $ids)
->whereNotNull('deleted_at')
->findResultSet()
@ -408,23 +461,11 @@ class Newsletter extends Model {
}
function getQueue($columns = '*') {
return SendingQueue::select($columns)
->where('newsletter_id', $this->id)
->orderByDesc('updated_at')
->findOne();
return SendingTask::getByNewsletterId($this->id);
}
function withSendingQueue() {
$queue = $this->getQueue(array(
'id',
'newsletter_id',
'newsletter_rendered_subject',
'status',
'count_processed',
'count_total',
'scheduled_at',
'created_at'
));
$queue = $this->getQueue();
if($queue === false) {
$this->queue = false;
} else {
@ -445,9 +486,9 @@ class Newsletter extends Model {
function withTotalSent() {
// total of subscribers who received the email
$this->total_sent = (int)SendingQueue::where('newsletter_id', $this->id)
->where('status', SendingQueue::STATUS_COMPLETED)
->sum('count_processed');
$this->total_sent = (int)SendingQueue::findTaskByNewsletterId($this->id)
->where('tasks.status', SendingQueue::STATUS_COMPLETED)
->sum('queues.count_processed');
return $this;
}
@ -480,8 +521,9 @@ class Newsletter extends Model {
} else {
$row = $statisticsExpr
->join(MP_SENDING_QUEUES_TABLE, array("queue_id", "=", "qt.id"), "qt")
->join(MP_SCHEDULED_TASKS_TABLE, array("qt.task_Id", "=", "tasks.id"), "tasks")
->where(array(
"qt.status" => SendingQueue::STATUS_COMPLETED,
"tasks.status" => SendingQueue::STATUS_COMPLETED,
"stat.newsletter_id" => $this->id,
))->findOne();
}
@ -511,8 +553,13 @@ class Newsletter extends Model {
'queues.newsletter_id = newsletters.id',
'queues'
)
->where('queues.status', SendingQueue::STATUS_COMPLETED)
->whereGte('queues.processed_at', Carbon::now()->subMonths(3))
->join(
MP_SCHEDULED_TASKS_TABLE,
'queues.task_id = tasks.id',
'tasks'
)
->where('tasks.status', SendingQueue::STATUS_COMPLETED)
->whereGte('tasks.processed_at', Carbon::now()->subMonths(3))
->count();
@ -862,10 +909,15 @@ class Newsletter extends Model {
'queues.newsletter_id = newsletters.id',
'queues'
)
->where('queues.status', SendingQueue::STATUS_COMPLETED)
->join(
MP_SCHEDULED_TASKS_TABLE,
'queues.task_id = tasks.id',
'tasks'
)
->where('tasks.status', SendingQueue::STATUS_COMPLETED)
->whereNull('newsletters.deleted_at')
->select('queues.processed_at')
->orderByDesc('queues.processed_at');
->select('tasks.processed_at')
->orderByDesc('tasks.processed_at');
if(!empty($segment_ids)) {
$orm->join(

22
lib/Models/ScheduledTask.php
View File

@ -7,10 +7,32 @@ class ScheduledTask extends Model {
public static $_table = MP_SCHEDULED_TASKS_TABLE;
const STATUS_COMPLETED = 'completed';
const STATUS_SCHEDULED = 'scheduled';
const STATUS_PAUSED = 'paused';
const PRIORITY_HIGH = 1;
const PRIORITY_MEDIUM = 5;
const PRIORITY_LOW = 10;
function subscribers() {
return $this->hasManyThrough(
__NAMESPACE__.'\Subscriber',
__NAMESPACE__.'\ScheduledTaskSubscriber',
'task_id',
'subscriber_id'
);
}
function pause() {
$this->set('status', self::STATUS_PAUSED);
$this->save();
return ($this->getErrors() === false && $this->id() > 0);
}
function resume() {
$this->setExpr('status', 'NULL');
$this->save();
return ($this->getErrors() === false && $this->id() > 0);
}
function complete() {
$this->processed_at = current_time('mysql');
$this->set('status', self::STATUS_COMPLETED);

13
lib/Models/ScheduledTaskSubscriber.php
View File

@ -10,6 +10,10 @@ class ScheduledTaskSubscriber extends Model {
public static $_table = MP_SCHEDULED_TASK_SUBSCRIBERS_TABLE;
public static $_id_column = array('task_id', 'subscriber_id');
function task() {
return $this->hasOne(__NAMESPACE__ . '\ScheduledTask', 'id', 'task_id');
}
static function createOrUpdate($data = array()) {
if(!is_array($data) || empty($data['task_id']) || empty($data['subscriber_id'])) {
return;
@ -21,6 +25,11 @@ class ScheduledTaskSubscriber extends Model {
));
}
static function setSubscribers($task_id, array $subscriber_ids) {
static::clearSubscribers($task_id);
return static::addSubscribers($task_id, $subscriber_ids);
}
static function addSubscribers($task_id, array $subscriber_ids) {
foreach($subscriber_ids as $subscriber_id) {
self::createOrUpdate(array(
@ -30,6 +39,10 @@ class ScheduledTaskSubscriber extends Model {
}
}
static function clearSubscribers($task_id) {
return self::where('task_id', $task_id)->deleteMany();
}
static function getUnprocessedCount($task_id) {
return self::getCount($task_id, self::STATUS_UNPROCESSED);
}

111
lib/Models/SendingQueue.php
View File

@ -3,6 +3,7 @@ namespace MailPoet\Models;
use MailPoet\Util\Helpers;
use MailPoet\WP\Emoji;
use MailPoet\Tasks\Subscribers as TaskSubscribers;
if(!defined('ABSPATH')) exit;
@ -23,6 +24,10 @@ class SendingQueue extends Model {
));
}
function task() {
return $this->hasOne(__NAMESPACE__ . '\ScheduledTask', 'id', 'task_id');
}
function newsletter() {
return $this->has_one(__NAMESPACE__ . '\Newsletter', 'id', 'newsletter_id');
}
@ -31,9 +36,7 @@ class SendingQueue extends Model {
if($this->count_processed === $this->count_total) {
return false;
} else {
$this->set('status', self::STATUS_PAUSED);
$this->save();
return ($this->getErrors() === false && $this->id() > 0);
return $this->task()->findOne()->pause();
}
}
@ -41,23 +44,16 @@ class SendingQueue extends Model {
if($this->count_processed === $this->count_total) {
return $this->complete();
} else {
$this->setExpr('status', 'NULL');
$this->save();
return ($this->getErrors() === false && $this->id() > 0);
return $this->task()->findOne()->resume();
}
}
function complete() {
$this->set('status', self::STATUS_COMPLETED);
$this->save();
return ($this->getErrors() === false && $this->id() > 0);
return $this->task()->findOne()->complete();
}
function save() {
$this->newsletter_rendered_body = $this->getNewsletterRenderedBody();
if(!is_serialized($this->subscribers) && !is_null($this->subscribers)) {
$this->set('subscribers', serialize($this->subscribers));
}
if(!Helpers::isJson($this->newsletter_rendered_body) && !is_null($this->newsletter_rendered_body)) {
$this->set(
'newsletter_rendered_body',
@ -69,22 +65,10 @@ class SendingQueue extends Model {
$this->priority = self::PRIORITY_MEDIUM;
}
parent::save();
$this->subscribers = $this->getSubscribers();
$this->newsletter_rendered_body = $this->getNewsletterRenderedBody();
return $this;
}
function getSubscribers() {
if(!is_serialized($this->subscribers)) {
return $this->subscribers;
}
$subscribers = unserialize($this->subscribers);
if(empty($subscribers['processed'])) {
$subscribers['processed'] = array();
}
return $subscribers;
}
function getNewsletterRenderedBody($type = false) {
$rendered_newsletter = $this->decodeRenderedNewsletterBodyObject($this->newsletter_rendered_body);
return ($type && !empty($rendered_newsletter[$type])) ?
@ -115,57 +99,16 @@ class SendingQueue extends Model {
}
function isSubscriberProcessed($subscriber_id) {
$subscribers = $this->getSubscribers();
return in_array($subscriber_id, $subscribers['processed']);
return (new TaskSubscribers($this->task()->findOne()))
->isSubscriberProcessed($subscriber_id);
}
function asArray() {
$model = parent::asArray();
$model['subscribers'] = $this->getSubscribers();
$model['newsletter_rendered_body'] = $this->getNewsletterRenderedBody();
return $model;
}
function removeSubscribers($subscribers_to_remove) {
$subscribers = $this->getSubscribers();
$subscribers['to_process'] = array_values(
array_diff(
$subscribers['to_process'],
$subscribers_to_remove
)
);
$this->subscribers = $subscribers;
$this->updateCount();
}
function updateProcessedSubscribers($processed_subscribers) {
$subscribers = $this->getSubscribers();
$subscribers['processed'] = array_merge(
$subscribers['processed'],
$processed_subscribers
);
$subscribers['to_process'] = array_values(
array_diff(
$subscribers['to_process'],
$processed_subscribers
)
);
$this->subscribers = $subscribers;
return $this->updateCount()->getErrors() === false;
}
function updateCount() {
$this->subscribers = $this->getSubscribers();
$this->count_processed = count($this->subscribers['processed']);
$this->count_to_process = count($this->subscribers['to_process']);
$this->count_total = $this->count_processed + $this->count_to_process;
if(!$this->count_to_process) {
$this->processed_at = current_time('mysql');
$this->status = self::STATUS_COMPLETED;
}
return $this->save();
}
private function decodeRenderedNewsletterBodyObject($rendered_body) {
if(is_serialized($rendered_body)) {
return $this->decodeEmojisInBody(unserialize($rendered_body));
@ -175,4 +118,36 @@ class SendingQueue extends Model {
}
return $rendered_body;
}
}
static function getTasks() {
return ScheduledTask::table_alias('tasks')
->join(
MP_SENDING_QUEUES_TABLE,
'tasks.id = queues.task_id',
'queues'
);
}
static function joinWithTasks() {
return static::table_alias('queues')
->join(
MP_SCHEDULED_TASKS_TABLE,
'tasks.id = queues.task_id',
'tasks'
);
}
static function joinWithSubscribers() {
return static::joinWithTasks()
->join(
MP_SCHEDULED_TASK_SUBSCRIBERS_TABLE,
'tasks.id = subscribers.task_id',
'subscribers'
);
}
static function findTaskByNewsletterId($newsletter_id) {
return static::joinWithTasks()
->where('queues.newsletter_id', $newsletter_id);
}
}

17
lib/Newsletter/Scheduler/Scheduler.php
View File

@ -7,6 +7,7 @@ use MailPoet\Models\NewsletterOption;
use MailPoet\Models\NewsletterOptionField;
use MailPoet\Models\NewsletterPost;
use MailPoet\Models\SendingQueue;
use MailPoet\Tasks\Sending as SendingTask;
class Scheduler {
const SECONDS_IN_HOUR = 3600;
@ -73,18 +74,14 @@ class Scheduler {
}
static function createWelcomeNotificationQueue($newsletter, $subscriber_id) {
$previously_scheduled_notification = SendingQueue::where('newsletter_id', $newsletter->id)
->whereLike('subscribers', '%' . serialize(array($subscriber_id)) . '%')
$previously_scheduled_notification = SendingQueue::joinWithSubscribers()
->where('queues.newsletter_id', $newsletter->id)
->where('subscribers.subscriber_id', $subscriber_id)
->findOne();
if(!empty($previously_scheduled_notification)) return;
$queue = SendingQueue::create();
$queue = SendingTask::create();
$queue->newsletter_id = $newsletter->id;
$queue->subscribers = serialize(
array(
'to_process' => array($subscriber_id)
)
);
$queue->count_total = $queue->count_to_process = 1;
$queue->setSubscribers(array($subscriber_id));
$after_time_type = $newsletter->afterTimeType;
$after_time_number = $newsletter->afterTimeNumber;
$scheduled_at = null;
@ -116,7 +113,7 @@ class Scheduler {
->where('scheduled_at', $next_run_date)
->findOne();
if($existing_queue) return;
$queue = SendingQueue::create();
$queue = SendingTask::create();
$queue->newsletter_id = $newsletter->id;
$queue->status = SendingQueue::STATUS_SCHEDULED;
$queue->scheduled_at = $next_run_date;

198
lib/Tasks/Sending.php Normal file
View File

@ -0,0 +1,198 @@
<?php
namespace MailPoet\Tasks;
use Carbon\Carbon;
use MailPoet\Models\ScheduledTask;
use MailPoet\Models\ScheduledTaskSubscriber;
use MailPoet\Models\SendingQueue;
use MailPoet\Util\Helpers;
if(!defined('ABSPATH')) exit;
/**
* A facade class containing all necessary models to work with a sending queue
*/
class Sending {
const TASK_TYPE = 'sending';
private $task;
private $queue;
private $task_subscribers;
private $queue_fields = array(
'id',
'task_id',
'newsletter_id',
'newsletter_rendered_subject',
'newsletter_rendered_body',
'count_total',
'count_processed',
'count_to_process'
);
public function __construct(ScheduledTask $task, SendingQueue $queue) {
$this->task = $task;
$this->queue = $queue;
$this->task_subscribers = new Subscribers($task);
}
static function create(ScheduledTask $task = null, SendingQueue $queue = null) {
if(is_null($task) && is_null($queue)) {
$task = ScheduledTask::create();
$task->type = self::TASK_TYPE;
$task->save();
$queue = SendingQueue::create();
$queue->newsletter_id = 0;
$queue->task_id = $task->id;
$queue->save();
}
return new self($task, $queue);
}
static function createFromTask(ScheduledTask $task) {
$queue = SendingQueue::where('task_id', $task->id)->findOne();
if(!$queue) {
return false;
}
return self::create($task, $queue);
}
static function createFromQueue(SendingQueue $queue) {
$task = $queue->task()->findOne();
if(!$task) {
return false;
}
return self::create($task, $queue);
}
static function getByNewsletterId($newsletter_id) {
$queue = SendingQueue::where('newsletter_id', $newsletter_id)
->orderByDesc('updated_at')
->findOne();
if(!$queue) {
return false;
}
return self::createFromQueue($queue);
}
public function asArray() {
$queue = array_intersect_key(
$this->queue->asArray(),
array_flip($this->queue_fields)
);
$task = $this->task->asArray();
return array_merge($task, $queue);
}
public function save() {
$this->task->save();
$this->queue->save();
return $this;
}
public function delete() {
$this->task_subscribers->removeAllSubscribers();
$this->task->delete();
$this->queue->delete();
}
public function queue() {
return $this->queue;
}
public function task() {
return $this->task;
}
public function taskSubscribers() {
return $this->task_subscribers;
}
public function getSubscribers() {
$subscribers = $this->task_subscribers->getSubscribers()->findArray();
return Helpers::arrayColumn($subscribers, 'subscriber_id');
}
public function setSubscribers(array $subscriber_ids) {
$this->task_subscribers->setSubscribers($subscriber_ids);
$this->updateCount();
}
public function removeSubscribers(array $subscriber_ids) {
$this->task_subscribers->removeSubscribers($subscriber_ids);
$this->updateCount();
}
public function removeAllSubscribers() {
$this->task_subscribers->removeAllSubscribers();
$this->updateCount();
}
public function updateProcessedSubscribers(array $processed_subscribers) {
$this->task_subscribers->updateProcessedSubscribers($processed_subscribers);
$this->updateCount();
}
function updateCount() {
$this->queue->count_processed = ScheduledTaskSubscriber::getProcessedCount($this->task->id);
$this->queue->count_to_process = ScheduledTaskSubscriber::getUnprocessedCount($this->task->id);
$this->queue->count_total = $this->queue->count_processed + $this->queue->count_to_process;
return $this->queue->save();
}
public function __get($prop) {
if($this->isQueueProperty($prop)) {
return $this->queue->$prop;
} else {
return $this->task->$prop;
}
}
public function __set($prop, $value) {
if($this->isQueueProperty($prop)) {
$this->queue->$prop = $value;
} else {
$this->task->$prop = $value;
}
}
public function __call($name, $args) {
$obj = method_exists($this->queue, $name) ? $this->queue : $this->task;
return call_user_func_array(array($obj, $name), $args);
}
private function isQueueProperty($prop) {
return in_array($prop, $this->queue_fields);
}
static function getScheduledQueues() {
$tasks = ScheduledTask::where('status', ScheduledTask::STATUS_SCHEDULED)
->whereLte('scheduled_at', Carbon::createFromTimestamp(current_time('timestamp')))
->where('type', 'sending')
->findMany();
$result = array();
foreach($tasks as $task) {
$result[] = static::createFromTask($task);
}
return $result;
}
static function getRunningQueues() {
$tasks = ScheduledTask::orderByAsc('priority')
->orderByAsc('created_at')
->whereNull('deleted_at')
->whereNull('status')
->where('type', 'sending')
->findMany();
$result = array();
foreach($tasks as $task) {
$result[] = static::createFromTask($task);
}
return $result;
}
}

17
lib/Tasks/Subscribers.php
View File

@ -13,6 +13,10 @@ class Subscribers {
$this->task = $task;
}
function setSubscribers(array $subscriber_ids) {
ScheduledTaskSubscriber::setSubscribers($this->task->id, $subscriber_ids);
}
function getSubscribers() {
return ScheduledTaskSubscriber::where('task_id', $this->task->id);
}
@ -20,17 +24,24 @@ class Subscribers {
function isSubscriberProcessed($subscriber_id) {
$subscriber = $this->getSubscribers()
->where('subscriber_id', $subscriber_id)
->where('processed', ScheduledTaskSubscriber::STATUS_PROCESSED)
->findOne();
return !empty($subscriber);
}
function removeSubscribers($subscribers_to_remove) {
function removeSubscribers(array $subscribers_to_remove) {
$this->getSubscribers()
->whereIn('subscriber_id', $subscribers_to_remove)
->deleteMany();
$this->checkCompleted();
}
function removeAllSubscribers() {
$this->getSubscribers()
->deleteMany();
$this->checkCompleted();
}
function updateProcessedSubscribers(array $processed_subscribers) {
$this->getSubscribers()
->whereIn('subscriber_id', $processed_subscribers)
@ -40,8 +51,8 @@ class Subscribers {
$this->checkCompleted();
}
private function checkCompleted() {
if(!ScheduledTaskSubscriber::getUnprocessedCount($this->task->id)) {
private function checkCompleted($count = null) {
if(!$count && !ScheduledTaskSubscriber::getUnprocessedCount($this->task->id)) {
$this->task->complete();
}
}