From 999a0b3edec7813a4b70a36b069ee7a861dcefe5 Mon Sep 17 00:00:00 2001
From: Vlad <vlad@mailpoet.com>
Date: Fri, 17 Jun 2016 07:27:06 -0400
Subject: [PATCH] - Refactors sending queue worker by breaking it into smaller
 tasks - Adds arrayUnique method to Helpers for multidimensional arrays

---
 lib/Cron/Workers/SendingQueue.php             | 407 ------------------
 .../Workers/SendingQueue/SendingQueue.php     | 181 ++++++++
 lib/Cron/Workers/SendingQueue/Tasks/Links.php |  33 ++
 .../Workers/SendingQueue/Tasks/Mailer.php     |  95 ++++
 .../Workers/SendingQueue/Tasks/Newsletter.php |  96 +++++
 lib/Cron/Workers/SendingQueue/Tasks/Posts.php |  29 ++
 .../Workers/SendingQueue/Tasks/Shortcodes.php |  14 +
 .../Workers/SendingQueue/Tasks/Statistics.php |  30 ++
 .../SendingQueue/Tasks/Subscribers.php        |  41 ++
 lib/Models/StatisticsNewsletters.php          |   3 +
 .../Renderer/PostProcess/OpenTracking.php     |  10 +-
 lib/Newsletter/Shortcodes/Shortcodes.php      |  24 +-
 lib/Util/Helpers.php                          |  10 +
 13 files changed, 556 insertions(+), 417 deletions(-)
 delete mode 100644 lib/Cron/Workers/SendingQueue.php
 create mode 100644 lib/Cron/Workers/SendingQueue/SendingQueue.php
 create mode 100644 lib/Cron/Workers/SendingQueue/Tasks/Links.php
 create mode 100644 lib/Cron/Workers/SendingQueue/Tasks/Mailer.php
 create mode 100644 lib/Cron/Workers/SendingQueue/Tasks/Newsletter.php
 create mode 100644 lib/Cron/Workers/SendingQueue/Tasks/Posts.php
 create mode 100644 lib/Cron/Workers/SendingQueue/Tasks/Shortcodes.php
 create mode 100644 lib/Cron/Workers/SendingQueue/Tasks/Statistics.php
 create mode 100644 lib/Cron/Workers/SendingQueue/Tasks/Subscribers.php

diff --git a/lib/Cron/Workers/SendingQueue.php b/lib/Cron/Workers/SendingQueue.php
deleted file mode 100644
index 2125be99cb..0000000000
--- a/lib/Cron/Workers/SendingQueue.php
+++ /dev/null
@@ -1,407 +0,0 @@
-<?php
-namespace MailPoet\Cron\Workers;
-
-use MailPoet\Cron\CronHelper;
-use MailPoet\Mailer\Mailer;
-use MailPoet\Models\Newsletter;
-use MailPoet\Models\SendingQueue as SendingQueueModel;
-use MailPoet\Models\NewsletterPost;
-use MailPoet\Models\Setting;
-use MailPoet\Models\StatisticsNewsletters;
-use MailPoet\Models\Subscriber;
-use MailPoet\Newsletter\Links\Links;
-use MailPoet\Newsletter\Renderer\PostProcess\OpenTracking;
-use MailPoet\Newsletter\Renderer\Renderer;
-use MailPoet\Newsletter\Shortcodes\Shortcodes;
-use MailPoet\Util\Helpers;
-
-if(!defined('ABSPATH')) exit;
-
-class SendingQueue {
-  public $mta_config;
-  public $mta_log;
-  public $processing_method;
-  private $timer;
-  const BATCH_SIZE = 50;
-  const DIVIDER = '***MailPoet***';
-
-  function __construct($timer = false) {
-    $this->mta_config = $this->getMailerConfig();
-    $this->mta_log = $this->getMailerLog();
-    $this->processing_method = ($this->mta_config['method'] === 'MailPoet') ?
-      'processBulkSubscribers' :
-      'processIndividualSubscriber';
-    $this->timer = ($timer) ? $timer : microtime(true);
-    CronHelper::checkExecutionTimer($this->timer);
-  }
-
-  function process() {
-    foreach($this->getQueues() as $queue) {
-      $newsletter = Newsletter::findOne($queue->newsletter_id);
-      if(!$newsletter) {
-        $queue->delete();
-        continue;
-      }
-      $newsletter = $newsletter->asArray();
-      $newsletter['body'] = $this->getOrRenderNewsletterBody($queue, $newsletter);
-      if($newsletter['type'] === 'notification' &&
-          strpos($newsletter['body']['html'], 'data-post-id') === false
-      ){
-        $queue->delete();
-        continue;
-      }
-      $queue->subscribers = (object) unserialize($queue->subscribers);
-      if(!isset($queue->subscribers->processed)) {
-        $queue->subscribers->processed = array();
-      }
-      if(!isset($queue->subscribers->failed)) {
-        $queue->subscribers->failed = array();
-      }
-      $mailer = $this->configureMailer($newsletter);
-      foreach(array_chunk($queue->subscribers->to_process, self::BATCH_SIZE) as
-              $subscribers_ids) {
-        $subscribers = Subscriber::whereIn('id', $subscribers_ids)
-          ->findArray();
-        if(count($subscribers_ids) !== count($subscribers)) {
-          $queue->subscribers->to_process = $this->recalculateSubscriberCount(
-            Helpers::arrayColumn($subscribers, 'id'),
-            $subscribers_ids,
-            $queue->subscribers->to_process
-          );
-        }
-        if(!count($queue->subscribers->to_process)) {
-          $this->updateQueue($queue);
-          continue;
-        }
-        $queue->subscribers = call_user_func_array(
-          array(
-            $this,
-            $this->processing_method
-          ),
-          array(
-            $mailer,
-            $newsletter,
-            $subscribers,
-            $queue
-          )
-        );
-      }
-    }
-  }
-
-  function getOrRenderNewsletterBody($queue, $newsletter) {
-    // check if newsletter has been rendered, in which case return its contents
-    // or render and save for future reuse
-    if($queue->newsletter_rendered_body === null) {
-      if((boolean) Setting::getValue('tracking.enabled')) {
-        // insert tracking code
-        add_filter('mailpoet_rendering_post_process', function($template) {
-          return OpenTracking::process($template);
-        });
-        // render newsletter
-        $rendered_newsletter = $this->renderNewsletter($newsletter);
-        // process link shortcodes, extract and save links in the database
-        $processed_newsletter = $this->processLinksAndShortcodes(
-          $this->joinObject($rendered_newsletter),
-          $newsletter['id'],
-          $queue->id
-        );
-        list($newsletter['body']['html'], $newsletter['body']['text']) =
-          $this->splitObject($processed_newsletter);
-      }
-      else {
-        // render newsletter
-        $newsletter['body'] = $this->renderNewsletter($newsletter);
-      }
-      $this->extractAndSaveNewsletterPosts(
-        $newsletter['id'],
-        $newsletter['body']['html']
-      );
-      $queue->newsletter_rendered_body = json_encode($newsletter['body']);
-      $queue->save();
-    } else {
-      $newsletter['body'] = json_decode($queue->newsletter_rendered_body);
-    }
-    return (array) $newsletter['body'];
-  }
-
-  function processBulkSubscribers($mailer, $newsletter, $subscribers, $queue) {
-    foreach($subscribers as $subscriber) {
-      $processed_newsletters[] =
-        $this->processNewsletterBeforeSending($newsletter, $subscriber, $queue);
-      if(!$queue->newsletter_rendered_subject) {
-        $queue->newsletter_rendered_subject = $processed_newsletters[0]['subject'];
-      }
-      $transformed_subscribers[] =
-        $mailer->transformSubscriber($subscriber);
-    }
-    $result = $this->sendNewsletter(
-      $mailer,
-      $processed_newsletters,
-      $transformed_subscribers
-    );
-    $subscribers_ids = Helpers::arrayColumn($subscribers, 'id');
-    if(!$result) {
-      $queue->subscribers->failed = array_merge(
-        $queue->subscribers->failed,
-        $subscribers_ids
-      );
-    } else {
-      $newsletter_statistics =
-        array_map(function($data) use ($newsletter, $subscribers_ids, $queue) {
-          return array(
-            $newsletter['id'],
-            $subscribers_ids[$data],
-            $queue->id
-          );
-        }, range(0, count($transformed_subscribers) - 1));
-      $newsletter_statistics = Helpers::flattenArray($newsletter_statistics);
-      $this->updateMailerLog();
-      $this->updateNewsletterStatistics($newsletter_statistics);
-      $queue->subscribers->processed = array_merge(
-        $queue->subscribers->processed,
-        $subscribers_ids
-      );
-    }
-    $this->updateQueue($queue);
-    $this->checkSendingLimit();
-    CronHelper::checkExecutionTimer($this->timer);
-    return $queue->subscribers;
-  }
-
-  function processIndividualSubscriber($mailer, $newsletter, $subscribers, $queue) {
-    foreach($subscribers as $subscriber) {
-      $this->checkSendingLimit();
-      $processed_newsletter = $this->processNewsletterBeforeSending($newsletter, $subscriber, $queue);
-      if(!$queue->newsletter_rendered_subject) {
-        $queue->newsletter_rendered_subject = $processed_newsletter['subject'];
-      }
-      $transformed_subscriber = $mailer->transformSubscriber($subscriber);
-      $result = $this->sendNewsletter(
-        $mailer,
-        $processed_newsletter,
-        $transformed_subscriber
-      );
-      if(!$result) {
-        $queue->subscribers->failed[] = $subscriber['id'];
-      } else {
-        $queue->subscribers->processed[] = $subscriber['id'];
-        $newsletter_statistics = array(
-          $newsletter['id'],
-          $subscriber['id'],
-          $queue->id
-        );
-        $this->updateMailerLog();
-        $this->updateNewsletterStatistics($newsletter_statistics);
-      }
-      $this->updateQueue($queue);
-      CronHelper::checkExecutionTimer($this->timer);
-    }
-    return $queue->subscribers;
-  }
-
-  function updateNewsletterStatistics($data) {
-    return StatisticsNewsletters::createMultiple($data);
-  }
-
-  function renderNewsletter($newsletter) {
-    $renderer = new Renderer($newsletter);
-    return $renderer->render();
-  }
-
-  function processLinksAndShortcodes($content, $newsletter_id, $queue_id) {
-    // process only link shortcodes
-    $shortcodes = new Shortcodes($newsletter = false, $subscriber = false, $queue_id);
-    $content = $shortcodes->replace(
-      $content,
-      $categories = array('link')
-    );
-    // extract and save links and link shortcodes
-    list($content, $processed_links) =
-      Links::process(
-        $content,
-        $links = false,
-        $process_link_shortcodes = true,
-        $queue_id
-      );
-    Links::save($processed_links, $newsletter_id, $queue_id);
-    return $content;
-  }
-
-  function processNewsletterBeforeSending($newsletter, $subscriber = false, $queue) {
-    $data_for_shortcodes = array(
-      $newsletter['subject'],
-      $newsletter['body']['html'],
-      $newsletter['body']['text']
-    );
-    $processed_newsletter = $this->replaceShortcodes(
-      $newsletter,
-      $subscriber,
-      $queue,
-      $this->joinObject($data_for_shortcodes)
-    );
-    if((boolean) Setting::getValue('tracking.enabled')) {
-      $processed_newsletter = Links::replaceSubscriberData(
-        $newsletter['id'],
-        $subscriber['id'],
-        $queue->id,
-        $processed_newsletter
-      );
-    }
-    list($newsletter['subject'],
-      $newsletter['body']['html'],
-      $newsletter['body']['text']
-      ) = $this->splitObject($processed_newsletter);
-    return $newsletter;
-  }
-
-  function replaceShortcodes($newsletter, $subscriber, $queue, $body) {
-    $shortcodes = new Shortcodes(
-      $newsletter,
-      $subscriber,
-      $queue
-    );
-    return $shortcodes->replace($body);
-  }
-
-  function sendNewsletter($mailer, $newsletter, $subscriber) {
-    return $mailer->mailer_instance->send(
-      $newsletter,
-      $subscriber
-    );
-  }
-
-  function configureMailer($newsletter) {
-    $sender['address'] = (!empty($newsletter['sender_address'])) ?
-      $newsletter['sender_address'] :
-      false;
-    $sender['name'] = (!empty($newsletter['sender_name'])) ?
-      $newsletter['sender_name'] :
-      false;
-    $reply_to['address'] = (!empty($newsletter['reply_to_address'])) ?
-      $newsletter['reply_to_address'] :
-      false;
-    $reply_to['name'] = (!empty($newsletter['reply_to_name'])) ?
-      $newsletter['reply_to_name'] :
-      false;
-    if(!$sender['address']) {
-      $sender = false;
-    }
-    if(!$reply_to['address']) {
-      $reply_to = false;
-    }
-    $mailer = new Mailer($method = false, $sender, $reply_to);
-    return $mailer;
-  }
-
-  function getQueues() {
-    return SendingQueueModel::orderByDesc('priority')
-      ->whereNull('deleted_at')
-      ->whereNull('status')
-      ->findResultSet();
-  }
-
-  function updateQueue($queue) {
-    $queue = clone($queue);
-    $queue->subscribers->to_process = array_diff(
-      $queue->subscribers->to_process,
-      array_merge(
-        $queue->subscribers->processed,
-        $queue->subscribers->failed
-      )
-    );
-    $queue->subscribers->to_process = array_values(
-      $queue->subscribers->to_process
-    );
-    $queue->count_processed =
-      count($queue->subscribers->processed) + count($queue->subscribers->failed);
-    $queue->count_to_process = count($queue->subscribers->to_process);
-    $queue->count_failed = count($queue->subscribers->failed);
-    $queue->count_total =
-      $queue->count_processed + $queue->count_to_process;
-    if(!$queue->count_to_process) {
-      $queue->processed_at = current_time('mysql');
-      $queue->status = SendingQueueModel::STATUS_COMPLETED;
-
-      // set newsletter status to sent
-      $newsletter = Newsletter::findOne($queue->newsletter_id);
-      // if it's a standard newsletter, update its status
-      if($newsletter->type === Newsletter::TYPE_STANDARD) {
-        $newsletter->setStatus(Newsletter::STATUS_SENT);
-      } else if($newsletter->type === Newsletter::TYPE_NOTIFICATION) {
-        // TODO: Check with Vlad
-      }
-    }
-    $queue->subscribers = serialize((array) $queue->subscribers);
-    $queue->save();
-  }
-
-  function updateMailerLog() {
-    $this->mta_log['sent']++;
-    return Setting::setValue('mta_log', $this->mta_log);
-  }
-
-  function getMailerConfig() {
-    $mta_config = Setting::getValue('mta');
-    if(!$mta_config) {
-      throw new \Exception(__('Mailer is not configured.'));
-    }
-    return $mta_config;
-  }
-
-  function getMailerLog() {
-    $mta_log = Setting::getValue('mta_log');
-    if(!$mta_log) {
-      $mta_log = array(
-        'sent' => 0,
-        'started' => time()
-      );
-      Setting::setValue('mta_log', $mta_log);
-    }
-    return $mta_log;
-  }
-
-  function checkSendingLimit() {
-    $frequency_interval = (int)$this->mta_config['frequency']['interval'] * 60;
-    $frequency_limit = (int)$this->mta_config['frequency']['emails'];
-    $elapsed_time = time() - (int)$this->mta_log['started'];
-    if($this->mta_log['sent'] === $frequency_limit &&
-      $elapsed_time <= $frequency_interval
-    ) {
-      throw new \Exception(__('Sending frequency limit has been reached.'));
-    }
-    if($elapsed_time > $frequency_interval) {
-      $this->mta_log = array(
-        'sent' => 0,
-        'started' => time()
-      );
-      Setting::setValue('mta_log', $this->mta_log);
-    }
-    return;
-  }
-
-  function recalculateSubscriberCount(
-    $found_subscriber, $existing_subscribers, $subscribers_to_process) {
-    $subscibers_to_exclude = array_diff($existing_subscribers, $found_subscriber);
-    return array_diff($subscribers_to_process, $subscibers_to_exclude);
-  }
-
-  function extractAndSaveNewsletterPosts($newletter_id, $content) {
-    preg_match_all('/data-post-id="(\d+)"/ism', $content, $posts);
-    $posts = $posts[1];
-    foreach($posts as $post) {
-      $newletter_post = NewsletterPost::create();
-      $newletter_post->newsletter_id = $newletter_id;
-      $newletter_post->post_id = $post;
-      $newletter_post->save();
-    }
-  }
-
-  private function joinObject($object = array()) {
-    return implode(self::DIVIDER, $object);
-  }
-
-  private function splitObject($object = array()) {
-    return explode(self::DIVIDER, $object);
-  }
-}
\ No newline at end of file
diff --git a/lib/Cron/Workers/SendingQueue/SendingQueue.php b/lib/Cron/Workers/SendingQueue/SendingQueue.php
new file mode 100644
index 0000000000..c4a63f08b4
--- /dev/null
+++ b/lib/Cron/Workers/SendingQueue/SendingQueue.php
@@ -0,0 +1,181 @@
+<?php
+namespace MailPoet\Cron\Workers\SendingQueue;
+
+use MailPoet\Cron\CronHelper;
+use MailPoet\Cron\Workers\SendingQueue\Tasks\Mailer as MailerTask;
+use MailPoet\Cron\Workers\SendingQueue\Tasks\Newsletter as NewsletterTask;
+use MailPoet\Cron\Workers\SendingQueue\Tasks\Statistics as StatisticsTask;
+use MailPoet\Cron\Workers\SendingQueue\Tasks\Subscribers as SubscribersTask;
+use MailPoet\Models\SendingQueue as SendingQueueModel;
+use MailPoet\Models\Setting;
+use MailPoet\Models\Subscriber;
+use MailPoet\Util\Helpers;
+
+if(!defined('ABSPATH')) exit;
+
+class SendingQueue {
+  public $mta_config;
+  public $mta_log;
+  private $timer;
+  const BATCH_SIZE = 50;
+  const STATUS_COMPLETED = 'completed';
+
+  function __construct($timer = false) {
+    $this->mta_config = MailerTask::getMailerConfig();
+    $this->mta_log = MailerTask::getMailerLog();
+    $this->timer = ($timer) ? $timer : microtime(true);
+    CronHelper::checkExecutionTimer($this->timer);
+  }
+
+  function process() {
+    foreach($this->getQueues() as $queue) {
+      $newsletter = NewsletterTask::getAndPreProcess($queue->asArray());
+      if(!$newsletter) {
+        $queue->delete();
+        continue;
+      }
+      if(is_null($queue->newsletter_rendered_body)) {
+        $queue->newsletter_rendered_body = json_encode($newsletter['rendered_body']);
+        $queue->save();
+      }
+      $queue->subscribers = SubscribersTask::get($queue->asArray());
+      // configure mailer with newsletter data (from/reply-to)
+      $mailer = MailerTask::configureMailer($newsletter);
+      $processing_method = MailerTask::getProcessingMethod($this->mta_config);
+      foreach(array_chunk($queue->subscribers['to_process'], self::BATCH_SIZE) as
+              $subscribers_to_process_ids) {
+        $subscribers = Subscriber::whereIn('id', $subscribers_to_process_ids)
+            ->findArray();
+        // if some subscribers weren't found, remove them from the processing list
+        if(count($subscribers) !== count($subscribers_to_process_ids)) {
+          $queue->subscribers['to_process'] = Subscribers::updateCount(
+            Helpers::arrayColumn($subscribers, 'id'),
+            $subscribers_to_process_ids,
+            $queue->subscribers['to_process']
+          );
+        }
+        if(!count($queue->subscribers['to_process'])) {
+          $this->updateQueue($queue);
+          continue;
+        }
+        $queue->subscribers = call_user_func_array(
+          array($this, $processing_method),
+          array($mailer, $newsletter, $subscribers, $queue)
+        );
+      }
+    }
+  }
+
+  function processBulkSubscribers($mailer, $newsletter, $subscribers, $queue) {
+    $subscriber_log = array();
+    $subscribers_ids = Helpers::arrayColumn($subscribers, 'id');
+    foreach($subscribers as $subscriber) {
+      // render shortcodes and replace subscriber data in tracked links
+      $prepared_newsletters[] =
+        NewsletterTask::prepareNewsletterForSending(
+          $newsletter,
+          $subscriber,
+          $queue->asArray()
+        );
+      if(!$queue->newsletter_rendered_subject) {
+        $queue->newsletter_rendered_subject = $prepared_newsletters[0]['subject'];
+      }
+      // format subscriber name/address according to mailer settings
+      $prepared_subscribers[] = MailerTask::prepareSubscriberForSending(
+        $mailer,
+        $subscriber
+      );
+    }
+    // send
+    $result = MailerTask::send($mailer, $prepared_newsletters, $prepared_subscribers);
+    if(!$result) {
+      // record failed subscribers
+      $subscriber_log['failed'] = SubscribersTask::updateFailedList(
+        $queue->subscribers['failed'],
+        $subscribers_ids
+      );
+    } else {
+      StatisticsTask::updateBulkNewsletterStatistics(
+        $subscribers_ids,
+        $newsletter['id'],
+        $queue->id
+      );
+      MailerTask::updateMailerLog($this->mta_log);
+      $subscriber_log['processed'] = array_merge(
+        $queue->subscribers['processed'],
+        $subscribers_ids
+      );
+    }
+    // TODO
+    //$queue = $this->updateQueue($queue, $subscriber_log);
+    MailerTask::checkSendingLimit($this->mta_config, $this->mta_log);
+    CronHelper::checkExecutionTimer($this->timer);
+    return $queue->subscribers;
+  }
+
+ /* function processIndividualSubscriber($mailer, $newsletter, $subscribers, $queue) {
+    foreach($subscribers as $subscriber) {
+      $this->checkSendingLimit();
+      $processed_newsletter = $this->prepareNewsletterForSending($newsletter, $subscriber, $queue);
+      if(!$queue->newsletter_rendered_subject) {
+        $queue->newsletter_rendered_subject = $processed_newsletter['subject'];
+      }
+      $transformed_subscriber = $mailer->transformSubscriber($subscriber);
+      $result = $this->sendNewsletter(
+        $mailer,
+        $processed_newsletter,
+        $transformed_subscriber
+      );
+      if(!$result) {
+        $queue->subscribers['failed'][] = $subscriber['id'];
+      } else {
+        $queue->subscribers['processed'][] = $subscriber['id'];
+        $newsletter_statistics = array(
+          $newsletter['id'],
+          $subscriber['id'],
+          $queue->id
+        );
+        $this->updateMailerLog();
+        $this->updateNewsletterStatistics($newsletter_statistics);
+      }
+      $this->updateQueue($queue);
+      CronHelper::checkExecutionTimer($this->timer);
+    }
+    return $queue->subscribers;
+  }*/
+
+  function getQueues() {
+    return SendingQueueModel::orderByDesc('priority')
+      ->whereNull('deleted_at')
+      ->whereNull('status')
+      ->findResultSet();
+  }
+
+  function updateQueue($queue) {
+    // TODO
+    return;
+    $queue = clone($queue);
+    $queue->subscribers['to_process'] = array_diff(
+      $queue->subscribers['to_process'],
+      array_merge(
+        $queue->subscribers['processed'],
+        $queue->subscribers['failed']
+      )
+    );
+    $queue->subscribers['to_process'] = array_values(
+      $queue->subscribers['to_process']
+    );
+    $queue->count_processed =
+      count($queue->subscribers['processed']) + count($queue->subscribers['failed']);
+    $queue->count_to_process = count($queue->subscribers['to_process']);
+    $queue->count_failed = count($queue->subscribers['failed']);
+    $queue->count_total =
+      $queue->count_processed + $queue->count_to_process;
+    if(!$queue->count_to_process) {
+      $queue->processed_at = current_time('mysql');
+      $queue->status = self::STATUS_COMPLETED;
+    }
+    $queue->subscribers = serialize((array) $queue->subscribers);
+    $queue->save();
+  }
+}
\ No newline at end of file
diff --git a/lib/Cron/Workers/SendingQueue/Tasks/Links.php b/lib/Cron/Workers/SendingQueue/Tasks/Links.php
new file mode 100644
index 0000000000..8da6229264
--- /dev/null
+++ b/lib/Cron/Workers/SendingQueue/Tasks/Links.php
@@ -0,0 +1,33 @@
+<?php
+namespace MailPoet\Cron\Workers\SendingQueue\Tasks;
+
+use MailPoet\Newsletter\Links\Links as NewsletterLinks;
+use MailPoet\Util\Helpers;
+
+if(!defined('ABSPATH')) exit;
+
+class Links {
+  static function process(array $newsletter, array $queue) {
+    list($newsletter, $links) = self::hashAndReplaceLinks($newsletter, $queue);
+    self::saveLinks($links, $newsletter, $queue);
+    return $newsletter;
+  }
+
+  static function hashAndReplaceLinks(array $newsletter, array $queue) {
+    // join HTML and TEXT rendered body into a text string
+    $content = Helpers::joinObject($newsletter['rendered_body']);
+    list($content, $links) = NewsletterLinks::process($content);
+    // split the processed body with hashed links back to HTML and TEXT
+    list($newsletter['rendered_body']['html'], $newsletter['rendered_body']['text'])
+      = Helpers::splitObject($content);
+    return array(
+      $newsletter,
+      $links
+    );
+  }
+
+  static function saveLinks($links, $newsletter, $queue) {
+    return NewsletterLinks::save($links, $newsletter['id'], $queue['id']);
+  }
+}
+
diff --git a/lib/Cron/Workers/SendingQueue/Tasks/Mailer.php b/lib/Cron/Workers/SendingQueue/Tasks/Mailer.php
new file mode 100644
index 0000000000..9262167fca
--- /dev/null
+++ b/lib/Cron/Workers/SendingQueue/Tasks/Mailer.php
@@ -0,0 +1,95 @@
+<?php
+namespace MailPoet\Cron\Workers\SendingQueue\Tasks;
+
+use MailPoet\Mailer\Mailer as MailerFactory;
+use MailPoet\Models\Setting;
+
+if(!defined('ABSPATH')) exit;
+
+class Mailer {
+  static function configureMailer(array $newsletter) {
+    $sender['address'] = (!empty($newsletter['sender_address'])) ?
+      $newsletter['sender_address'] :
+      false;
+    $sender['name'] = (!empty($newsletter['sender_name'])) ?
+      $newsletter['sender_name'] :
+      false;
+    $reply_to['address'] = (!empty($newsletter['reply_to_address'])) ?
+      $newsletter['reply_to_address'] :
+      false;
+    $reply_to['name'] = (!empty($newsletter['reply_to_name'])) ?
+      $newsletter['reply_to_name'] :
+      false;
+    if(!$sender['address']) {
+      $sender = false;
+    }
+    if(!$reply_to['address']) {
+      $reply_to = false;
+    }
+    $mailer = new MailerFactory($method = false, $sender, $reply_to);
+    return $mailer;
+  }
+
+  static function getMailerConfig() {
+    $mta_config = Setting::getValue('mta');
+    if(!$mta_config) {
+      throw new \Exception(__('Mailer is not configured.'));
+    }
+    return $mta_config;
+  }
+
+  static function updateMailerLog($mta_log) {
+    $mta_log['sent']++;
+    Setting::setValue('mta_log', $mta_log);
+    return $mta_log;
+  }
+
+  static function getMailerLog() {
+    $mta_log = Setting::getValue('mta_log');
+    if(!$mta_log) {
+      $mta_log = array(
+        'sent' => 0,
+        'started' => time()
+      );
+      Setting::setValue('mta_log', $mta_log);
+    }
+    return $mta_log;
+  }
+
+  static function getProcessingMethod($mta_config) {
+    return ($mta_config['method'] === 'MailPoet') ?
+      'processBulkSubscribers' :
+      'processIndividualSubscriber';
+  }
+
+  static function prepareSubscriberForSending($mailer, $subscriber) {
+    return ($mailer instanceof \MailPoet\Mailer\Mailer) ?
+      $mailer->transformSubscriber($subscriber) :
+      false;
+  }
+
+  static function send($mailer, $newsletter, $subscriber) {
+    return ($mailer instanceof \MailPoet\Mailer\Mailer) ?
+      $mailer->mailer_instance->send($newsletter, $subscriber) :
+      false;
+  }
+
+  static function checkSendingLimit($mta_config, $mta_log) {
+    $frequency_interval = (int) $mta_config['frequency']['interval'] * 60;
+    $frequency_limit = (int) $mta_config['frequency']['emails'];
+    $elapsed_time = time() - (int) $mta_log['started'];
+    if($mta_log['sent'] === $frequency_limit &&
+      $elapsed_time <= $frequency_interval
+    ) {
+      throw new \Exception(__('Sending frequency limit has been reached.'));
+    }
+    if($elapsed_time > $frequency_interval) {
+      $mta_log = array(
+        'sent' => 0,
+        'started' => time()
+      );
+      Setting::setValue('mta_log', $mta_log);
+    }
+    return;
+  }
+}
\ No newline at end of file
diff --git a/lib/Cron/Workers/SendingQueue/Tasks/Newsletter.php b/lib/Cron/Workers/SendingQueue/Tasks/Newsletter.php
new file mode 100644
index 0000000000..92e827bad3
--- /dev/null
+++ b/lib/Cron/Workers/SendingQueue/Tasks/Newsletter.php
@@ -0,0 +1,96 @@
+<?php
+namespace MailPoet\Cron\Workers\SendingQueue\Tasks;
+
+use MailPoet\Cron\Workers\SendingQueue\Tasks\Links as LinksTask;
+use MailPoet\Cron\Workers\SendingQueue\Tasks\Posts as PostsTask;
+use MailPoet\Cron\Workers\SendingQueue\Tasks\Shortcodes as ShortcodesTask;
+use MailPoet\Models\Newsletter as NewsletterModel;
+use MailPoet\Models\Setting;
+use MailPoet\Newsletter\Links\Links as NewsletterLinks;
+use MailPoet\Newsletter\Renderer\PostProcess\OpenTracking;
+use MailPoet\Newsletter\Renderer\Renderer;
+use MailPoet\Util\Helpers;
+
+if(!defined('ABSPATH')) exit;
+
+class Newsletter {
+  static function get($newsletter_id) {
+    $newsletter = NewsletterModel::findOne($newsletter_id);
+    return ($newsletter) ? $newsletter->asArray() : false;
+  }
+
+  static function getAndPreProcess(array $queue) {
+    $newsletter = self::get($queue['newsletter_id']);
+    if(!$newsletter) {
+      return false;
+    }
+    // if the newsletter was previously rendered, return it
+    // otherwise, process/render it
+    if(!is_null($queue['newsletter_rendered_body'])) {
+      $newsletter['rendered_body'] = json_decode($queue['newsletter_rendered_body'], true);
+      return $newsletter;
+    }
+    // if tracking is enabled, do additional processing
+    if((boolean) Setting::getValue('tracking.enabled')) {
+      // add tracking image
+      OpenTracking::addTrackingImage();
+      // render newsletter
+      $newsletter = self::render($newsletter);
+      // hash and save all links
+      $newsletter = LinksTask::process($newsletter, $queue);
+    } else {
+      // render newsletter
+      $newsletter = self::render($newsletter);
+    }
+    // check if this is a post notification and if it contains posts
+    $newsletter_contains_posts = strpos($newsletter['rendered_body']['html'], 'data-post-id');
+    if($newsletter['type'] === 'notification' && !$newsletter_contains_posts) {
+      return false;
+    }
+    // save all posts
+    $newsletter = PostsTask::extractAndSave($newsletter);
+    return $newsletter;
+  }
+
+  static function render($newsletter) {
+    $renderer = new Renderer($newsletter);
+    $newsletter['rendered_body'] = $renderer->render();
+    return $newsletter;
+  }
+
+  static function prepareNewsletterForSending(
+    array $newsletter, array $subscriber, array $queue
+  ) {
+    // shortcodes and links will be replaced in the subject, html and text body
+    // to speed the processing, join content into a continuous string
+    $prepared_newsletter = Helpers::joinObject(
+      array(
+        $newsletter['subject'],
+        $newsletter['rendered_body']['html'],
+        $newsletter['rendered_body']['text']
+      )
+    );
+    $prepared_newsletter = ShortcodesTask::process(
+      $prepared_newsletter,
+      $newsletter,
+      $subscriber,
+      $queue
+    );
+    if((boolean) Setting::getValue('tracking.enabled')) {
+      $prepared_newsletter = NewsletterLinks::replaceSubscriberData(
+        $newsletter['id'],
+        $subscriber['id'],
+        $queue['id'],
+        $prepared_newsletter
+      );
+    }
+    $prepared_newsletter = Helpers::splitObject($prepared_newsletter);
+    return array(
+      'subject' => $prepared_newsletter[0],
+      'body' => array(
+        'html' => $prepared_newsletter[1],
+        'text' => $prepared_newsletter[2]
+      )
+    );
+  }
+}
\ No newline at end of file
diff --git a/lib/Cron/Workers/SendingQueue/Tasks/Posts.php b/lib/Cron/Workers/SendingQueue/Tasks/Posts.php
new file mode 100644
index 0000000000..c5bd23cee6
--- /dev/null
+++ b/lib/Cron/Workers/SendingQueue/Tasks/Posts.php
@@ -0,0 +1,29 @@
+<?php
+namespace MailPoet\Cron\Workers\SendingQueue\Tasks;
+
+use MailPoet\Models\NewsletterPost;
+
+if(!defined('ABSPATH')) exit;
+
+class Posts {
+  static function extractAndSave(array $newsletter) {
+    if(empty($newsletter['rendered_body']['html']) || empty($newsletter['id'])) {
+      return;
+    }
+    preg_match_all(
+      '/data-post-id="(\d+)"/ism',
+      $newsletter['rendered_body']['html'],
+      $matached_posts);
+    $matached_posts = $matached_posts[1];
+    if(!count($matached_posts)) {
+      return $newsletter;
+    }
+    foreach($matached_posts as $post) {
+      $newletter_post = NewsletterPost::create();
+      $newletter_post->newsletter_id = $newsletter['id'];
+      $newletter_post->post_id = $post;
+      $newletter_post->save();
+    }
+    return $newsletter;
+  }
+}
\ No newline at end of file
diff --git a/lib/Cron/Workers/SendingQueue/Tasks/Shortcodes.php b/lib/Cron/Workers/SendingQueue/Tasks/Shortcodes.php
new file mode 100644
index 0000000000..7871a3c114
--- /dev/null
+++ b/lib/Cron/Workers/SendingQueue/Tasks/Shortcodes.php
@@ -0,0 +1,14 @@
+<?php
+namespace MailPoet\Cron\Workers\SendingQueue\Tasks;
+
+use MailPoet\Newsletter\Shortcodes\Shortcodes as NewsletterShortcodes;
+
+if(!defined('ABSPATH')) exit;
+
+class Shortcodes {
+  static function process($content, array $newsletter, array $subscriber, array $queue) {
+    $shortcodes = new NewsletterShortcodes($newsletter, $subscriber, $queue);
+    return $shortcodes->replace($content);
+  }
+}
+
diff --git a/lib/Cron/Workers/SendingQueue/Tasks/Statistics.php b/lib/Cron/Workers/SendingQueue/Tasks/Statistics.php
new file mode 100644
index 0000000000..ceb2f368ee
--- /dev/null
+++ b/lib/Cron/Workers/SendingQueue/Tasks/Statistics.php
@@ -0,0 +1,30 @@
+<?php
+namespace MailPoet\Cron\Workers\SendingQueue\Tasks;
+
+use MailPoet\Models\StatisticsNewsletters;
+use MailPoet\Util\Helpers;
+
+if(!defined('ABSPATH')) exit;
+
+class Statistics {
+  static function updateBulkNewsletterStatistics(
+    array $processed_subscribers_ids, $newsletter_id, $queue_id
+  ) {
+    $newsletter_statistics = array();
+    foreach($processed_subscribers_ids as $subscriber_id) {
+      $newsletter_statistics[] = array(
+        $newsletter_id,
+        $subscriber_id,
+        $queue_id
+      );
+    }
+    $newsletter_statistics = Helpers::flattenArray($newsletter_statistics);
+    return StatisticsNewsletters::createMultiple($newsletter_statistics);
+  }
+
+  static function updateBulkNewsletterStatistics(
+    array $processed_subscribers_ids, $newsletter_id, $queue_id
+  ) {
+
+  }
+}
\ No newline at end of file
diff --git a/lib/Cron/Workers/SendingQueue/Tasks/Subscribers.php b/lib/Cron/Workers/SendingQueue/Tasks/Subscribers.php
new file mode 100644
index 0000000000..0d58694d2f
--- /dev/null
+++ b/lib/Cron/Workers/SendingQueue/Tasks/Subscribers.php
@@ -0,0 +1,41 @@
+<?php
+namespace MailPoet\Cron\Workers\SendingQueue\Tasks;
+
+if(!defined('ABSPATH')) exit;
+
+class Subscribers {
+  static function get(array $queue) {
+    $subscribers = unserialize($queue['subscribers']);
+    if(empty($subscribers['processed'])) {
+      $subscribers['processed'] = array();
+    }
+    if(empty($subscribers['failed'])) {
+      $subscribers['failed'] = array();
+    }
+    return $subscribers;
+  }
+
+  static function updateToProcessList(
+    array $existing_subscribers_ids,
+    array $subscribers_to_process_ids,
+    array $queue_subscribers_to_process_ids
+  ) {
+    // compare existing subscriber to the ones that queued for processing
+    $subscibers_to_exclude = array_diff(
+      $subscribers_to_process_ids,
+      $existing_subscribers_ids
+    );
+    // remove nonexistent subscribers from the processing list
+    return array_diff(
+      $queue_subscribers_to_process_ids,
+      $subscibers_to_exclude
+    );
+  }
+
+  static function updateFailedList(array $subscribers, array $failed_subscribers) {
+    return array_merge(
+      $subscribers,
+      $failed_subscribers
+    );
+  }
+}
\ No newline at end of file
diff --git a/lib/Models/StatisticsNewsletters.php b/lib/Models/StatisticsNewsletters.php
index 3f6d5e39d5..026ed4e1e9 100644
--- a/lib/Models/StatisticsNewsletters.php
+++ b/lib/Models/StatisticsNewsletters.php
@@ -11,6 +11,9 @@ class StatisticsNewsletters extends Model {
   }
 
   static function createMultiple($data) {
+    if (count($data) % 3 !== 0) {
+      return false;
+    }
     return self::rawExecute(
       'INSERT INTO `' . self::$_table . '` ' .
       '(newsletter_id, subscriber_id, queue_id) ' .
diff --git a/lib/Newsletter/Renderer/PostProcess/OpenTracking.php b/lib/Newsletter/Renderer/PostProcess/OpenTracking.php
index 1884e044af..a5f256aa2f 100644
--- a/lib/Newsletter/Renderer/PostProcess/OpenTracking.php
+++ b/lib/Newsletter/Renderer/PostProcess/OpenTracking.php
@@ -8,12 +8,18 @@ class OpenTracking {
     $DOM = new \pQuery();
     $DOM = $DOM->parseStr($template);
     $template = $DOM->query('body');
-    $open_tracking_link = sprintf(
+    $open_tracking_image = sprintf(
       '<img alt="" class="" src="%s/%s"/>',
       home_url(),
       esc_attr('?mailpoet&endpoint=track&action=open&data=' . Links::DATA_TAG)
     );
-    $template->html($template->html() . $open_tracking_link);
+    $template->html($template->html() . $open_tracking_image);
     return $DOM->__toString();
   }
+
+  static function addTrackingImage() {
+    add_filter('mailpoet_rendering_post_process', function ($template) {
+      return OpenTracking::process($template);
+    });
+  }
 }
\ No newline at end of file
diff --git a/lib/Newsletter/Shortcodes/Shortcodes.php b/lib/Newsletter/Shortcodes/Shortcodes.php
index 5e551365f0..06d24288b8 100644
--- a/lib/Newsletter/Shortcodes/Shortcodes.php
+++ b/lib/Newsletter/Shortcodes/Shortcodes.php
@@ -5,6 +5,7 @@ class Shortcodes {
   public $newsletter;
   public $subscriber;
   public $queue;
+  const SHORTCODE_CATEGORY_NAMESPACE = __NAMESPACE__ . '\\Categories\\';
 
   function __construct(
     $newsletter = false,
@@ -22,14 +23,17 @@ class Shortcodes {
       $queue;
   }
 
-  function extract($content, $categories= false) {
+  function extract($content, $categories = false) {
     $categories = (is_array($categories)) ? implode('|', $categories) : false;
     $regex = sprintf(
       '/\[%s:.*?\]/ism',
       ($categories) ? '(?:' . $categories . ')' : '(?:\w+)'
     );
     preg_match_all($regex, $content, $shortcodes);
-    return array_unique($shortcodes[0]);
+    $shortcodes = $shortcodes[0];
+    return (count($shortcodes)) ?
+      array_unique($shortcodes) :
+      false;
   }
 
   function match($shortcode) {
@@ -43,18 +47,19 @@ class Shortcodes {
 
   function process($shortcodes, $content = false) {
     $processed_shortcodes = array_map(
-      function($shortcode) use($content) {
+      function($shortcode) use ($content) {
         $shortcode_details = $this->match($shortcode);
-        $shortcode_category = isset($shortcode_details['category']) ?
+        $shortcode_category = !empty($shortcode_details['category']) ?
           ucfirst($shortcode_details['category']) :
           false;
-        $shortcode_action = isset($shortcode_details['action']) ?
+        $shortcode_action = !empty($shortcode_details['action']) ?
           $shortcode_details['action'] :
           false;
         $shortcode_class =
-          __NAMESPACE__ . '\\Categories\\' . $shortcode_category;
-        $shortcode_default_value = isset($shortcode_details['default'])
-          ? $shortcode_details['default'] : false;
+          self::SHORTCODE_CATEGORY_NAMESPACE . $shortcode_category;
+        $shortcode_default_value = !empty($shortcode_details['default']) ?
+          $shortcode_details['default'] :
+          false;
         if(!class_exists($shortcode_class)) {
           $custom_shortcode = apply_filters(
             'mailpoet_newsletter_shortcode',
@@ -82,6 +87,9 @@ class Shortcodes {
 
   function replace($content, $categories = false) {
     $shortcodes = $this->extract($content, $categories);
+    if(!$shortcodes) {
+      return $content;
+    }
     $processed_shortcodes = $this->process($shortcodes, $content);
     $shortcodes = array_intersect_key($shortcodes, $processed_shortcodes);
     return str_replace($shortcodes, $processed_shortcodes, $content);
diff --git a/lib/Util/Helpers.php b/lib/Util/Helpers.php
index 1d68e54767..a9db03e29b 100644
--- a/lib/Util/Helpers.php
+++ b/lib/Util/Helpers.php
@@ -2,6 +2,8 @@
 namespace MailPoet\Util;
 
 class Helpers {
+  const DIVIDER = '***MailPoet***';
+
   static function getMaxPostSize($bytes = false) {
     $maxPostSize = ini_get('post_max_size');
     if(!$bytes) return $maxPostSize;
@@ -122,4 +124,12 @@ class Helpers {
       )
     );
   }
+
+  static function joinObject($object = array()) {
+    return implode(self::DIVIDER, $object);
+  }
+
+  static function splitObject($object = array()) {
+    return explode(self::DIVIDER, $object);
+  }
 }
\ No newline at end of file