- Saves sent posts during rendering by sending queue worker

- Prevents empty notification emails from being sent
- Hooks to WP's post update and rewrite post notification logic
- Prevents scheduling multiple queues of the same newsletter
- Fixes issue with segments not updating when scheduling a newsletter
- Removes depreciated hash field & associated logic

lib/Config/Hooks.php

@@ -13,6 +13,7 @@ class Hooks {
     $this->setupImageSize();
     $this->setupListing();
     $this->setupSubscriptionEvents();
+    $this->setupPostNotifications();
   }
 
   function setupSubscriptionEvents() {
@@ -146,4 +147,12 @@ class Hooks {
       return $status;
     }
   }
+
+  function setupPostNotifications() {
+    add_filter(
+      'publish_post',
+      '\MailPoet\Newsletter\Scheduler\Scheduler::schedulePostNotification',
+      10, 1
+    );
+  }
 }

lib/Config/Migrator.php

@@ -102,7 +102,6 @@ class Migrator {
       'id mediumint(9) NOT NULL AUTO_INCREMENT,',
       'newsletter_id mediumint(9) NOT NULL,',
       'newsletter_rendered_body longtext,',
-      'newsletter_rendered_body_hash varchar(250) NULL DEFAULT NULL,',
       'newsletter_rendered_subject varchar(250) NULL DEFAULT NULL,',
       'subscribers longtext,',
       'status varchar(12) NULL DEFAULT NULL,',

lib/Cron/Workers/Scheduler.php

@@ -2,15 +2,11 @@
 namespace MailPoet\Cron\Workers;
 
 use Carbon\Carbon;
-use Cron\CronExpression as Cron;
 use MailPoet\Cron\CronHelper;
 use MailPoet\Models\Newsletter;
 use MailPoet\Models\SendingQueue;
-use MailPoet\Models\Setting;
 use MailPoet\Models\Subscriber;
 use MailPoet\Models\SubscriberSegment;
-use MailPoet\Newsletter\Renderer\PostProcess\OpenTracking;
-use MailPoet\Newsletter\Renderer\Renderer;
 use MailPoet\Util\Helpers;
 
 require_once(ABSPATH . 'wp-includes/pluggable.php');
@@ -19,6 +15,7 @@ if(!defined('ABSPATH')) exit;
 
 class Scheduler {
   public $timer;
+  const UNCONFIRMED_SUBSCRIBER_RESCHEDULE_TIMEOUT = 5;
 
   function __construct($timer = false) {
     $this->timer = ($timer) ? $timer : microtime(true);
@@ -30,17 +27,19 @@ class Scheduler {
       ->whereLte('scheduled_at', Carbon::createFromTimestamp(current_time('timestamp')))
       ->findMany();
     if(!count($scheduled_queues)) return;
-   foreach($scheduled_queues as $i=>$queue) {
+    foreach($scheduled_queues as $i => $queue) {
       $newsletter = Newsletter::filter('filterWithOptions')
         ->findOne($queue->newsletter_id);
       if(!$newsletter || $newsletter->deleted_at !== null) {
         $queue->delete();
-      }
+      } else {
-      else if($newsletter->type === 'welcome') {
+        if($newsletter->type === 'welcome') {
-        $this->processWelcomeNewsletter($newsletter, $queue);
+          $this->processWelcomeNewsletter($newsletter, $queue);
-      }
+        } else {
-      else if($newsletter->type === 'notification') {
+          if($newsletter->type === 'notification') {
-        $this->processPostNotificationNewsletter($newsletter, $queue);
+            $this->processPostNotificationNewsletter($newsletter, $queue);
+          }
+        }
       }
       CronHelper::checkExecutionTimer($this->timer);
     }
@@ -50,14 +49,15 @@ class Scheduler {
     $subscriber = unserialize($queue->subscribers);
     $subscriber_id = $subscriber['to_process'][0];
     if($newsletter->event === 'segment') {
-      if ($this->verifyMailPoetSubscriber($subscriber_id, $newsletter, $queue) === false) {
+      if($this->verifyMailPoetSubscriber($subscriber_id, $newsletter, $queue) === false) {
         return;
       }
-    }
+    } else {
-    else if($newsletter->event === 'user') {
+      if($newsletter->event === 'user') {
-      if ($this->verifyWPSubscriber($subscriber_id, $newsletter) === false) {
+        if($this->verifyWPSubscriber($subscriber_id, $newsletter) === false) {
-        $queue->delete();
+          $queue->delete();
-        return;
+          return;
+        }
       }
     }
     $queue->status = null;
@@ -65,18 +65,19 @@ class Scheduler {
   }
 
   function processPostNotificationNewsletter($newsletter, $queue) {
-    $next_run_date = $this->getQueueNextRunDate($newsletter->schedule);
     $segments = unserialize($newsletter->segments);
+    if(!count($segments)) {
+      $queue->delete();
+      return;
+    }
     $subscribers = Subscriber::getSubscribedInSegments($segments)
       ->findArray();
     $subscribers = Helpers::arrayColumn($subscribers, 'subscriber_id');
     $subscribers = array_unique($subscribers);
-    if(!count($subscribers) || !$this->checkIfNewsletterChanged($newsletter)) {
+    if(!count($subscribers)) {
-      $queue->scheduled_at = $next_run_date;
+      $queue->delete();
-      $queue->save();
       return;
     }
-    // update current queue
     $queue->subscribers = serialize(
       array(
         'to_process' => $subscribers
@@ -85,38 +86,35 @@ class Scheduler {
     $queue->count_total = $queue->count_to_process = count($subscribers);
     $queue->status = null;
     $queue->save();
-    // schedule newsletter for next delivery
-    $new_queue = SendingQueue::create();
-    $new_queue->newsletter_id = $newsletter->id;
-    $new_queue->scheduled_at = $next_run_date;
-    $new_queue->status = 'scheduled';
-    $new_queue->save();
   }
 
-  private function verifyMailPoetSubscriber($subscriber_id, $newsletter, $queue) {
+  function verifyMailPoetSubscriber($subscriber_id, $newsletter, $queue) {
     // check if subscriber is in proper segment
     $subscriber_in_segment =
       SubscriberSegment::where('subscriber_id', $subscriber_id)
         ->where('segment_id', $newsletter->segment)
         ->where('status', 'subscribed')
         ->findOne();
-    if (!$subscriber_in_segment) {
+    if(!$subscriber_in_segment) {
       $queue->delete();
       return false;
     }
     // check if subscriber is confirmed (subscribed)
-    $subscriber = $subscriber_in_segment->subscriber()->findOne();
+    $subscriber = $subscriber_in_segment->subscriber()
-    if ($subscriber->status !== 'subscribed') {
+      ->findOne();
+    if($subscriber->status !== 'subscribed') {
       // reschedule delivery in 5 minutes
       $scheduled_at = Carbon::createFromTimestamp(current_time('timestamp'));
-      $queue->scheduled_at = $scheduled_at->addMinutes(5);
+      $queue->scheduled_at = $scheduled_at->addMinutes(
+        self::UNCONFIRMED_SUBSCRIBER_RESCHEDULE_TIMEOUT
+      );
       $queue->save();
       return false;
     }
     return true;
   }
 
-  private function verifyWPSubscriber($subscriber_id, $newsletter) {
+  function verifyWPSubscriber($subscriber_id, $newsletter) {
     // check if user has the proper role
     $subscriber = Subscriber::findOne($subscriber_id);
     if(!$subscriber || $subscriber->wp_user_id === null) {
@@ -124,45 +122,10 @@ class Scheduler {
     }
     $wp_user = (array) get_userdata($subscriber->wp_user_id);
     if($newsletter->role !== \MailPoet\Newsletter\Scheduler\Scheduler::WORDPRESS_ALL_ROLES
-      && !in_array($newsletter->role, $wp_user['roles'])) {
+      && !in_array($newsletter->role, $wp_user['roles'])
+    ) {
       return false;
     }
     return true;
   }
-
-  // TODO: function will be depreciated once new post notification logic is done
-  private function checkIfNewsletterChanged($newsletter) {
-    $previous_queue = SendingQueue::whereNull('status')
-      ->where('newsletter_id', $newsletter->id)
-      ->orderByDesc('id')
-      ->findOne();
-    if ($previous_queue) return false;
-    $running_queue = SendingQueue::whereNull('status')
-      ->where('newsletter_id', $newsletter->id)
-      ->orderByDesc('id')
-      ->findOne();
-    if ($running_queue) return false;
-    $last_run_queue = SendingQueue::where('status', 'completed')
-      ->where('newsletter_id', $newsletter->id)
-      ->orderByDesc('id')
-      ->findOne();
-    if(!$last_run_queue) return true;
-    if((boolean) Setting::getValue('tracking.enabled')) {
-      // insert tracking code
-      add_filter('mailpoet_rendering_post_process', function ($template) {
-        return OpenTracking::process($template);
-      });
-    }
-    $renderer = new Renderer($newsletter->asArray());
-    $rendered_newsletter = $renderer->render();
-    $new_hash = md5($rendered_newsletter['text']);
-    $old_hash = $last_run_queue->newsletter_rendered_body_hash;
-    return $new_hash !== $old_hash;
-  }
-
-  private function getQueueNextRunDate($schedule) {
-    $schedule = Cron::factory($schedule);
-    return $schedule->getNextRunDate(current_time('mysql'))
-      ->format('Y-m-d H:i:s');
-  }
 }

lib/Cron/Workers/SendingQueue.php

@@ -4,6 +4,7 @@ namespace MailPoet\Cron\Workers;
 use MailPoet\Cron\CronHelper;
 use MailPoet\Mailer\Mailer;
 use MailPoet\Models\Newsletter;
+use MailPoet\Models\NewsletterPost;
 use MailPoet\Models\Setting;
 use MailPoet\Models\StatisticsNewsletters;
 use MailPoet\Models\Subscriber;
@@ -42,6 +43,11 @@ class SendingQueue {
       }
       $newsletter = $newsletter->asArray();
       $newsletter['body'] = $this->getOrRenderNewsletterBody($queue, $newsletter);
+      if ($newsletter['type'] === 'notification' &&
+          !preg_match_all('/data-post-id/', $newsletter['body']['html'])) {
+        $queue->delete();
+        continue;
+      }
       $queue->subscribers = (object) unserialize($queue->subscribers);
       if(!isset($queue->subscribers->processed)) {
         $queue->subscribers->processed = array();
@@ -91,8 +97,7 @@ class SendingQueue {
           return OpenTracking::process($template);
         });
         // render newsletter
-        list($rendered_newsletter, $queue->newsletter_rendered_body_hash) =
+        $rendered_newsletter = $this->renderNewsletter($newsletter);
-          $this->renderNewsletter($newsletter);
         // process link shortcodes, extract and save links in the database
         $processed_newsletter = $this->processLinksAndShortcodes(
           $this->joinObject($rendered_newsletter),
@@ -104,9 +109,12 @@ class SendingQueue {
       }
       else {
         // render newsletter
-        list($newsletter['body'], $queue->newsletter_rendered_body_hash) =
+        $newsletter['body'] = $this->renderNewsletter($newsletter);
-          $this->renderNewsletter($newsletter);
       }
+      $this->extractAndSaveNewsletterPosts(
+        $newsletter['id'],
+        $newsletter['body']['html']
+      );
       $queue->newsletter_rendered_body = json_encode($newsletter['body']);
       $queue->save();
     } else {
@@ -196,9 +204,7 @@ class SendingQueue {
 
   function renderNewsletter($newsletter) {
     $renderer = new Renderer($newsletter);
-    $rendered_newsletter = $renderer->render();
+    return $renderer->render();
-    $rendered_newsletter_hash = md5($rendered_newsletter['text']);
-    return array($rendered_newsletter, $rendered_newsletter_hash);
   }
 
   function processLinksAndShortcodes($content, $newsletter_id, $queue_id) {
@@ -369,6 +375,17 @@ class SendingQueue {
     return array_diff($subscribers_to_process, $subscibers_to_exclude);
   }
 
+  function extractAndSaveNewsletterPosts($newletter_id, $content) {
+    preg_match_all('/data-post-id="(\d+)"/ism', $content, $posts);
+    $posts = $posts[1];
+    foreach($posts as $post) {
+      $newletter_post = NewsletterPost::create();
+      $newletter_post->newsletter_id = $newletter_id;
+      $newletter_post->post_id = $post;
+      $newletter_post->save();
+    }
+  }
+
   private function joinObject($object = array()) {
     return implode(self::DIVIDER, $object);
   }

lib/Models/Subscriber.php

@@ -167,7 +167,10 @@ class Subscriber extends Model {
         }
 
         // welcome email
-        Scheduler::welcomeForSegmentSubscription($subscriber->id, $segment_ids);
+        Scheduler::scheduleSubscriberWelcomeNotification(
+          $subscriber->id,
+          $segment_ids
+        );
       }
     }
 

lib/Newsletter/Scheduler/Scheduler.php

@@ -5,6 +5,7 @@ use Carbon\Carbon;
 use MailPoet\Models\Newsletter;
 use MailPoet\Models\NewsletterOption;
 use MailPoet\Models\NewsletterOptionField;
+use MailPoet\Models\NewsletterPost;
 use MailPoet\Models\SendingQueue;
 
 class Scheduler {
@@ -12,10 +13,11 @@ class Scheduler {
   const LAST_WEEKDAY_FORMAT = 'L';
   const WORDPRESS_ALL_ROLES = 'mailpoet_all';
 
-  static function postNotification($newsletter_id) {
+  static function processPostNotificationSchedule($newsletter_id) {
     $newsletter = Newsletter::filter('filterWithOptions')
-      ->findOne($newsletter_id)
+      ->findOne($newsletter_id);
-      ->asArray();
+    if(!$newsletter) return;
+    $newsletter = $newsletter->asArray();
     $interval_type = $newsletter['intervalType'];
     $hour = (int) $newsletter['timeOfDay'] / self::SECONDS_IN_HOUR;
     $week_day = $newsletter['weekDay'];
@@ -56,20 +58,40 @@ class Scheduler {
     $relation->save();
   }
 
-  static function welcomeForSegmentSubscription($subscriber_id, array $segments) {
+  static function schedulePostNotification($post_id) {
-    $newsletters = self::getWelcomeNewsletters();
+    $newsletters = self::getNewsletters('notification');
+    if(!count($newsletters)) return;
+    foreach($newsletters as $newsletter) {
+      $post = NewsletterPost::where('newsletter_id', $newsletter['id'])
+        ->where('post_id', $post_id)
+        ->findOne();
+      if($post === false) {
+        $scheduled_notification = self::createPostNotificationQueue($newsletter);
+      }
+    }
+  }
+
+  static function scheduleSubscriberWelcomeNotification(
+    $subscriber_id,
+    array $segments
+  ) {
+    $newsletters = self::getWelcomeNewsletters('welcome');
     if(!count($newsletters)) return;
     foreach($newsletters as $newsletter) {
       if($newsletter['event'] === 'segment' &&
         in_array($newsletter['segment'], $segments)
       ) {
-        self::createSendingQueueEntry($newsletter, $subscriber_id);
+        self::createWelcomeNotificationQueue($newsletter, $subscriber_id);
       }
     }
   }
 
-  static function welcomeForNewWPUser($subscriber_id, array $wp_user, $old_user_data) {
+  static function scheduleWPUserWelcomeNotification(
-    $newsletters = self::getWelcomeNewsletters();
+    $subscriber_id,
+    array $wp_user,
+    $old_user_data
+  ) {
+    $newsletters = self::getNewsletters('welcome');
     if(!count($newsletters)) return;
     foreach($newsletters as $newsletter) {
       if($newsletter['event'] === 'user') {
@@ -86,20 +108,20 @@ class Scheduler {
         if($newsletter['role'] === self::WORDPRESS_ALL_ROLES ||
           in_array($newsletter['role'], $wp_user['roles'])
         ) {
-          self::createSendingQueueEntry($newsletter, $subscriber_id);
+          self::createWelcomeNotificationQueue($newsletter, $subscriber_id);
         }
       }
     }
   }
 
-  private static function getWelcomeNewsletters() {
+  static function getNewsletters($type) {
-    return Newsletter::where('type', 'welcome')
+    return Newsletter::where('type', $type)
       ->whereNull('deleted_at')
       ->filter('filterWithOptions')
       ->findArray();
   }
 
-  private static function createSendingQueueEntry($newsletter, $subscriber_id) {
+  static function createWelcomeNotificationQueue($newsletter, $subscriber_id) {
     $queue = SendingQueue::create();
     $queue->newsletter_id = $newsletter['id'];
     $queue->subscribers = serialize(
@@ -129,4 +151,25 @@ class Scheduler {
     $queue->scheduled_at = $scheduled_at;
     $queue->save();
   }
+
+  static function createPostNotificationQueue($newsletter) {
+    $next_run_date = self::getNextRunDate($newsletter['schedule']);
+    // do not schedule duplicate queues for the same time
+    $existing_queue = SendingQueue::where('newsletter_id', $newsletter['id'])
+      ->where('scheduled_at', $next_run_date)
+      ->findOne();
+    if($existing_queue) return;
+    $queue = SendingQueue::create();
+    $queue->newsletter_id = $newsletter['id'];
+    $queue->status = 'scheduled';
+    $queue->scheduled_at = $next_run_date;
+    $queue->save();
+    return $queue;
+  }
+
+  static function getNextRunDate($schedule) {
+    $schedule = \Cron\CronExpression::factory($schedule);
+    return $schedule->getNextRunDate(current_time('mysql'))
+      ->format('Y-m-d H:i:s');
+  }
 }

lib/Router/Newsletters.php

@@ -284,7 +284,7 @@ class Newsletters {
         isset($data['type']) &&
         $data['type'] === 'notification'
       ) {
-        Scheduler::postNotification($newsletter->id);
+        Scheduler::processPostNotificationSchedule($newsletter->id);
       }
       return array(
         'result' => true,

lib/Router/SendingQueue.php

@@ -38,6 +38,7 @@ class SendingQueue {
     $options = array();
     if(isset($data['options'])) {
       $options = $data['options'];
+      $options['segments'] = serialize($data['segments']);
       unset($data['options']);
     }
     if ($options &&
@@ -56,14 +57,12 @@ class SendingQueue {
             $relation->newsletter_id = $newsletter->id;
             $relation->option_field_id = $option_field['id'];
           }
-          $relation->value = ($option_field['name'] === 'segments') ?
+          $relation->value = $options[$option_field['name']];
-            serialize($data['segments']) :
-            $options[$option_field['name']];
           $relation->save();
         }
       }
       if ($newsletter->type === 'notification') {
-        Scheduler::postNotification($newsletter->id);
+        Scheduler::processPostNotificationSchedule($newsletter->id);
       }
       $newsletter = Newsletter::filter('filterWithOptions')
         ->findOne($data['newsletter_id']);

lib/Segments/WP.php

@@ -48,7 +48,7 @@ class WP {
             $segment->addSubscriber($subscriber->id);
           }
           if(isset($schedule_welcome_newsletter)) {
-            Scheduler::welcomeForNewWPUser(
+            Scheduler::scheduleWPUserWelcomeNotification(
               $subscriber->id,
               (array) $wp_user,
               $old_wp_user_data