- Renames/refactors Import and Export classes/views/JS

- Updates Import and Export to ignore trashed subscribers
- Updates tests
Closes #245

assets/js/src/subscribers/importExport/import.js

@@ -20,6 +20,7 @@ define(
      return;
    }
    jQuery(document).ready(function () {
+     jQuery('input[name="select_method"]').attr('checked', false);
      // configure router
      router = new (Backbone.Router.extend({
        routes: {
@@ -1000,7 +1001,6 @@ define(
        }
 
        function toggleNextStepButton(condition) {
-         console.log(condition);
          var disabled = 'button-disabled';
          if (condition === 'on') {
            nextStepButton.removeClass(disabled);

lib/Config/Menu.php

@@ -249,7 +249,7 @@ class Menu {
 
     $data['segments'] = Segment::findArray();
 
-    echo $this->renderer->render('subscribers.html', $data);
+    echo $this->renderer->render('subscribers/subscribers.html', $data);
   }
 
   function segments() {
@@ -290,13 +290,13 @@ class Menu {
   function import() {
     $import = new BootStrapMenu('import');
     $data = $import->bootstrap();
-    echo $this->renderer->render('import.html', $data);
+    echo $this->renderer->render('subscribers/importExport/import.html', $data);
   }
 
   function export() {
     $export = new BootStrapMenu('export');
     $data = $export->bootstrap();
-    echo $this->renderer->render('export.html', $data);
+    echo $this->renderer->render('subscribers/importExport/export.html', $data);
   }
 
   function formEditor() {

lib/Models/Segment.php

@@ -102,6 +102,10 @@ class Segment extends Model {
       ->left_outer_join(
         MP_SUBSCRIBER_SEGMENT_TABLE,
         array(self::$_table.'.id', '=', MP_SUBSCRIBER_SEGMENT_TABLE.'.segment_id'))
+      ->left_outer_join(
+        MP_SUBSCRIBERS_TABLE,
+        array(MP_SUBSCRIBER_SEGMENT_TABLE.'.subscriber_id', '=', MP_SUBSCRIBERS_TABLE.'.id'))
+      ->whereNull(MP_SUBSCRIBERS_TABLE.'.deleted_at')
       ->group_by(self::$_table.'.id')
       ->group_by(self::$_table.'.name')
       ->findArray();
@@ -114,16 +118,18 @@ class Segment extends Model {
       'LEFT JOIN ' . self::$_table . ' segments ON segments.id = relation.segment_id ' .
       'LEFT JOIN ' . MP_SUBSCRIBERS_TABLE . ' subscribers ON subscribers.id = relation.subscriber_id ' .
       (($withConfirmedSubscribers) ?
-        'WHERE subscribers.status = 1 ' :
+        'WHERE subscribers.status = "subscribed" ' :
         'WHERE relation.segment_id IS NOT NULL ') .
+      'AND subscribers.deleted_at IS NULL ' .
       'GROUP BY segments.id) ' .
       'UNION ALL ' .
       '(SELECT 0 as id, "' . __('Not In List') . '" as name, COUNT(*) as subscribers ' .
       'FROM ' . MP_SUBSCRIBERS_TABLE . ' subscribers ' .
       'LEFT JOIN ' . MP_SUBSCRIBER_SEGMENT_TABLE . ' relation on relation.subscriber_id = subscribers.id ' .
       (($withConfirmedSubscribers) ?
-        'WHERE relation.subscriber_id is NULL AND subscribers.status = 1 ' :
+        'WHERE relation.subscriber_id is NULL AND subscribers.status = "subscribed" ' :
         'WHERE relation.subscriber_id is NULL ') .
+      'AND subscribers.deleted_at IS NULL ' .
       'HAVING subscribers) ' .
       'ORDER BY name'
     )->findArray();

lib/Subscribers/ImportExport/Export/Export.php

@@ -9,7 +9,6 @@ use MailPoet\Models\SubscriberSegment;
 use MailPoet\Subscribers\ImportExport\BootStrapMenu;
 use MailPoet\Util\Helpers;
 use MailPoet\Util\XLSXWriter;
-use Symfony\Component\Console\Helper\Helper;
 
 class Export {
   public function __construct($data) {
@@ -93,7 +92,12 @@ class Export {
           );
           $lastSegment = $subscriber['segment_name'];
         }
-        $writer->writeSheet(array_merge($headerRow, $rows), 'MailPoet');
+        $writer->writeSheet(
+          array_merge($headerRow, $rows),
+          ($this->groupBySegmentOption) ?
+            ucwords($subscriber['segment_name']) :
+            'MailPoet'
+        );
         $writer->writeToFile($this->exportFile);
       }
     } catch (Exception $e) {
@@ -155,6 +159,8 @@ class Export {
       $subscribers =
         $subscribers->where(Subscriber::$_table . '.status', 'subscribed');
     }
+    $subscribers = $subscribers->whereNull(Subscriber::$_table . '.deleted_at');
+
     return $subscribers->findArray();
   }
 

lib/Subscribers/ImportExport/Import/Import.php

@@ -1,10 +1,10 @@
 <?php
 namespace MailPoet\Subscribers\ImportExport\Import;
 
-use MailPoet\Subscribers\ImportExport\BootStrapMenu;
 use MailPoet\Models\Subscriber;
 use MailPoet\Models\SubscriberCustomField;
 use MailPoet\Models\SubscriberSegment;
+use MailPoet\Subscribers\ImportExport\BootStrapMenu;
 use MailPoet\Util\Helpers;
 
 class Import {
@@ -27,7 +27,9 @@ class Import {
     $subscriberFields = $this->subscriberFields;
     $subscriberCustomFields = $this->subscriberCustomFields;
     $subscribersData = $this->subscribersData;
-    $subscribersData = $this->filterSubscriberStatus($subscribersData);
+    list ($subscribersData, $subscriberFields) =
+      $this->filterSubscriberStatus($subscribersData, $subscriberFields);
+    $this->deleteExistingTrashedSubscribers($subscribersData);
     list($subscribersData, $subscriberFields) = $this->extendSubscribersAndFields(
       $subscribersData, $subscriberFields
     );
@@ -84,6 +86,7 @@ class Import {
       array_map(function ($subscriberEmails) {
         return Subscriber::selectMany(array('email'))
           ->whereIn('email', $subscriberEmails)
+          ->whereNull('deleted_at')
           ->findArray();
       }, array_chunk($subscribersData['email'], 200))
     );
@@ -131,6 +134,25 @@ class Import {
     );
   }
 
+  function deleteExistingTrashedSubscribers($subscribersData) {
+    $existingTrashedRecords = array_filter(
+      array_map(function ($subscriberEmails) {
+        return Subscriber::selectMany(array('id'))
+          ->whereIn('email', $subscriberEmails)
+          ->whereNotNull('deleted_at')
+          ->findArray();
+      }, array_chunk($subscribersData['email'], 200))
+    );
+    if(!$existingTrashedRecords) return;
+    $existingTrashedRecords = Helpers::flattenArray($existingTrashedRecords);
+    foreach (array_chunk($existingTrashedRecords, 200) as $subscriberIds) {
+      Subscriber::whereIn('id', $subscriberIds)
+        ->deleteMany();
+      SubscriberSegment::whereIn('subscriber_id', $subscriberIds)
+        ->deleteMany();
+    }
+  }
+
   function extendSubscribersAndFields($subscribersData, $subscriberFields) {
     $subscribersData['created_at'] = $this->filterSubscriberCreatedAtDate();
     $subscriberFields[] = 'created_at';
@@ -164,8 +186,16 @@ class Import {
     return array_fill(0, $this->subscribersCount, $this->currentTime);
   }
 
-  function filterSubscriberStatus($subscribersData) {
+  function filterSubscriberStatus($subscribersData, $subscriberFields) {
-    if(!in_array('status', $this->subscriberFields)) return $subscribersData;
+    if(!in_array('status', $subscriberFields)) {
+      $subscribersData['status'] =
+        array_fill(0, count($subscribersData['email']), 'subscribed');
+      $subscriberFields[] = 'status';
+      return array(
+        $subscribersData,
+        $subscriberFields
+      );
+    }
     $statuses = array(
       'subscribed' => array(
         'subscribed',
@@ -198,7 +228,10 @@ class Import {
       }
       return 'subscribed'; // make "subscribed" a default status
     }, $subscribersData['status']);
-    return $subscribersData;
+    return array(
+      $subscribersData,
+      $subscriberFields
+    );
   }
 
   function createOrUpdateSubscribers(

tests/unit/Subscribers/ImportExport/BootStrapMenuCest.php

@@ -1,10 +1,10 @@
 <?php
 
-use MailPoet\Subscribers\ImportExport\BootStrapMenu;
 use MailPoet\Models\CustomField;
 use MailPoet\Models\Segment;
 use MailPoet\Models\Subscriber;
 use MailPoet\Models\SubscriberSegment;
+use MailPoet\Subscribers\ImportExport\BootStrapMenu;
 
 class BootStrapMenuCest {
   function _before() {
@@ -22,13 +22,13 @@ class BootStrapMenuCest {
       array(
         'first_name' => 'John',
         'last_name' => 'Mailer',
-        'status' => 0,
+        'status' => 'unconfirmed',
         'email' => 'john@mailpoet.com'
       ),
       array(
         'first_name' => 'Mike',
         'last_name' => 'Smith',
-        'status' => 1,
+        'status' => 'subscribed',
         'email' => 'mike@maipoet.com'
       )
     );
@@ -52,6 +52,28 @@ class BootStrapMenuCest {
     expect($segments[1]['subscriberCount'])->equals(1);
   }
 
+  function itCanGetSegmentsForImportWithoutTrashedSubscribers() {
+    $this->_createSegmentsAndSubscribers();
+    $segments = $this->bootStrapImportMenu->getSegments();
+    expect(count($segments))->equals(2);
+    $subscriber = Subscriber::findOne(1);
+    $subscriber->deleted_at = date('Y-m-d H:i:s');
+    $subscriber->save();
+    $segments = $this->bootStrapImportMenu->getSegments();
+    expect(count($segments))->equals(1);
+  }
+
+  function itCanGetSegmentsForExportWithoutTrashedSubscribers() {
+    $this->_createSegmentsAndSubscribers();
+    $segments = $this->bootStrapExportMenu->getSegments();
+    expect(count($segments))->equals(2);
+    $subscriber = Subscriber::findOne(1);
+    $subscriber->deleted_at = date('Y-m-d H:i:s');
+    $subscriber->save();
+    $segments = $this->bootStrapExportMenu->getSegments();
+    expect(count($segments))->equals(1);
+  }
+
   function itCanGetSegmentsForExport() {
     $this->_createSegmentsAndSubscribers();
     $segments = $this->bootStrapExportMenu->getSegments();
@@ -270,13 +292,9 @@ class BootStrapMenuCest {
   }
 
   function _after() {
-    ORM::forTable(Subscriber::$_table)
+    ORM::raw_execute('TRUNCATE ' . Subscriber::$_table);
-      ->deleteMany();
+    ORM::raw_execute('TRUNCATE ' . Segment::$_table);
-    ORM::forTable(CustomField::$_table)
+    ORM::raw_execute('TRUNCATE ' . SubscriberSegment::$_table);
-      ->deleteMany();
+    ORM::raw_execute('TRUNCATE ' . CustomField::$_table);
-    ORM::forTable(Segment::$_table)
-      ->deleteMany();
-    ORM::forTable(SubscriberSegment::$_table)
-      ->deleteMany();
   }
 }

tests/unit/Subscribers/ImportExport/Import/ImportCest.php

@@ -105,28 +105,39 @@ class ImportCest {
     expect($fields)->equals(array(39));
   }
 
-  function itCanFilterSubscriberState() {
+  function itCanFilterSubscriberStatus() {
-    $data = array(
+    $subscibersData = $this->subscribersData;
+    $subscriberFields = $this->subscriberFields;
+    list($subscibersData, $subsciberFields) =
+      $this->import->filterSubscriberStatus($subscibersData, $subscriberFields);
+    // subscribers' status was set to "subscribed" & status column was added
+    // to subscribers fields
+    expect(array_pop($subsciberFields))->equals('status');
+    expect($subscibersData['status'][0])->equals('subscribed');
+    expect(count($subscibersData['status']))->equals(2);
+    $subscriberFields[] = 'status';
+    $subscibersData = array(
       'status' => array(
-        //subscribed
+        #subscribed
         'subscribed',
         'confirmed',
         1,
         '1',
         'true',
-        //unconfirmed
+        #unconfirmed
         'unconfirmed',
         0,
         "0",
-        //unsubscribed
+        #unsubscribed
         'unsubscribed',
         -1,
         '-1',
         'false'
       ),
     );
-    $statuses = $this->import->filterSubscriberStatus($data);
+    list($subscibersData, $subsciberFields) =
-    expect($statuses)->equals(
+      $this->import->filterSubscriberStatus($subscibersData, $subscriberFields);
+    expect($subscibersData)->equals(
       array(
         'status' => array(
           'subscribed',
@@ -170,6 +181,41 @@ class ImportCest {
       ->equals($subscribersData['first_name'][1]);
   }
 
+  function itCanDeleteTrashedSubscribers() {
+    $subscribersData = $this->subscribersData;
+    $subscriberFields = $this->subscriberFields;
+    $subscribersData['deleted_at'] = array(
+        null,
+        date('Y-m-d H:i:s')
+    );
+    $subscriberFields[] = 'deleted_at';
+    $this->import->createOrUpdateSubscribers(
+      'create',
+      $subscribersData,
+      $subscriberFields,
+      false
+    );
+    $dbSubscribers = Helpers::arrayColumn(
+      Subscriber::select('id')
+        ->findArray(),
+      'id'
+    );
+    expect(count($dbSubscribers))->equals(2);
+    $this->import->addSubscribersToSegments(
+      $dbSubscribers,
+      $this->segments
+    );
+    $subscribersSegments = SubscriberSegment::findArray();
+    expect(count($subscribersSegments))->equals(4);
+    $this->import->deleteExistingTrashedSubscribers(
+      $subscribersData
+    );
+    $subscribersSegments = SubscriberSegment::findArray();
+    $dbSubscribers = Subscriber::findArray();
+    expect(count($subscribersSegments))->equals(2);
+    expect(count($dbSubscribers))->equals(1);
+  }
+
   function itCanCreateOrUpdateCustomFields() {
     $subscribersData = $this->subscribersData;
     $this->import->createOrUpdateSubscribers(
@@ -231,6 +277,22 @@ class ImportCest {
     expect(count($subscribersSegments))->equals(4);
   }
 
+  function itCanDeleteExistingTrashedSubscribers() {
+    $subscribersData = $this->subscribersData;
+    $subscriberFields = $this->subscriberFields;
+    $subscriberFields[] = 'deleted_at';
+    $subscribersData['deleted_at'] = array(
+      null,
+      date('Y-m-d H:i:s')
+    );
+    $this->import->createOrUpdateSubscribers(
+      'create',
+      $subscribersData,
+      $subscriberFields,
+      false
+    );
+  }
+
   function itCanProcess() {
     $import = clone($this->import);
     $result = $import->process();
@@ -253,11 +315,8 @@ class ImportCest {
   }
 
   function _after() {
-    ORM::forTable(Subscriber::$_table)
+    ORM::raw_execute('TRUNCATE ' . Subscriber::$_table);
-      ->deleteMany();
+    ORM::raw_execute('TRUNCATE ' . SubscriberSegment::$_table);
-    ORM::forTable(SubscriberCustomField::$_table)
+    ORM::raw_execute('TRUNCATE ' . SubscriberCustomField::$_table);
-      ->deleteMany();
-    ORM::forTable(SubscriberSegment::$_table)
-      ->deleteMany();
   }
 }

views/subscribers/importExport/export.html

@@ -123,7 +123,7 @@
     exportData = {
      segments: segments.length || null,
      segmentsWithConfirmedSubscribers: segmentsWithConfirmedSubscribers.length || null,
-     exportConfirmedOption: false,
+     exportConfirmedOption: true,
      groupBySegmentOption: (segments.length > 1 || segmentsWithConfirmedSubscribers.length > 1) ? true : null
     };
 </script>

views/subscribers/importExport/import.html

@@ -1,5 +1,4 @@
 <% extends 'layout.html' %>
-
 <% block content %>
 <div id="mailpoet_subscribers_import" class="wrap">
   <h2 class="title">
@@ -7,11 +6,11 @@
     <a class="add-new-h2" href="?page=mailpoet-subscribers#/"><%= __('Back to list') %></a>
   </h2>
   <!-- STEP 1: method selection -->
-  <% include 'import/step1.html' %>
+  <% include 'subscribers/importExport/import/step1.html' %>
   <!-- STEP 2: subscriber data manipulation -->
-  <% include 'import/step2.html' %>
+  <% include 'subscribers/importExport/import/step2.html' %>
   <!-- STEP 3: results -->
-  <% include 'import/step3.html' %>
+  <% include 'subscribers/importExport/import/step3.html' %>
 </div>
 
 <%= stylesheet('importExport.css') %>
@@ -64,6 +63,7 @@
 <script type="text/javascript">
   var
     maxPostSize = '<%= maxPostSize %>',
+    maxPostSizeBytes = '<%= maxPostSizeBytes %>',
     importData = {},
     mailpoetColumnsSelect2 = <%= subscriberFieldsSelect2|raw %>,
     mailpoetColumns = <%= subscriberFields|raw %>,

webpack.config.js

@@ -100,8 +100,8 @@ config.push(_.extend({}, baseConfig, {
       'segments/segments.jsx',
       'forms/forms.jsx',
       'settings/tabs.js',
-      'import/import.js',
+      'subscribers/importExport/import.js',
-      'export/export.js',
+      'subscribers/importExport/export.js',
       'helpscout'
     ],
     form_editor: [