- Rewrites export to support large datasets

- Updates unit tests
- Fixes an issue with export UI not displaying subscribers without segment

lib/Subscribers/ImportExport/BootStrapMenu.php

@@ -17,6 +17,8 @@ class BootStrapMenu {
       Segment::getSegmentsWithSubscriberCount() :
       Segment::getSegmentsForExport($with_confirmed_subscribers);
     return array_map(function($segment) {
+      if (!$segment['name']) $segment['name'] = __('Not In Segment');
+      if (!$segment['id']) $segment['id'] = 0;
       return array(
         'id' => $segment['id'],
         'name' => $segment['name'],
@@ -31,7 +33,7 @@ class BootStrapMenu {
       'first_name' => __('First name'),
       'last_name' => __('Last name'),
       'status' => __('Status')
-      // TODO: add additional fiels from MP2
+      // TODO: add additional fields from MP2
       /*
             'confirmed_ip' => __('IP address')
             'confirmed_at' => __('Subscription date')

lib/Subscribers/ImportExport/Export/Export.php

@@ -17,22 +17,30 @@ class Export {
   public $segments;
   public $subscribers_without_segment;
   public $subscriber_fields;
+  public $subscriber_custom_fields;
+  public $formatted_subscriber_fields;
   public $export_path;
   public $export_file;
   public $export_file_URL;
-  public $profiler_start;
+  public $subscriber_batch_size;
 
   public function __construct($data) {
+    set_time_limit(0);
     $this->export_confirmed_option = $data['export_confirmed_option'];
     $this->export_format_option = $data['export_format_option'];
     $this->group_by_segment_option = $data['group_by_segment_option'];
     $this->segments = $data['segments'];
     $this->subscribers_without_segment = array_search(0, $this->segments);
     $this->subscriber_fields = $data['subscriber_fields'];
+    $this->subscriber_custom_fields = $this->getSubscriberCustomFields();
+    $this->formatted_subscriber_fields = $this->formatSubscriberFields(
+      $this->subscriber_fields,
+      $this->subscriber_custom_fields
+    );
     $this->export_path = Env::$temp_path;
     $this->export_file = $this->getExportFile($this->export_format_option);
     $this->export_file_URL = $this->getExportFileURL($this->export_file);
-    $this->profiler_start = microtime(true);
+    $this->subscriber_batch_size = 15000;
   }
 
   function process() {
@@ -40,75 +48,12 @@ class Export {
       if(is_writable($this->export_path) === false) {
         throw new \Exception(__("Couldn't save export file on the server."));
       }
-      $subscribers = $this->getSubscribers();
+      $processed_subscribers = call_user_func(
-      $subscriber_custom_fields = $this->getSubscriberCustomFields();
+        array(
-      $formatted_subscriber_fields = $this->formatSubscriberFields(
+          $this,
-        $this->subscriber_fields,
+          'generate' . strtoupper($this->export_format_option)
-        $subscriber_custom_fields
+        )
       );
-      if($this->export_format_option === 'csv') {
-        $CSV_file = fopen($this->export_file, 'w');
-        $format_CSV = function($row) {
-          return '"' . str_replace('"', '\"', $row) . '"';
-        };
-        // add UTF-8 BOM (3 bytes, hex EF BB BF) at the start of the file for
-        // Excel to automatically recognize the encoding
-        fwrite($CSV_file, chr(0xEF) . chr(0xBB) . chr(0xBF));
-        if($this->group_by_segment_option) {
-          $formatted_subscriber_fields[] = __('Segment');
-        }
-        fwrite(
-          $CSV_file,
-          implode(
-            ',',
-            array_map(
-              $format_CSV,
-              $formatted_subscriber_fields
-            )
-          ) . "\n"
-        );
-        foreach($subscribers as $subscriber) {
-          $row = $this->formatSubscriberData($subscriber);
-          if($this->group_by_segment_option) {
-            $row[] = ucwords($subscriber['segment_name']);
-          }
-          fwrite($CSV_file, implode(',', array_map($format_CSV, $row)) . "\n");
-        }
-        fclose($CSV_file);
-      } else {
-        $writer = new XLSXWriter();
-        $writer->setAuthor('MailPoet (www.mailpoet.com)');
-        $header_row = array($formatted_subscriber_fields);
-        $last_segment = false;
-        $rows = array();
-        foreach($subscribers as $subscriber) {
-          if($last_segment && $last_segment !== $subscriber['segment_name'] &&
-            $this->group_by_segment_option
-          ) {
-            $writer->writeSheet(
-              array_merge($header_row, $rows), ucwords($last_segment)
-            );
-            $rows = array();
-          }
-          // detect RTL language and set Excel to properly display the sheet
-          $RTL_regex = '/\p{Arabic}|\p{Hebrew}/u';
-          if(!$writer->rtl && (
-              preg_grep($RTL_regex, $subscriber) ||
-              preg_grep($RTL_regex, $formatted_subscriber_fields))
-          ) {
-            $writer->rtl = true;
-          }
-          $rows[] = $this->formatSubscriberData($subscriber);
-          $last_segment = $subscriber['segment_name'];
-        }
-        $writer->writeSheet(
-          array_merge($header_row, $rows),
-          ($this->group_by_segment_option) ?
-            ucwords($subscriber['segment_name']) :
-            __('All Segments')
-        );
-        $writer->writeToFile($this->export_file);
-      }
     } catch(\Exception $e) {
       return array(
         'result' => false,
@@ -118,14 +63,118 @@ class Export {
     return array(
       'result' => true,
       'data' => array(
-        'totalExported' => count($subscribers),
+        'totalExported' => $processed_subscribers,
         'exportFileURL' => $this->export_file_URL
-      ),
+      )
-      'profiler' => $this->timeExecution()
     );
   }
 
-  function getSubscribers() {
+  function generateCSV() {
+    $processed_subscribers = 0;
+    $offset = 0;
+    $formatted_subscriber_fields = $this->formatted_subscriber_fields;
+    $CSV_file = fopen($this->export_file, 'w');
+    $format_CSV = function($row) {
+      return '"' . str_replace('"', '\"', $row) . '"';
+    };
+    // add UTF-8 BOM (3 bytes, hex EF BB BF) at the start of the file for
+    // Excel to automatically recognize the encoding
+    fwrite($CSV_file, chr(0xEF) . chr(0xBB) . chr(0xBF));
+    if($this->group_by_segment_option) {
+      $formatted_subscriber_fields[] = __('Segment');
+    }
+    fwrite(
+      $CSV_file,
+      implode(
+        ',',
+        array_map(
+          $format_CSV,
+          $formatted_subscriber_fields
+        )
+      ) . PHP_EOL
+    );
+    do {
+      $subscribers = $this->getSubscribers($offset, $this->subscriber_batch_size);
+      $processed_subscribers += count($subscribers);
+      foreach($subscribers as $subscriber) {
+        $row = $this->formatSubscriberData($subscriber);
+        if($this->group_by_segment_option) {
+          $row[] = ucwords($subscriber['segment_name']);
+        }
+        fwrite($CSV_file, implode(',', array_map($format_CSV, $row)) . "\n");
+      }
+      $offset += $this->subscriber_batch_size;
+    } while(count($subscribers) === $this->subscriber_batch_size);
+    fclose($CSV_file);
+    return $processed_subscribers;
+  }
+
+  function generateXLSX() {
+    $processed_subscribers = 0;
+    $offset = 0;
+    $XLSX_writer = new XLSXWriter();
+    $XLSX_writer->setAuthor('MailPoet (www.mailpoet.com)');
+    $last_segment = false;
+    $processed_segments = array();
+    do {
+      $subscribers = $this->getSubscribers($offset, $this->subscriber_batch_size);
+      $processed_subscribers += count($subscribers);
+      foreach($subscribers as $i => $subscriber) {
+        $current_segment = ucwords($subscriber['segment_name']);
+        // Sheet header (1st row) will be written only if:
+        // * This is the first time we're processing a segment
+        // * "Group by subscriber option" is turned AND the previous subscriber's
+        // segment is different from the current subscriber's segment
+        // Header will NOT be written if:
+        // * We have already processed the segment. Because SQL results are not
+        // sorted by segment name (due to slow queries when using ORDER BY and LIMIT),
+        // we need to keep track of processed segments so that we do not create header
+        // multiple times when switching from one segment to another and back.
+        if((!count($processed_segments) ||
+            ($last_segment !== $current_segment && $this->group_by_segment_option)
+          ) &&
+          (!in_array($last_segment, $processed_segments) ||
+            !in_array($current_segment, $processed_segments)
+          )
+        ) {
+          $this->writeXLSX(
+            $XLSX_writer,
+            $subscriber['segment_name'],
+            $this->formatted_subscriber_fields
+          );
+          $processed_segments[] = $current_segment;
+        }
+        $last_segment = ucwords($subscriber['segment_name']);
+        // detect RTL language and set Excel to properly display the sheet
+        $RTL_regex = '/\p{Arabic}|\p{Hebrew}/u';
+        if(!$XLSX_writer->rtl && (
+            preg_grep($RTL_regex, $subscriber) ||
+            preg_grep($RTL_regex, $this->formatted_subscriber_fields))
+        ) {
+          $XLSX_writer->rtl = true;
+        }
+        $this->writeXLSX(
+          $XLSX_writer,
+          $last_segment,
+          $this->formatSubscriberData($subscriber)
+        );
+      }
+      $offset += $this->subscriber_batch_size;
+    } while(count($subscribers) === $this->subscriber_batch_size);
+    $XLSX_writer->writeToFile($this->export_file);
+    return $processed_subscribers;
+  }
+
+  function writeXLSX($XLSX_writer, $segment, $data) {
+    return $XLSX_writer->writeSheetRow(
+      ($this->group_by_segment_option) ?
+        ucwords($segment) :
+        __('All Segments'),
+      $data
+    );
+  }
+
+  function getSubscribers($offset, $limit) {
     $subscribers = Subscriber::
     left_outer_join(
       SubscriberSegment::$_table,
@@ -141,7 +190,6 @@ class Export {
           '=',
           SubscriberSegment::$_table . '.segment_id'
         ))
-      ->orderByAsc('segment_name')
       ->filter('filterWithCustomFieldsForExport');
     if($this->subscribers_without_segment !== false) {
       $subscribers = $subscribers
@@ -168,9 +216,11 @@ class Export {
       $subscribers =
         $subscribers->where(Subscriber::$_table . '.status', 'subscribed');
     }
-    $subscribers = $subscribers->whereNull(Subscriber::$_table . '.deleted_at');
+    $subscribers = $subscribers
-
+      ->whereNull(Subscriber::$_table . '.deleted_at')
-    return $subscribers->findArray();
+      ->limit(sprintf('%d, %d', $offset, $limit))
+      ->findArray();
+    return $subscribers;
   }
 
   function getExportFileURL($file) {
@@ -216,9 +266,4 @@ class Export {
       return $subscriber[$field];
     }, $this->subscriber_fields);
   }
-
-  function timeExecution() {
-    $profiler_end = microtime(true);
-    return ($profiler_end - $this->profiler_start) / 60;
-  }
 }

tests/unit/Subscribers/ImportExport/Export/ExportCest.php

@@ -122,6 +122,15 @@ class ExportCest {
           '1'
         )
       );
+    expect($this->export->subscriber_custom_fields)
+      ->equals($this->export->getSubscriberCustomFields());
+    expect($this->export->formatted_subscriber_fields)
+      ->equals(
+        $this->export->formatSubscriberFields(
+          $this->export->subscriber_fields,
+          $this->export->subscriber_custom_fields
+        )
+      );
     expect(
       preg_match(
         '|' .
@@ -137,6 +146,7 @@ class ExportCest {
         '|'
         , $this->export->export_file_URL)
     )->equals(1);
+    expect($this->export->subscriber_batch_size)->notNull();
   }
 
   function itCanGetSubscriberCustomFields() {
@@ -156,7 +166,7 @@ class ExportCest {
   }
 
   function itProperlyReturnsSubscriberCustomFields() {
-    $subscribers = $this->export->getSubscribers();
+    $subscribers = $this->export->getSubscribers(0, 10);
     foreach($subscribers as $subscriber) {
       if($subscriber['email'] === $this->subscribers_data[1]) {
         expect($subscriber['Country'])
@@ -167,37 +177,37 @@ class ExportCest {
 
   function itCanGetSubscribers() {
     $this->export->segments = array(1);
-    $subscribers = $this->export->getSubscribers();
+    $subscribers = $this->export->getSubscribers(0, 10);
     expect(count($subscribers))->equals(2);
     $this->export->segments = array(2);
-    $subscribers = $this->export->getSubscribers();
+    $subscribers = $this->export->getSubscribers(0, 10);
     expect(count($subscribers))->equals(2);
     $this->export->segments = array(
       1,
       2
     );
-    $subscribers = $this->export->getSubscribers();
+    $subscribers = $this->export->getSubscribers(0, 10);
     expect(count($subscribers))->equals(3);
   }
 
   function itCanGroupSubscribersBySegments() {
     $this->export->group_by_segment_option = true;
     $this->export->subscribers_without_segment = true;
-    $subscribers = $this->export->getSubscribers();
+    $subscribers = $this->export->getSubscribers(0, 10);
     expect(count($subscribers))->equals(5);
   }
 
   function itCanGetSubscribersOnlyWithoutSegments() {
     $this->export->segments = array(0);
     $this->export->subscribers_without_segment = true;
-    $subscribers = $this->export->getSubscribers();
+    $subscribers = $this->export->getSubscribers(0, 10);
     expect(count($subscribers))->equals(1);
     expect($subscribers[0]['segment_name'])->equals('Not In Segment');
   }
 
   function itCanGetOnlyConfirmedSubscribers() {
     $this->export->export_confirmed_option = true;
-    $subscribers = $this->export->getSubscribers();
+    $subscribers = $this->export->getSubscribers(0, 10);
     expect(count($subscribers))->equals(1);
     expect($subscribers[0]['email'])
       ->equals($this->subscribers_data[1]['email']);
@@ -207,7 +217,7 @@ class ExportCest {
     SubscriberSegment::where('subscriber_id', 3)
       ->findOne()
       ->delete();
-    $subscribers = $this->export->getSubscribers();
+    $subscribers = $this->export->getSubscribers(0, 10);
     expect(count($subscribers))->equals(2);
   }
 
@@ -223,8 +233,8 @@ class ExportCest {
     $this->export->export_format_option = 'csv';
     $this->export->process();
     $CSV_file_size = filesize($this->export->export_file);
-    $this->export->export_file = $this->export->getExportFile('xls');
+    $this->export->export_file = $this->export->getExportFile('xlsx');
-    $this->export->export_format_option = 'xls';
+    $this->export->export_format_option = 'xlsx';
     $this->export->process();
     $XLS_file_size = filesize($this->export->export_file);
     expect($CSV_file_size)->greaterThan(0);