Cavemanon/piratepoet
7
Fork 0
Code Pull Requests Actions Releases 16 Activity

- Rewrites export to support large datasets

Browse Source 
- Updates unit tests
- Fixes an issue with export UI not displaying subscribers without segment
This commit is contained in:
Vlad
2016-03-03 13:30:36 -05:00
parent 2ec6bc8c99
commit eb380499d9
3 changed files with 151 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lib/Subscribers/ImportExport/BootStrapMenu.php
View File

@ -17,6 +17,8 @@ class BootStrapMenu {
Segment::getSegmentsWithSubscriberCount() :
Segment::getSegmentsForExport($with_confirmed_subscribers);
return array_map(function($segment) {
if (!$segment['name']) $segment['name'] = __('Not In Segment');
if (!$segment['id']) $segment['id'] = 0;
return array(
'id' => $segment['id'],
'name' => $segment['name'],
@ -31,7 +33,7 @@ class BootStrapMenu {
'first_name' => __('First name'),
'last_name' => __('Last name'),
'status' => __('Status')
// TODO: add additional fiels from MP2
// TODO: add additional fields from MP2
/*
'confirmed_ip' => __('IP address')
'confirmed_at' => __('Subscription date')

211
lib/Subscribers/ImportExport/Export/Export.php
View File

@ -17,22 +17,30 @@ class Export {
public $segments;
public $subscribers_without_segment;
public $subscriber_fields;
public $subscriber_custom_fields;
public $formatted_subscriber_fields;
public $export_path;
public $export_file;
public $export_file_URL;
public $profiler_start;
public $subscriber_batch_size;
public function __construct($data) {
set_time_limit(0);
$this->export_confirmed_option = $data['export_confirmed_option'];
$this->export_format_option = $data['export_format_option'];
$this->group_by_segment_option = $data['group_by_segment_option'];
$this->segments = $data['segments'];
$this->subscribers_without_segment = array_search(0, $this->segments);
$this->subscriber_fields = $data['subscriber_fields'];
$this->subscriber_custom_fields = $this->getSubscriberCustomFields();
$this->formatted_subscriber_fields = $this->formatSubscriberFields(
$this->subscriber_fields,
$this->subscriber_custom_fields
);
$this->export_path = Env::$temp_path;
$this->export_file = $this->getExportFile($this->export_format_option);
$this->export_file_URL = $this->getExportFileURL($this->export_file);
$this->profiler_start = microtime(true);
$this->subscriber_batch_size = 15000;
}
function process() {
@ -40,75 +48,12 @@ class Export {
if(is_writable($this->export_path) === false) {
throw new \Exception(__("Couldn't save export file on the server."));
}
$subscribers = $this->getSubscribers();
$subscriber_custom_fields = $this->getSubscriberCustomFields();
$formatted_subscriber_fields = $this->formatSubscriberFields(
$this->subscriber_fields,
$subscriber_custom_fields
$processed_subscribers = call_user_func(
array(
$this,
'generate' . strtoupper($this->export_format_option)
)
);
if($this->export_format_option === 'csv') {
$CSV_file = fopen($this->export_file, 'w');
$format_CSV = function($row) {
return '"' . str_replace('"', '\"', $row) . '"';
};
// add UTF-8 BOM (3 bytes, hex EF BB BF) at the start of the file for
// Excel to automatically recognize the encoding
fwrite($CSV_file, chr(0xEF) . chr(0xBB) . chr(0xBF));
if($this->group_by_segment_option) {
$formatted_subscriber_fields[] = __('Segment');
}
fwrite(
$CSV_file,
implode(
',',
array_map(
$format_CSV,
$formatted_subscriber_fields
)
) . "\n"
);
foreach($subscribers as $subscriber) {
$row = $this->formatSubscriberData($subscriber);
if($this->group_by_segment_option) {
$row[] = ucwords($subscriber['segment_name']);
}
fwrite($CSV_file, implode(',', array_map($format_CSV, $row)) . "\n");
}
fclose($CSV_file);
} else {
$writer = new XLSXWriter();
$writer->setAuthor('MailPoet (www.mailpoet.com)');
$header_row = array($formatted_subscriber_fields);
$last_segment = false;
$rows = array();
foreach($subscribers as $subscriber) {
if($last_segment && $last_segment !== $subscriber['segment_name'] &&
$this->group_by_segment_option
) {
$writer->writeSheet(
array_merge($header_row, $rows), ucwords($last_segment)
);
$rows = array();
}
// detect RTL language and set Excel to properly display the sheet
$RTL_regex = '/\p{Arabic}|\p{Hebrew}/u';
if(!$writer->rtl && (
preg_grep($RTL_regex, $subscriber) ||
preg_grep($RTL_regex, $formatted_subscriber_fields))
) {
$writer->rtl = true;
}
$rows[] = $this->formatSubscriberData($subscriber);
$last_segment = $subscriber['segment_name'];
}
$writer->writeSheet(
array_merge($header_row, $rows),
($this->group_by_segment_option) ?
ucwords($subscriber['segment_name']) :
__('All Segments')
);
$writer->writeToFile($this->export_file);
}
} catch(\Exception $e) {
return array(
'result' => false,
@ -118,14 +63,118 @@ class Export {
return array(
'result' => true,
'data' => array(
'totalExported' => count($subscribers),
'totalExported' => $processed_subscribers,
'exportFileURL' => $this->export_file_URL
),
'profiler' => $this->timeExecution()
)
);
}
function getSubscribers() {
function generateCSV() {
$processed_subscribers = 0;
$offset = 0;
$formatted_subscriber_fields = $this->formatted_subscriber_fields;
$CSV_file = fopen($this->export_file, 'w');
$format_CSV = function($row) {
return '"' . str_replace('"', '\"', $row) . '"';
};
// add UTF-8 BOM (3 bytes, hex EF BB BF) at the start of the file for
// Excel to automatically recognize the encoding
fwrite($CSV_file, chr(0xEF) . chr(0xBB) . chr(0xBF));
if($this->group_by_segment_option) {
$formatted_subscriber_fields[] = __('Segment');
}
fwrite(
$CSV_file,
implode(
',',
array_map(
$format_CSV,
$formatted_subscriber_fields
)
) . PHP_EOL
);
do {
$subscribers = $this->getSubscribers($offset, $this->subscriber_batch_size);
$processed_subscribers += count($subscribers);
foreach($subscribers as $subscriber) {
$row = $this->formatSubscriberData($subscriber);
if($this->group_by_segment_option) {
$row[] = ucwords($subscriber['segment_name']);
}
fwrite($CSV_file, implode(',', array_map($format_CSV, $row)) . "\n");
}
$offset += $this->subscriber_batch_size;
} while(count($subscribers) === $this->subscriber_batch_size);
fclose($CSV_file);
return $processed_subscribers;
}
function generateXLSX() {
$processed_subscribers = 0;
$offset = 0;
$XLSX_writer = new XLSXWriter();
$XLSX_writer->setAuthor('MailPoet (www.mailpoet.com)');
$last_segment = false;
$processed_segments = array();
do {
$subscribers = $this->getSubscribers($offset, $this->subscriber_batch_size);
$processed_subscribers += count($subscribers);
foreach($subscribers as $i => $subscriber) {
$current_segment = ucwords($subscriber['segment_name']);
// Sheet header (1st row) will be written only if:
// * This is the first time we're processing a segment
// * "Group by subscriber option" is turned AND the previous subscriber's
// segment is different from the current subscriber's segment
// Header will NOT be written if:
// * We have already processed the segment. Because SQL results are not
// sorted by segment name (due to slow queries when using ORDER BY and LIMIT),
// we need to keep track of processed segments so that we do not create header
// multiple times when switching from one segment to another and back.
if((!count($processed_segments) ||
($last_segment !== $current_segment && $this->group_by_segment_option)
) &&
(!in_array($last_segment, $processed_segments) ||
!in_array($current_segment, $processed_segments)
)
) {
$this->writeXLSX(
$XLSX_writer,
$subscriber['segment_name'],
$this->formatted_subscriber_fields
);
$processed_segments[] = $current_segment;
}
$last_segment = ucwords($subscriber['segment_name']);
// detect RTL language and set Excel to properly display the sheet
$RTL_regex = '/\p{Arabic}|\p{Hebrew}/u';
if(!$XLSX_writer->rtl && (
preg_grep($RTL_regex, $subscriber) ||
preg_grep($RTL_regex, $this->formatted_subscriber_fields))
) {
$XLSX_writer->rtl = true;
}
$this->writeXLSX(
$XLSX_writer,
$last_segment,
$this->formatSubscriberData($subscriber)
);
}
$offset += $this->subscriber_batch_size;
} while(count($subscribers) === $this->subscriber_batch_size);
$XLSX_writer->writeToFile($this->export_file);
return $processed_subscribers;
}
function writeXLSX($XLSX_writer, $segment, $data) {
return $XLSX_writer->writeSheetRow(
($this->group_by_segment_option) ?
ucwords($segment) :
__('All Segments'),
$data
);
}
function getSubscribers($offset, $limit) {
$subscribers = Subscriber::
left_outer_join(
SubscriberSegment::$_table,
@ -141,7 +190,6 @@ class Export {
'=',
SubscriberSegment::$_table . '.segment_id'
))
->orderByAsc('segment_name')
->filter('filterWithCustomFieldsForExport');
if($this->subscribers_without_segment !== false) {
$subscribers = $subscribers
@ -168,9 +216,11 @@ class Export {
$subscribers =
$subscribers->where(Subscriber::$_table . '.status', 'subscribed');
}
$subscribers = $subscribers->whereNull(Subscriber::$_table . '.deleted_at');
return $subscribers->findArray();
$subscribers = $subscribers
->whereNull(Subscriber::$_table . '.deleted_at')
->limit(sprintf('%d, %d', $offset, $limit))
->findArray();
return $subscribers;
}
function getExportFileURL($file) {
@ -216,9 +266,4 @@ class Export {
return $subscriber[$field];
}, $this->subscriber_fields);
}
function timeExecution() {
$profiler_end = microtime(true);
return ($profiler_end - $this->profiler_start) / 60;
}
}

30
tests/unit/Subscribers/ImportExport/Export/ExportCest.php
View File

@ -122,6 +122,15 @@ class ExportCest {
'1'
)
);
expect($this->export->subscriber_custom_fields)
->equals($this->export->getSubscriberCustomFields());
expect($this->export->formatted_subscriber_fields)
->equals(
$this->export->formatSubscriberFields(
$this->export->subscriber_fields,
$this->export->subscriber_custom_fields
)
);
expect(
preg_match(
'|' .
@ -137,6 +146,7 @@ class ExportCest {
'|'
, $this->export->export_file_URL)
)->equals(1);
expect($this->export->subscriber_batch_size)->notNull();
}
function itCanGetSubscriberCustomFields() {
@ -156,7 +166,7 @@ class ExportCest {
}
function itProperlyReturnsSubscriberCustomFields() {
$subscribers = $this->export->getSubscribers();
$subscribers = $this->export->getSubscribers(0, 10);
foreach($subscribers as $subscriber) {
if($subscriber['email'] === $this->subscribers_data[1]) {
expect($subscriber['Country'])
@ -167,37 +177,37 @@ class ExportCest {
function itCanGetSubscribers() {
$this->export->segments = array(1);
$subscribers = $this->export->getSubscribers();
$subscribers = $this->export->getSubscribers(0, 10);
expect(count($subscribers))->equals(2);
$this->export->segments = array(2);
$subscribers = $this->export->getSubscribers();
$subscribers = $this->export->getSubscribers(0, 10);
expect(count($subscribers))->equals(2);
$this->export->segments = array(
1,
2
);
$subscribers = $this->export->getSubscribers();
$subscribers = $this->export->getSubscribers(0, 10);
expect(count($subscribers))->equals(3);
}
function itCanGroupSubscribersBySegments() {
$this->export->group_by_segment_option = true;
$this->export->subscribers_without_segment = true;
$subscribers = $this->export->getSubscribers();
$subscribers = $this->export->getSubscribers(0, 10);
expect(count($subscribers))->equals(5);
}
function itCanGetSubscribersOnlyWithoutSegments() {
$this->export->segments = array(0);
$this->export->subscribers_without_segment = true;
$subscribers = $this->export->getSubscribers();
$subscribers = $this->export->getSubscribers(0, 10);
expect(count($subscribers))->equals(1);
expect($subscribers[0]['segment_name'])->equals('Not In Segment');
}
function itCanGetOnlyConfirmedSubscribers() {
$this->export->export_confirmed_option = true;
$subscribers = $this->export->getSubscribers();
$subscribers = $this->export->getSubscribers(0, 10);
expect(count($subscribers))->equals(1);
expect($subscribers[0]['email'])
->equals($this->subscribers_data[1]['email']);
@ -207,7 +217,7 @@ class ExportCest {
SubscriberSegment::where('subscriber_id', 3)
->findOne()
->delete();
$subscribers = $this->export->getSubscribers();
$subscribers = $this->export->getSubscribers(0, 10);
expect(count($subscribers))->equals(2);
}
@ -223,8 +233,8 @@ class ExportCest {
$this->export->export_format_option = 'csv';
$this->export->process();
$CSV_file_size = filesize($this->export->export_file);
$this->export->export_file = $this->export->getExportFile('xls');
$this->export->export_format_option = 'xls';
$this->export->export_file = $this->export->getExportFile('xlsx');
$this->export->export_format_option = 'xlsx';
$this->export->process();
$XLS_file_size = filesize($this->export->export_file);
expect($CSV_file_size)->greaterThan(0);