Cavemanon/piratepoet
8
Fork 0
Code Pull Requests Actions Releases 16 Activity

moving subscribers fetching logic into a separated class.

Browse Source
This commit is contained in:
Amine Ben hammou
2018-03-08 16:50:41 +01:00
parent cca4fc02c0
commit b9c31698d5
3 changed files with 164 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
lib/Subscribers/ImportExport/Export/DefaultSubscribersGetter.php Normal file
View File

@@ -0,0 +1,129 @@
<?php
namespace MailPoet\Subscribers\ImportExport\Export;
use MailPoet\Models\Segment;
use MailPoet\Models\Subscriber;
use MailPoet\Models\SubscriberSegment;
/**
* Gets batches of subscribers from default segments.
*/
class DefaultSubscribersGetter {
protected $segments_ids;
protected $batch_size;
protected $offset;
protected $finished;
protected $get_subscribers_without_segment;
public function __construct($segments_ids, $batch_size) {
$this->get_subscribers_without_segment = (array_search(0, $segments_ids) !== false);
$this->segments_ids = $this->filterSegmentIds($segments_ids);
$this->batch_size = $batch_size;
$this->offset = 0;
$this->finished = false;
}
protected function filterSegmentIds($ids) {
$ids = array_map(function($data) {
return $data['id'];
}, Segment::select('id')
->whereIn('id', $ids)
->where('type', Segment::TYPE_DEFAULT)
->findArray()
);
if($this->get_subscribers_without_segment) {
$ids[] = 0;
}
return $ids;
}
/**
* Resets the `offset` and `finished` properties;
* to be able to start getting subscribers again.
*/
public function reset() {
$this->offset = 0;
$this->finished = false;
}
/**
* Gets the next batch of subscribers or `false` no more!
*/
public function get() {
if($this->finished) {
return false;
}
// define returned columns
$subscribers = Subscriber::selectMany(
'first_name',
'last_name',
'email',
'subscribed_ip',
array(
'global_status' => Subscriber::$_table . '.status'
),
array(
'list_status' => SubscriberSegment::$_table . '.status'
)
);
// JOIN subscribers on segment and subscriber_segment tables
$subscribers = $subscribers
->left_outer_join(
SubscriberSegment::$_table,
array(
Subscriber::$_table . '.id',
'=',
SubscriberSegment::$_table . '.subscriber_id'
)
)
->left_outer_join(
Segment::$_table,
array(
Segment::$_table . '.id',
'=',
SubscriberSegment::$_table . '.segment_id'
)
)
->filter('filterWithCustomFieldsForExport')
->groupBy(Subscriber::$_table . '.id')
->groupBy(Segment::$_table . '.id');
if($this->get_subscribers_without_segment !== false) {
// if there are subscribers who do not belong to any segment, use
// a CASE function to group them under "Not In Segment"
$subscribers = $subscribers
->selectExpr(
'MAX(CASE WHEN ' . Segment::$_table . '.name IS NOT NULL ' .
'THEN ' . Segment::$_table . '.name ' .
'ELSE "' . __('Not In Segment', 'mailpoet') . '" END) as segment_name'
)
->whereRaw(
SubscriberSegment::$_table . '.segment_id IN (' .
rtrim(str_repeat('?,', count($this->segments_ids)), ',') . ') ' .
'OR ' . SubscriberSegment::$_table . '.segment_id IS NULL ',
$this->segments_ids
);
} else {
// if all subscribers belong to at least one segment, select the segment name
$subscribers = $subscribers
->selectExpr('MAX(' . Segment::$_table . '.name) as segment_name')
->whereIn(SubscriberSegment::$_table . '.segment_id', $this->segments_ids);
}
$subscribers = $subscribers
->whereNull(Subscriber::$_table . '.deleted_at')
->offset($this->offset)
->limit($this->batch_size)
->findArray();
$this->offset += $this->batch_size;
if(count($subscribers) < $this->batch_size) {
$this->finished = true;
}
return $subscribers;
}
}

100
lib/Subscribers/ImportExport/Export/Export.php
View File

@@ -13,24 +13,28 @@ use MailPoet\Util\Security;
use MailPoet\Util\XLSXWriter;
class Export {
const SUBSCRIBER_BATCH_SIZE = 15000;
public $export_format_option;
public $segments;
public $subscribers_without_segment;
public $subscriber_fields;
public $subscriber_custom_fields;
public $formatted_subscriber_fields;
public $export_path;
public $export_file;
public $export_file_URL;
public $subscriber_batch_size;
public $default_subscribers_getter;
public function __construct($data) {
if(strpos(@ini_get('disable_functions'), 'set_time_limit') === false) {
set_time_limit(0);
}
$this->default_subscribers_getter = new DefaultSubscribersGetter(
$data['segments'],
self::SUBSCRIBER_BATCH_SIZE
);
$this->export_format_option = $data['export_format_option'];
$this->segments = $data['segments'];
$this->subscribers_without_segment = array_search(0, $this->segments);
$this->subscriber_fields = $data['subscriber_fields'];
$this->subscriber_custom_fields = $this->getSubscriberCustomFields();
$this->formatted_subscriber_fields = $this->formatSubscriberFields(
@@ -40,10 +44,10 @@ class Export {
$this->export_path = Env::$temp_path;
$this->export_file = $this->getExportFile($this->export_format_option);
$this->export_file_URL = $this->getExportFileURL($this->export_file);
$this->subscriber_batch_size = 15000;
}
function process() {
$this->default_subscribers_getter->reset();
try {
if(is_writable($this->export_path) === false) {
throw new \Exception(__('The export file could not be saved on the server.', 'mailpoet'));
@@ -68,7 +72,6 @@ class Export {
function generateCSV() {
$processed_subscribers = 0;
$offset = 0;
$formatted_subscriber_fields = $this->formatted_subscriber_fields;
$CSV_file = fopen($this->export_file, 'w');
$format_CSV = function($row) {
@@ -88,29 +91,29 @@ class Export {
)
) . PHP_EOL
);
do {
$subscribers = $this->getSubscribers($offset, $this->subscriber_batch_size);
$subscribers = $this->getSubscribers();
while($subscribers !== false) {
$processed_subscribers += count($subscribers);
foreach($subscribers as $subscriber) {
$row = $this->formatSubscriberData($subscriber);
$row[] = ucwords($subscriber['segment_name']);
fwrite($CSV_file, implode(',', array_map($format_CSV, $row)) . "\n");
}
$offset += $this->subscriber_batch_size;
} while(count($subscribers) === $this->subscriber_batch_size);
$subscribers = $this->getSubscribers();
}
fclose($CSV_file);
return $processed_subscribers;
}
function generateXLSX() {
$processed_subscribers = 0;
$offset = 0;
$XLSX_writer = new XLSXWriter();
$XLSX_writer->setAuthor('MailPoet (www.mailpoet.com)');
$last_segment = false;
$processed_segments = array();
do {
$subscribers = $this->getSubscribers($offset, $this->subscriber_batch_size);
$subscribers = $this->getSubscribers();
while($subscribers !== false) {
$processed_subscribers += count($subscribers);
foreach($subscribers as $i => $subscriber) {
$current_segment = ucwords($subscriber['segment_name']);
@@ -147,8 +150,8 @@ class Export {
$this->formatSubscriberData($subscriber)
);
}
$offset += $this->subscriber_batch_size;
} while(count($subscribers) === $this->subscriber_batch_size);
$subscribers = $this->getSubscribers();
}
$XLSX_writer->writeToFile($this->export_file);
return $processed_subscribers;
}
@@ -157,69 +160,8 @@ class Export {
return $XLSX_writer->writeSheetRow(ucwords($segment), $data);
}
function getSubscribers($offset, $limit) {
// define returned columns
$subscribers = Subscriber::selectMany(
'first_name',
'last_name',
'email',
'subscribed_ip',
array(
'global_status' => Subscriber::$_table . '.status'
),
array(
'list_status' => SubscriberSegment::$_table . '.status'
)
);
// JOIN subscribers on segment and subscriber_segment tables
$subscribers = $subscribers
->left_outer_join(
SubscriberSegment::$_table,
array(
Subscriber::$_table . '.id',
'=',
SubscriberSegment::$_table . '.subscriber_id'
)
)
->left_outer_join(
Segment::$_table,
array(
Segment::$_table . '.id',
'=',
SubscriberSegment::$_table . '.segment_id'
)
)
->filter('filterWithCustomFieldsForExport')
->groupBy(Subscriber::$_table . '.id')
->groupBy(Segment::$_table . '.id');
if($this->subscribers_without_segment !== false) {
// if there are subscribers who do not belong to any segment, use
// a CASE function to group them under "Not In Segment"
$subscribers = $subscribers
->selectExpr(
'MAX(CASE WHEN ' . Segment::$_table . '.name IS NOT NULL ' .
'THEN ' . Segment::$_table . '.name ' .
'ELSE "' . __('Not In Segment', 'mailpoet') . '" END) as segment_name'
)
->whereRaw(
SubscriberSegment::$_table . '.segment_id IN (' .
rtrim(str_repeat('?,', count($this->segments)), ',') . ') ' .
'OR ' . SubscriberSegment::$_table . '.segment_id IS NULL ',
$this->segments
);
} else {
// if all subscribers belong to at least one segment, select the segment name
$subscribers = $subscribers
->selectExpr('MAX(' . Segment::$_table . '.name) as segment_name')
->whereIn(SubscriberSegment::$_table . '.segment_id', $this->segments);
}
$subscribers = $subscribers
->whereNull(Subscriber::$_table . '.deleted_at')
->offset($offset)
->limit($limit)
->findArray();
function getSubscribers() {
$subscribers = $this->default_subscribers_getter->get();
return $subscribers;
}

38
tests/unit/Subscribers/ImportExport/Export/ExportTest.php
View File

@@ -9,6 +9,7 @@ use MailPoet\Models\Subscriber;
use MailPoet\Models\SubscriberCustomField;
use MailPoet\Models\SubscriberSegment;
use MailPoet\Subscribers\ImportExport\Export\Export;
use MailPoet\Subscribers\ImportExport\Export\DefaultSubscribersGetter;
class ExportTest extends \MailPoetTest {
function _before() {
@@ -103,15 +104,6 @@ class ExportTest extends \MailPoetTest {
function testItCanConstruct() {
expect($this->export->export_format_option)
->equals('csv');
expect($this->export->segments)
->equals(
array(
1,
2
)
);
expect($this->export->subscribers_without_segment)
->equals(0);
expect($this->export->subscriber_fields)
->equals(
array(
@@ -143,7 +135,6 @@ class ExportTest extends \MailPoetTest {
basename($this->export->export_file) .
'|', $this->export->export_file_URL)
)->equals(1);
expect($this->export->subscriber_batch_size)->notNull();
}
function testItCanGetSubscriberCustomFields() {
@@ -173,30 +164,29 @@ class ExportTest extends \MailPoetTest {
}
function testItCanGetSubscribers() {
$this->export->segments = array(1);
$subscribers = $this->export->getSubscribers(0, 10);
$this->export->default_subscribers_getter = new DefaultSubscribersGetter(array(1), 100);
$subscribers = $this->export->getSubscribers();
expect($subscribers)->count(2);
$this->export->segments = array(2);
$subscribers = $this->export->getSubscribers(0, 10);
$this->export->default_subscribers_getter = new DefaultSubscribersGetter(array(2), 100);
$subscribers = $this->export->getSubscribers();
expect($subscribers)->count(2);
$this->export->segments = array(
1,
2
);
$subscribers = $this->export->getSubscribers(0, 10);
$this->export->default_subscribers_getter = new DefaultSubscribersGetter(array(1, 2), 100);
$subscribers = $this->export->getSubscribers();
expect($subscribers)->count(4);
}
function testItAlwaysGroupsSubscribersBySegments() {
$this->export->subscribers_without_segment = true;
$subscribers = $this->export->getSubscribers(0, 10);
$this->export->default_subscribers_getter = new DefaultSubscribersGetter(array(0, 1, 2), 100);
$subscribers = $this->export->getSubscribers();
expect($subscribers)->count(5);
}
function testItCanGetSubscribersOnlyWithoutSegments() {
$this->export->segments = array(0);
$this->export->subscribers_without_segment = true;
$subscribers = $this->export->getSubscribers(0, 10);
$this->export->default_subscribers_getter = new DefaultSubscribersGetter(array(0), 100);
$subscribers = $this->export->getSubscribers();
expect($subscribers)->count(1);
expect($subscribers[0]['segment_name'])->equals('Not In Segment');
}