Cavemanon/piratepoet
7
Fork 0
Code Pull Requests Actions Releases 16 Activity

Merge pull request #359 from mailpoet/import_batch_processing

Browse Source 
Import update
This commit is contained in:
Tautvidas Sipavičius
2016-02-23 13:04:48 +02:00
parent 3e7d1690bd 24b98a1154
commit a883e1176c
8 changed files with 305 additions and 5160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
assets/js/src/subscribers/importExport/import.js
View File

@ -6,7 +6,8 @@ define(
'mailpoet', 'mailpoet',
'handlebars', 'handlebars',
'papaparse', 'papaparse',
'select2' 'select2',
'asyncqueue'
], ],
function ( function (
Backbone, Backbone,
@ -14,12 +15,14 @@ define(
jQuery, jQuery,
MailPoet, MailPoet,
Handlebars, Handlebars,
Papa Papa,
AsyncQueue
) { ) {
if (!jQuery('#mailpoet_subscribers_import').length) { if (!jQuery('#mailpoet_subscribers_import').length) {
return; return;
} }
jQuery(document).ready(function () { jQuery(document).ready(function () {
var noticeTimeout = 3000;
jQuery('input[name="select_method"]').attr('checked', false); jQuery('input[name="select_method"]').attr('checked', false);
// configure router // configure router
router = new (Backbone.Router.extend({ router = new (Backbone.Router.extend({
@ -124,7 +127,7 @@ define(
var pasteSize = encodeURI(pasteInputElement.val()).split(/%..|./).length - 1; var pasteSize = encodeURI(pasteInputElement.val()).split(/%..|./).length - 1;
if (pasteSize > maxPostSizeBytes) { if (pasteSize > maxPostSizeBytes) {
MailPoet.Notice.error(MailPoetI18n.maxPostSizeNotice, { MailPoet.Notice.error(MailPoetI18n.maxPostSizeNotice, {
timeout: 3000, timeout: noticeTimeout,
}); });
return; return;
} }
@ -144,7 +147,7 @@ define(
if (ext === null || ext[1].toLowerCase() !== 'csv') { if (ext === null || ext[1].toLowerCase() !== 'csv') {
this.value = ''; this.value = '';
MailPoet.Notice.error(MailPoetI18n.wrongFileFormat, { MailPoet.Notice.error(MailPoetI18n.wrongFileFormat, {
timeout: 3000, timeout: noticeTimeout,
}); });
} }
@ -194,7 +197,7 @@ define(
if (response.result === false) { if (response.result === false) {
MailPoet.Notice.hide(); MailPoet.Notice.hide();
MailPoet.Notice.error(response.errors, { MailPoet.Notice.error(response.errors, {
timeout: 3000, timeout: noticeTimeout,
}); });
jQuery('.mailpoet_mailchimp-key-status') jQuery('.mailpoet_mailchimp-key-status')
.removeClass() .removeClass()
@ -219,7 +222,7 @@ define(
MailPoet.Modal.loading(false); MailPoet.Modal.loading(false);
MailPoet.Notice.error( MailPoet.Notice.error(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', { MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', {
timeout: 3000, timeout: noticeTimeout,
} }
); );
}); });
@ -246,7 +249,7 @@ define(
else { else {
MailPoet.Notice.hide(); MailPoet.Notice.hide();
MailPoet.Notice.error(response.errors, { MailPoet.Notice.error(response.errors, {
timeout: 3000, timeout: noticeTimeout,
}); });
} }
MailPoet.Modal.loading(false); MailPoet.Modal.loading(false);
@ -254,7 +257,7 @@ define(
MailPoet.Modal.loading(false); MailPoet.Modal.loading(false);
MailPoet.Notice.error( MailPoet.Notice.error(
MailPoetI18n.serverError + result.statusText.toLowerCase() + '.', { MailPoetI18n.serverError + result.statusText.toLowerCase() + '.', {
timeout: 3000, timeout: noticeTimeout,
} }
); );
}); });
@ -346,7 +349,7 @@ define(
error: function () { error: function () {
MailPoet.Notice.hide(); MailPoet.Notice.hide();
MailPoet.Notice.error(MailPoetI18n.dataProcessingError, { MailPoet.Notice.error(MailPoetI18n.dataProcessingError, {
timeout: 3000, timeout: noticeTimeout,
}); });
}, },
complete: function (CSV) { complete: function (CSV) {
@ -430,7 +433,7 @@ define(
errorNotice = errorNotice.replace('[link]', MailPoetI18n.csvKBLink); errorNotice = errorNotice.replace('[link]', MailPoetI18n.csvKBLink);
errorNotice = errorNotice.replace('[/link]', '</a>'); errorNotice = errorNotice.replace('[/link]', '</a>');
MailPoet.Notice.error(errorNotice, { MailPoet.Notice.error(errorNotice, {
timeout: 3000, timeout: noticeTimeout,
}); });
} }
} }
@ -561,7 +564,7 @@ define(
if (!segmentSelectionNotice.length) { if (!segmentSelectionNotice.length) {
MailPoet.Notice.error(MailPoetI18n.segmentSelectionRequired, { MailPoet.Notice.error(MailPoetI18n.segmentSelectionRequired, {
static: true, static: true,
timeout: 3000, timeout: noticeTimeout,
scroll: true, scroll: true,
id: 'segmentSelection', id: 'segmentSelection',
hideClose: true hideClose: true
@ -640,7 +643,7 @@ define(
MailPoet.Modal.close(); MailPoet.Modal.close();
MailPoet.Notice.error( MailPoet.Notice.error(
MailPoetI18n.segmentCreateError + response.message + '.', { MailPoetI18n.segmentCreateError + response.message + '.', {
timeout: 3000, timeout: noticeTimeout,
} }
); );
} }
@ -649,7 +652,7 @@ define(
MailPoet.Modal.close(); MailPoet.Modal.close();
MailPoet.Notice.error( MailPoet.Notice.error(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', { MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', {
timeout: 3000 timeout: noticeTimeout
} }
); );
}); });
@ -856,7 +859,7 @@ define(
} }
else { else {
MailPoet.Notice.error(MailPoetI18n.customFieldCreateError, { MailPoet.Notice.error(MailPoetI18n.customFieldCreateError, {
timeout: 3000, timeout: noticeTimeout,
}); });
} }
MailPoet.Modal.loading(false); MailPoet.Modal.loading(false);
@ -865,7 +868,7 @@ define(
MailPoet.Modal.loading(false); MailPoet.Modal.loading(false);
MailPoet.Notice.error( MailPoet.Notice.error(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', { MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', {
timeout: 3000, timeout: noticeTimeout,
} }
); );
}); });
@ -930,7 +933,7 @@ define(
if (!jQuery('[data-id="notice_invalidEmail"]').length) { if (!jQuery('[data-id="notice_invalidEmail"]').length) {
MailPoet.Notice.error(MailPoetI18n.columnContainsInvalidElement, { MailPoet.Notice.error(MailPoetI18n.columnContainsInvalidElement, {
static: true, static: true,
timeout: 3000, timeout: noticeTimeout,
scroll: true, scroll: true,
hideClose: true, hideClose: true,
id: 'invalidEmail' id: 'invalidEmail'
@ -1010,7 +1013,7 @@ define(
if (preventNextStep && !jQuery('.mailpoet_invalidDate').length) { if (preventNextStep && !jQuery('.mailpoet_invalidDate').length) {
MailPoet.Notice.error(MailPoetI18n.columnContainsInvalidDate, { MailPoet.Notice.error(MailPoetI18n.columnContainsInvalidDate, {
static: true, static: true,
timeout: 3000, timeout: noticeTimeout,
scroll: true, scroll: true,
hideClose: true, hideClose: true,
id: 'invalidDate' id: 'invalidDate'
@ -1050,64 +1053,98 @@ define(
} }
MailPoet.Modal.loading(true); MailPoet.Modal.loading(true);
var subscribers = {}; var columns = {},
queue = new jQuery.AsyncQueue(),
batchNumber = 0,
batchSize = 500,
timestamp = Date.now() / 1000,
subscribers = [],
importResults = {
'created': 0,
'updated': 0,
'errors': [],
'segments': []
},
splitSubscribers = function (subscribers, size) {
return subscribers.reduce(function (res, item, index) {
if (index % size === 0) {
res.push([]);
}
res[res.length - 1].push(item);
return res;
}, []);
},
subscribers = splitSubscribers(importData.step1.subscribers, batchSize);
_.each(jQuery('select.mailpoet_subscribers_column_data_match'), _.each(jQuery('select.mailpoet_subscribers_column_data_match'),
function (column, index) { function (column, columnIndex) {
var columnId = jQuery(column).data('column-id'); var columnId = jQuery(column).data('column-id');
if (columnId === 'ignore') { if (columnId === 'ignore') {
return; return;
} }
subscribers[columnId] = []; columns[columnId] = columnIndex;
_.each(importData.step1.subscribers, function (subsciber) { });
subscribers[columnId].push(
_.chain(subsciber)
.pick(index)
.toArray()
.flatten()
.value()
);
});
subscribers[columnId] = _.flatten(subscribers[columnId]);
});
MailPoet.Ajax.post({ _.each(subscribers, function () {
endpoint: 'ImportExport', queue.add(function (queue) {
action: 'processImport', queue.pause();
data: JSON.stringify({ MailPoet.Ajax
subscribers: subscribers, .post({
segments: segmentSelectElement.val(), endpoint: 'ImportExport',
updateSubscribers: (jQuery(':radio[name="subscriber_update_option"]:checked').val() === 'yes') ? true : false action: 'processImport',
data: JSON.stringify({
columns: columns,
subscribers: subscribers[batchNumber],
timestamp: timestamp,
segments: segmentSelectElement.val(),
updateSubscribers: (jQuery(':radio[name="subscriber_update_option"]:checked').val() === 'yes') ? true : false
})
})
.done(function (response) {
if (response.result === false) {
importResults.errors.push(response.errors);
} else {
importResults.created = response.data.created;
importResults.updated = response.data.updated;
importResults.segments = response.data.segments;
}
queue.run();
})
.error(function (error) {
importResults.errors.push(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.'
);
queue.run();
});
batchNumber++;
}) })
}).done(function (response) { });
queue.run();
queue.onComplete(function () {
MailPoet.Modal.loading(false); MailPoet.Modal.loading(false);
if (response.result === false) { if (importResults.errors.length > 0 && !importResults.updated && !importResults.created) {
MailPoet.Notice.error(response.errors, { MailPoet.Notice.error(_.flatten(importResults.errors), {
timeout: 3000, timeout: noticeTimeout,
}); }
} else { );
mailpoetSegments = response.data.segments; }
response.data.segments = _.map(segmentSelectElement.select2('data'), else {
function (data) { mailpoetSegments = importResults.segments;
return data.name; importResults.segments = _.map(segmentSelectElement.select2('data'),
}); function (data) {
importData.step2 = response.data; return data.name;
});
importData.step2 = importResults;
enableSegmentSelection(mailpoetSegments); enableSegmentSelection(mailpoetSegments);
router.navigate('step3', {trigger: true}); router.navigate('step3', {trigger: true});
} }
}).error(function (error) {
MailPoet.Modal.loading(false);
MailPoet.Notice.error(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', {
timeout: 3000,
}
);
}); });
}); });
filterSubscribers(); filterSubscribers();
enableSegmentSelection(mailpoetSegments); enableSegmentSelection(mailpoetSegments);
}); });
router.on('route:step3', function () { router.on('route:step3', function () {
@ -1118,6 +1155,12 @@ define(
showCurrentStep(); showCurrentStep();
if (importData.step2.errors.length > 0) {
MailPoet.Notice.error(_.flatten(importData.step2.errors), {
timeout: noticeTimeout,
});
}
// display statistics // display statistics
var subscribersDataImportResultsTemplate = var subscribersDataImportResultsTemplate =
Handlebars Handlebars

79
assets/js/src/vendor/jquery.asyncqueue.js vendored Normal file
View File

@ -0,0 +1,79 @@
/*
* This file is part of the jquery plugin "asyncQueue".
*
* (c) Sebastien Roch <roch.sebastien@gmail.com>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
(function($){
$.AsyncQueue = function() {
var that = this,
queue = [],
failureFunc,
completeFunc,
paused = false,
lastCallbackData,
_run;
_run = function() {
var f = queue.shift();
if (f) {
f.apply(that, [that]);
if (paused === false) {
_run();
}
} else {
if(completeFunc){
completeFunc.apply(that);
}
}
}
this.onFailure = function(func) {
failureFunc = func;
}
this.onComplete = function(func) {
completeFunc = func;
}
this.add = function(func) {
queue.push(func);
return this;
}
this.storeData = function(dataObject) {
lastCallbackData = dataObject;
return this;
}
this.lastCallbackData = function () {
return lastCallbackData;
}
this.run = function() {
paused = false;
_run();
}
this.pause = function () {
paused = true;
return this;
}
this.failure = function() {
paused = true;
if (failureFunc) {
var args = [that];
for(i = 0; i < arguments.length; i++) {
args.push(arguments[i]);
}
failureFunc.apply(that, args);
}
}
return this;
}
})(jQuery);

4
lib/Models/Subscriber.php
View File

@ -509,7 +509,7 @@ class Subscriber extends Model {
); );
} }
static function updateMultiple($columns, $subscribers, $currentTime = false) { static function updateMultiple($columns, $subscribers, $updated_at = false) {
$ignoreColumnsOnUpdate = array( $ignoreColumnsOnUpdate = array(
'email', 'email',
'created_at' 'created_at'
@ -549,7 +549,7 @@ class Subscriber extends Model {
return self::rawExecute( return self::rawExecute(
'UPDATE `' . self::$_table . '` ' . 'UPDATE `' . self::$_table . '` ' .
'SET ' . implode(', ', $sql('statement')) . ' '. 'SET ' . implode(', ', $sql('statement')) . ' '.
(($currentTime) ? ', updated_at = "' . $currentTime . '" ' : '') . (($updated_at) ? ', updated_at = "' . $updated_at . '" ' : '') .
'WHERE email IN ' . 'WHERE email IN ' .
'(' . rtrim(str_repeat('?,', count($subscribers)), ',') . ')', '(' . rtrim(str_repeat('?,', count($subscribers)), ',') . ')',
array_merge( array_merge(

70
lib/Subscribers/ImportExport/Import/Import.php
View File

@ -14,21 +14,26 @@ class Import {
public $subscriber_fields; public $subscriber_fields;
public $subscriber_custom_fields; public $subscriber_custom_fields;
public $subscribers_count; public $subscribers_count;
public $import_time; public $created_at;
public $updated_at;
public $profiler_start; public $profiler_start;
public function __construct($data) { public function __construct($data) {
$this->subscribers_data = $data['subscribers']; $this->subscribers_data = $this->transformSubscribersData(
$data['subscribers'],
$data['columns']
);
$this->segments = $data['segments']; $this->segments = $data['segments'];
$this->update_subscribers = $data['updateSubscribers']; $this->update_subscribers = $data['updateSubscribers'];
$this->subscriber_fields = $this->getSubscriberFields( $this->subscriber_fields = $this->getSubscriberFields(
array_keys($this->subscribers_data) array_keys($data['columns'])
); );
$this->subscriber_custom_fields = $this->getCustomSubscriberFields( $this->subscriber_custom_fields = $this->getCustomSubscriberFields(
array_keys($this->subscribers_data) array_keys($data['columns'])
); );
$this->subscribers_count = count(reset($this->subscribers_data)); $this->subscribers_count = count(reset($this->subscribers_data));
$this->import_time = date('Y-m-d H:i:s'); $this->created_at = date('Y-m-d H:i:s', (int) $data['timestamp']);
$this->updated_at = date('Y-m-d H:i:s', (int) $data['timestamp'] + 1);
$this->profiler_start = microtime(true); $this->profiler_start = microtime(true);
} }
@ -63,14 +68,6 @@ class Import {
$subscriber_fields, $subscriber_fields,
$subscriber_custom_fields $subscriber_custom_fields
); );
if($created_subscribers) {
// subtract added from updated subscribers when DB operation takes <1s
$updated_subscribers = array_diff_key(
$updated_subscribers,
$created_subscribers,
$subscriber_custom_fields
);
}
} }
} catch(\PDOException $e) { } catch(\PDOException $e) {
return array( return array(
@ -90,14 +87,22 @@ class Import {
); );
} }
function transformSubscribersData($subscribers, $columns) {
foreach($columns as $column => $index) {
$transformed_subscribers[$column] = Helpers::arrayColumn($subscribers, $index);
}
return $transformed_subscribers;
}
function filterExistingAndNewSubscribers($subscribers_data) { function filterExistingAndNewSubscribers($subscribers_data) {
$chunk_size = 200;
$existing_records = array_filter( $existing_records = array_filter(
array_map(function($subscriber_emails) { array_map(function($subscriber_emails) {
return Subscriber::selectMany(array('email')) return Subscriber::selectMany(array('email'))
->whereIn('email', $subscriber_emails) ->whereIn('email', $subscriber_emails)
->whereNull('deleted_at') ->whereNull('deleted_at')
->findArray(); ->findArray();
}, array_chunk($subscribers_data['email'], 200)) }, array_chunk($subscribers_data['email'], $chunk_size))
); );
if(!$existing_records) { if(!$existing_records) {
return array( return array(
@ -144,17 +149,19 @@ class Import {
} }
function deleteExistingTrashedSubscribers($subscribers_data) { function deleteExistingTrashedSubscribers($subscribers_data) {
$chunk_size = 200;
$existing_trashed_records = array_filter( $existing_trashed_records = array_filter(
array_map(function($subscriber_emails) { array_map(function($subscriber_emails) {
return Subscriber::selectMany(array('id')) return Subscriber::selectMany(array('id'))
->whereIn('email', $subscriber_emails) ->whereIn('email', $subscriber_emails)
->whereNotNull('deleted_at') ->whereNotNull('deleted_at')
->findArray(); ->findArray();
}, array_chunk($subscribers_data['email'], 200)) }, array_chunk($subscribers_data['email'], $chunk_size))
); );
if(!$existing_trashed_records) return; if(!$existing_trashed_records) return;
$existing_trashed_records = Helpers::flattenArray($existing_trashed_records); $existing_trashed_records = Helpers::flattenArray($existing_trashed_records);
foreach(array_chunk($existing_trashed_records, 200) as $subscriber_ids) { foreach(array_chunk($existing_trashed_records, $chunk_size) as
$subscriber_ids) {
Subscriber::whereIn('id', $subscriber_ids) Subscriber::whereIn('id', $subscriber_ids)
->deleteMany(); ->deleteMany();
SubscriberSegment::whereIn('subscriber_id', $subscriber_ids) SubscriberSegment::whereIn('subscriber_id', $subscriber_ids)
@ -163,7 +170,8 @@ class Import {
} }
function extendSubscribersAndFields($subscribers_data, $subscriber_fields) { function extendSubscribersAndFields($subscribers_data, $subscriber_fields) {
$subscribers_data['created_at'] = $this->filterSubscriberCreatedAtDate(); $subscribers_data['created_at'] =
array_fill(0, $this->subscribers_count, $this->created_at);
$subscriber_fields[] = 'created_at'; $subscriber_fields[] = 'created_at';
return array( return array(
$subscribers_data, $subscribers_data,
@ -191,10 +199,6 @@ class Import {
); );
} }
function filterSubscriberCreatedAtDate() {
return array_fill(0, $this->subscribers_count, $this->import_time);
}
function filterSubscriberStatus($subscribers_data, $subscriber_fields) { function filterSubscriberStatus($subscribers_data, $subscriber_fields) {
if(!in_array('status', $subscriber_fields)) { if(!in_array('status', $subscriber_fields)) {
$subscribers_data['status'] = $subscribers_data['status'] =
@ -249,14 +253,14 @@ class Import {
$subscriber_fields, $subscriber_fields,
$subscriber_custom_fields $subscriber_custom_fields
) { ) {
$chunk_size = 100;
$subscribers_count = count(reset($subscribers_data)) - 1; $subscribers_count = count(reset($subscribers_data)) - 1;
$subscribers = array_map(function($index) use ($subscribers_data, $subscriber_fields) { $subscribers = array_map(function($index) use ($subscribers_data, $subscriber_fields) {
return array_map(function($field) use ($index, $subscribers_data) { return array_map(function($field) use ($index, $subscribers_data) {
return $subscribers_data[$field][$index]; return $subscribers_data[$field][$index];
}, $subscriber_fields); }, $subscriber_fields);
}, range(0, $subscribers_count)); }, range(0, $subscribers_count));
$import_time = ($action === 'update') ? date('Y-m-d H:i:s') : $this->import_time; foreach(array_chunk($subscribers, $chunk_size) as $data) {
foreach(array_chunk($subscribers, 100) as $data) {
if($action == 'create') { if($action == 'create') {
Subscriber::createMultiple( Subscriber::createMultiple(
$subscriber_fields, $subscriber_fields,
@ -267,18 +271,20 @@ class Import {
Subscriber::updateMultiple( Subscriber::updateMultiple(
$subscriber_fields, $subscriber_fields,
$data, $data,
$import_time $this->updated_at
); );
} }
} }
$result = Helpers::arrayColumn( // return id=>email array of results $query = Subscriber::selectMany(
Subscriber::selectMany( array(
array( 'id',
'id', 'email'
'email' ));
)) $query = ($action === 'update') ?
->where(($action === 'create') ? 'created_at' : 'updated_at', $import_time) $query->where('updated_at', $this->updated_at) :
->findArray(), $query->where('created_at', $this->created_at);
$result = Helpers::arrayColumn(
$query->findArray(),
'email', 'id' 'email', 'id'
); );
if($subscriber_custom_fields) { if($subscriber_custom_fields) {

135
tests/unit/Subscribers/ImportExport/Import/ImportCest.php
View File

@ -7,25 +7,33 @@ use MailPoet\Subscribers\ImportExport\Import\Import;
use MailPoet\Util\Helpers; use MailPoet\Util\Helpers;
class ImportCest { class ImportCest {
function __construct() { function _before() {
$this->JSON_data = json_decode(file_get_contents(dirname(__FILE__) . '/ImportTestData.json'), true); $this->data = array(
$this->subscribers_data = array( 'subscribers' => array(
'first_name' => array( array(
'Adam', 'Adam',
'Mary' 'Smith',
'adam@smith.com',
'France'
),
array(
'Mary',
'Jane',
'mary@jane.com',
'Brazil'
)
), ),
'last_name' => array( 'columns' => array(
'Smith', 'first_name' => 0,
'Jane' 'last_name' => 1,
'email' => 2,
777 => 3
), ),
'email' => array( 'segments' => array(
'adam@smith.com', 195
'mary@jane.com'
), ),
777 => array( 'timestamp' => time(),
'France', 'updateSubscribers' => true
'Brazil'
)
); );
$this->subscriber_fields = array( $this->subscriber_fields = array(
'first_name', 'first_name',
@ -34,25 +42,36 @@ class ImportCest {
); );
$this->segments = range(0, 1); $this->segments = range(0, 1);
$this->subscriber_custom_fields = array(777); $this->subscriber_custom_fields = array(777);
$this->import = new Import($this->JSON_data); $this->import = new Import($this->data);
$this->subscribers_data = $this->import->transformSubscribersData(
$this->data['subscribers'],
$this->data['columns']
);
} }
function itCanConstruct() { function itCanConstruct() {
expect($this->import->subscribers_data)->equals($this->JSON_data['subscribers']); expect(is_array($this->import->subscribers_data))->true();
expect($this->import->segments)->equals($this->JSON_data['segments']); expect($this->import->segments)->equals($this->data['segments']);
expect(is_array($this->import->subscriber_fields))->true(); expect(is_array($this->import->subscriber_fields))->true();
expect(is_array($this->import->subscriber_custom_fields))->true(); expect(is_array($this->import->subscriber_custom_fields))->true();
expect($this->import->subscribers_count)->equals( expect($this->import->subscribers_count)->equals(2);
count($this->JSON_data['subscribers']['email']) expect($this->import->created_at)->notEmpty();
); expect($this->import->updated_at)->notEmpty();
expect( }
preg_match(
'/\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}/', function itCanTransformSubscribers() {
$this->import->import_time) expect($this->import->subscribers_data['first_name'][0])
)->equals(1); ->equals($this->data['subscribers'][0][0]);
expect($this->import->subscribers_data['last_name'][0])
->equals($this->data['subscribers'][0][1]);
expect($this->import->subscribers_data['email'][0])
->equals($this->data['subscribers'][0][2]);
expect($this->import->subscribers_data['777'][0])
->equals($this->data['subscribers'][0][3]);
} }
function itCanFilterExistingAndNewSubscribers() { function itCanFilterExistingAndNewSubscribers() {
$subscribers_data = $this->subscribers_data;
$subscriber = Subscriber::create(); $subscriber = Subscriber::create();
$subscriber->hydrate( $subscriber->hydrate(
array( array(
@ -62,10 +81,10 @@ class ImportCest {
)); ));
$subscriber->save(); $subscriber->save();
list($existing, $new) = $this->import->filterExistingAndNewSubscribers( list($existing, $new) = $this->import->filterExistingAndNewSubscribers(
$this->subscribers_data $subscribers_data
); );
expect($existing['email'][0])->equals($this->subscribers_data['email'][0]); expect($existing['email'][0])->equals($subscribers_data['email'][0]);
expect($new['email'][0])->equals($this->subscribers_data['email'][1]); expect($new['email'][0])->equals($subscribers_data['email'][1]);
} }
function itCanExtendSubscribersAndFields() { function itCanExtendSubscribersAndFields() {
@ -106,17 +125,17 @@ class ImportCest {
} }
function itCanFilterSubscriberStatus() { function itCanFilterSubscriberStatus() {
$subscibers_data = $this->subscribers_data; $subscribers_data = $this->subscribers_data;
$subscriber_fields = $this->subscriber_fields; $subscriber_fields = $this->subscriber_fields;
list($subscibers_data, $subsciber_fields) = list($subscribers_data, $subsciber_fields) =
$this->import->filterSubscriberStatus($subscibers_data, $subscriber_fields); $this->import->filterSubscriberStatus($subscribers_data, $subscriber_fields);
// subscribers' status was set to "subscribed" & status column was added // subscribers' status was set to "subscribed" & status column was added
// to subscribers fields // to subscribers fields
expect(array_pop($subsciber_fields))->equals('status'); expect(array_pop($subsciber_fields))->equals('status');
expect($subscibers_data['status'][0])->equals('subscribed'); expect($subscribers_data['status'][0])->equals('subscribed');
expect(count($subscibers_data['status']))->equals(2); expect(count($subscribers_data['status']))->equals(2);
$subscriber_fields[] = 'status'; $subscriber_fields[] = 'status';
$subscibers_data = array( $subscribers_data = array(
'status' => array( 'status' => array(
#subscribed #subscribed
'subscribed', 'subscribed',
@ -135,9 +154,9 @@ class ImportCest {
'false' 'false'
), ),
); );
list($subscibers_data, $subsciber_fields) = list($subscribers_data, $subsciber_fields) =
$this->import->filterSubscriberStatus($subscibers_data, $subscriber_fields); $this->import->filterSubscriberStatus($subscribers_data, $subscriber_fields);
expect($subscibers_data)->equals( expect($subscribers_data)->equals(
array( array(
'status' => array( 'status' => array(
'subscribed', 'subscribed',
@ -185,8 +204,8 @@ class ImportCest {
$subscribers_data = $this->subscribers_data; $subscribers_data = $this->subscribers_data;
$subscriber_fields = $this->subscriber_fields; $subscriber_fields = $this->subscriber_fields;
$subscribers_data['deleted_at'] = array( $subscribers_data['deleted_at'] = array(
null, null,
date('Y-m-d H:i:s') date('Y-m-d H:i:s')
); );
$subscriber_fields[] = 'deleted_at'; $subscriber_fields[] = 'deleted_at';
$this->import->createOrUpdateSubscribers( $this->import->createOrUpdateSubscribers(
@ -255,6 +274,7 @@ class ImportCest {
->equals($subscribers_data[777][1]); ->equals($subscribers_data[777][1]);
} }
function itCanaddSubscribersToSegments() { function itCanaddSubscribersToSegments() {
$subscribers_data = $this->subscribers_data; $subscribers_data = $this->subscribers_data;
$this->import->createOrUpdateSubscribers( $this->import->createOrUpdateSubscribers(
@ -293,26 +313,27 @@ class ImportCest {
); );
} }
function itCanProcess() { function itCanUpdateSubscribers() {
$import = clone($this->import); $result = $this->import->process();
$result = $import->process();
expect($result['data']['created'])->equals(997);
expect($result['data']['updated'])->equals(0); expect($result['data']['updated'])->equals(0);
$result = $import->process(); $result = $this->import->process();
expect($result['data']['created'])->equals(0); expect($result['data']['updated'])->equals(2);
expect($result['data']['updated'])->equals(997); $this->import->update_subscribers = false;
Subscriber::where('email', 'mbanks4@blinklist.com') $result = $this->import->process();
expect($result['data']['updated'])->equals(0);
}
function itCanProcess() {
$result = $this->import->process();
expect($result['data']['created'])->equals(2);
Subscriber::where('email', 'mary@jane.com')
->findOne() ->findOne()
->delete(); ->delete();
// TODO: find a more elegant way to test this $timestamp = time() + 1;
$import->import_time = date('Y-m-d 12:i:s'); $this->import->created_at = date('Y-m-d H:i:s', $timestamp);
$result = $import->process(); $this->import->updated_at = date('Y-m-d H:i:s', $timestamp + 1);
$result = $this->import->process();
expect($result['data']['created'])->equals(1); expect($result['data']['created'])->equals(1);
expect($result['data']['updated'])->equals(996);
$import->update_subscribers = false;
$result = $import->process();
expect($result['data']['created'])->equals(0);
expect($result['data']['updated'])->equals(0);
} }
function _after() { function _after() {

5003
tests/unit/Subscribers/ImportExport/Import/ImportTestData.json
View File

File diff suppressed because it is too large Load Diff

2
views/subscribers/importExport/import/step1.html
View File

@ -60,8 +60,6 @@
</th> </th>
<td> <td>
<input type="file" id="file_local" accept=".csv" /> <input type="file" id="file_local" accept=".csv" />
&nbsp;
<%= __('total max upload file size : %s')|format(maxPostSize) %>
</td> </td>
</tr> </tr>
</tbody> </tbody>

5
webpack.config.js
View File

@ -17,7 +17,7 @@ baseConfig = {
resolve: { resolve: {
modulesDirectories: [ modulesDirectories: [
'node_modules', 'node_modules',
'assets/js/src' 'assets/js/src',
], ],
alias: { alias: {
'handlebars': 'handlebars/dist/handlebars.js', 'handlebars': 'handlebars/dist/handlebars.js',
@ -30,7 +30,8 @@ baseConfig = {
'filesaver$': 'filesaver/FileSaver.js', 'filesaver$': 'filesaver/FileSaver.js',
'papaparse': 'papaparse/papaparse.min.js', 'papaparse': 'papaparse/papaparse.min.js',
'helpscout': 'helpscout.js', 'helpscout': 'helpscout.js',
'html2canvas': 'html2canvas/dist/html2canvas.js' 'html2canvas': 'html2canvas/dist/html2canvas.js',
'asyncqueue': 'vendor/jquery.asyncqueue.js'
}, },
}, },
node: { node: {