Cavemanon/piratepoet
7
Fork 0
Code Pull Requests Actions Releases 16 Activity

Merge pull request #359 from mailpoet/import_batch_processing

Browse Source 
Import update
This commit is contained in:
Tautvidas Sipavičius
2016-02-23 13:04:48 +02:00
parent 3e7d1690bd 24b98a1154
commit a883e1176c
8 changed files with 305 additions and 5160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
assets/js/src/subscribers/importExport/import.js
View File

@ -6,7 +6,8 @@ define(
'mailpoet',
'handlebars',
'papaparse',
'select2'
'select2',
'asyncqueue'
],
function (
Backbone,
@ -14,12 +15,14 @@ define(
jQuery,
MailPoet,
Handlebars,
Papa
Papa,
AsyncQueue
) {
if (!jQuery('#mailpoet_subscribers_import').length) {
return;
}
jQuery(document).ready(function () {
var noticeTimeout = 3000;
jQuery('input[name="select_method"]').attr('checked', false);
// configure router
router = new (Backbone.Router.extend({
@ -124,7 +127,7 @@ define(
var pasteSize = encodeURI(pasteInputElement.val()).split(/%..|./).length - 1;
if (pasteSize > maxPostSizeBytes) {
MailPoet.Notice.error(MailPoetI18n.maxPostSizeNotice, {
timeout: 3000,
timeout: noticeTimeout,
});
return;
}
@ -144,7 +147,7 @@ define(
if (ext === null || ext[1].toLowerCase() !== 'csv') {
this.value = '';
MailPoet.Notice.error(MailPoetI18n.wrongFileFormat, {
timeout: 3000,
timeout: noticeTimeout,
});
}
@ -194,7 +197,7 @@ define(
if (response.result === false) {
MailPoet.Notice.hide();
MailPoet.Notice.error(response.errors, {
timeout: 3000,
timeout: noticeTimeout,
});
jQuery('.mailpoet_mailchimp-key-status')
.removeClass()
@ -219,7 +222,7 @@ define(
MailPoet.Modal.loading(false);
MailPoet.Notice.error(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', {
timeout: 3000,
timeout: noticeTimeout,
}
);
});
@ -246,7 +249,7 @@ define(
else {
MailPoet.Notice.hide();
MailPoet.Notice.error(response.errors, {
timeout: 3000,
timeout: noticeTimeout,
});
}
MailPoet.Modal.loading(false);
@ -254,7 +257,7 @@ define(
MailPoet.Modal.loading(false);
MailPoet.Notice.error(
MailPoetI18n.serverError + result.statusText.toLowerCase() + '.', {
timeout: 3000,
timeout: noticeTimeout,
}
);
});
@ -346,7 +349,7 @@ define(
error: function () {
MailPoet.Notice.hide();
MailPoet.Notice.error(MailPoetI18n.dataProcessingError, {
timeout: 3000,
timeout: noticeTimeout,
});
},
complete: function (CSV) {
@ -430,7 +433,7 @@ define(
errorNotice = errorNotice.replace('[link]', MailPoetI18n.csvKBLink);
errorNotice = errorNotice.replace('[/link]', '</a>');
MailPoet.Notice.error(errorNotice, {
timeout: 3000,
timeout: noticeTimeout,
});
}
}
@ -561,7 +564,7 @@ define(
if (!segmentSelectionNotice.length) {
MailPoet.Notice.error(MailPoetI18n.segmentSelectionRequired, {
static: true,
timeout: 3000,
timeout: noticeTimeout,
scroll: true,
id: 'segmentSelection',
hideClose: true
@ -640,7 +643,7 @@ define(
MailPoet.Modal.close();
MailPoet.Notice.error(
MailPoetI18n.segmentCreateError + response.message + '.', {
timeout: 3000,
timeout: noticeTimeout,
}
);
}
@ -649,7 +652,7 @@ define(
MailPoet.Modal.close();
MailPoet.Notice.error(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', {
timeout: 3000
timeout: noticeTimeout
}
);
});
@ -856,7 +859,7 @@ define(
}
else {
MailPoet.Notice.error(MailPoetI18n.customFieldCreateError, {
timeout: 3000,
timeout: noticeTimeout,
});
}
MailPoet.Modal.loading(false);
@ -865,7 +868,7 @@ define(
MailPoet.Modal.loading(false);
MailPoet.Notice.error(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', {
timeout: 3000,
timeout: noticeTimeout,
}
);
});
@ -930,7 +933,7 @@ define(
if (!jQuery('[data-id="notice_invalidEmail"]').length) {
MailPoet.Notice.error(MailPoetI18n.columnContainsInvalidElement, {
static: true,
timeout: 3000,
timeout: noticeTimeout,
scroll: true,
hideClose: true,
id: 'invalidEmail'
@ -1010,7 +1013,7 @@ define(
if (preventNextStep && !jQuery('.mailpoet_invalidDate').length) {
MailPoet.Notice.error(MailPoetI18n.columnContainsInvalidDate, {
static: true,
timeout: 3000,
timeout: noticeTimeout,
scroll: true,
hideClose: true,
id: 'invalidDate'
@ -1050,64 +1053,98 @@ define(
}
MailPoet.Modal.loading(true);
var subscribers = {};
var columns = {},
queue = new jQuery.AsyncQueue(),
batchNumber = 0,
batchSize = 500,
timestamp = Date.now() / 1000,
subscribers = [],
importResults = {
'created': 0,
'updated': 0,
'errors': [],
'segments': []
},
splitSubscribers = function (subscribers, size) {
return subscribers.reduce(function (res, item, index) {
if (index % size === 0) {
res.push([]);
}
res[res.length - 1].push(item);
return res;
}, []);
},
subscribers = splitSubscribers(importData.step1.subscribers, batchSize);
_.each(jQuery('select.mailpoet_subscribers_column_data_match'),
function (column, index) {
var columnId = jQuery(column).data('column-id');
if (columnId === 'ignore') {
return;
}
subscribers[columnId] = [];
_.each(importData.step1.subscribers, function (subsciber) {
subscribers[columnId].push(
_.chain(subsciber)
.pick(index)
.toArray()
.flatten()
.value()
);
});
subscribers[columnId] = _.flatten(subscribers[columnId]);
});
function (column, columnIndex) {
var columnId = jQuery(column).data('column-id');
if (columnId === 'ignore') {
return;
}
columns[columnId] = columnIndex;
});
MailPoet.Ajax.post({
endpoint: 'ImportExport',
action: 'processImport',
data: JSON.stringify({
subscribers: subscribers,
segments: segmentSelectElement.val(),
updateSubscribers: (jQuery(':radio[name="subscriber_update_option"]:checked').val() === 'yes') ? true : false
_.each(subscribers, function () {
queue.add(function (queue) {
queue.pause();
MailPoet.Ajax
.post({
endpoint: 'ImportExport',
action: 'processImport',
data: JSON.stringify({
columns: columns,
subscribers: subscribers[batchNumber],
timestamp: timestamp,
segments: segmentSelectElement.val(),
updateSubscribers: (jQuery(':radio[name="subscriber_update_option"]:checked').val() === 'yes') ? true : false
})
})
.done(function (response) {
if (response.result === false) {
importResults.errors.push(response.errors);
} else {
importResults.created = response.data.created;
importResults.updated = response.data.updated;
importResults.segments = response.data.segments;
}
queue.run();
})
.error(function (error) {
importResults.errors.push(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.'
);
queue.run();
});
batchNumber++;
})
}).done(function (response) {
});
queue.run();
queue.onComplete(function () {
MailPoet.Modal.loading(false);
if (response.result === false) {
MailPoet.Notice.error(response.errors, {
timeout: 3000,
});
} else {
mailpoetSegments = response.data.segments;
response.data.segments = _.map(segmentSelectElement.select2('data'),
function (data) {
return data.name;
});
importData.step2 = response.data;
if (importResults.errors.length > 0 && !importResults.updated && !importResults.created) {
MailPoet.Notice.error(_.flatten(importResults.errors), {
timeout: noticeTimeout,
}
);
}
else {
mailpoetSegments = importResults.segments;
importResults.segments = _.map(segmentSelectElement.select2('data'),
function (data) {
return data.name;
});
importData.step2 = importResults;
enableSegmentSelection(mailpoetSegments);
router.navigate('step3', {trigger: true});
}
}).error(function (error) {
MailPoet.Modal.loading(false);
MailPoet.Notice.error(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', {
timeout: 3000,
}
);
});
});
filterSubscribers();
enableSegmentSelection(mailpoetSegments);
});
router.on('route:step3', function () {
@ -1118,6 +1155,12 @@ define(
showCurrentStep();
if (importData.step2.errors.length > 0) {
MailPoet.Notice.error(_.flatten(importData.step2.errors), {
timeout: noticeTimeout,
});
}
// display statistics
var subscribersDataImportResultsTemplate =
Handlebars

79
assets/js/src/vendor/jquery.asyncqueue.js vendored Normal file
View File

@ -0,0 +1,79 @@
/*
* This file is part of the jquery plugin "asyncQueue".
*
* (c) Sebastien Roch <roch.sebastien@gmail.com>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
(function($){
$.AsyncQueue = function() {
var that = this,
queue = [],
failureFunc,
completeFunc,
paused = false,
lastCallbackData,
_run;
_run = function() {
var f = queue.shift();
if (f) {
f.apply(that, [that]);
if (paused === false) {
_run();
}
} else {
if(completeFunc){
completeFunc.apply(that);
}
}
}
this.onFailure = function(func) {
failureFunc = func;
}
this.onComplete = function(func) {
completeFunc = func;
}
this.add = function(func) {
queue.push(func);
return this;
}
this.storeData = function(dataObject) {
lastCallbackData = dataObject;
return this;
}
this.lastCallbackData = function () {
return lastCallbackData;
}
this.run = function() {
paused = false;
_run();
}
this.pause = function () {
paused = true;
return this;
}
this.failure = function() {
paused = true;
if (failureFunc) {
var args = [that];
for(i = 0; i < arguments.length; i++) {
args.push(arguments[i]);
}
failureFunc.apply(that, args);
}
}
return this;
}
})(jQuery);