Cavemanon/piratepoet
7
Fork 0
Code Pull Requests Actions Releases 16 Activity

- Removes file size limit in import

Browse Source 
- Implements chunked import processing
- Updates tests/migrator/Subscriber model
This commit is contained in:
Vlad
2016-02-20 18:53:24 -05:00
parent 580ac989aa
commit 499936e3ab
9 changed files with 203 additions and 5135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
assets/js/src/subscribers/importExport/import.js
View File

@ -6,7 +6,8 @@ define(
'mailpoet',
'handlebars',
'papaparse',
'select2'
'select2',
'asyncqueue'
],
function (
Backbone,
@ -14,12 +15,14 @@ define(
jQuery,
MailPoet,
Handlebars,
Papa
Papa,
AsyncQueue
) {
if (!jQuery('#mailpoet_subscribers_import').length) {
return;
}
jQuery(document).ready(function () {
console.log = function() {};
jQuery('input[name="select_method"]').attr('checked', false);
// configure router
router = new (Backbone.Router.extend({
@ -1050,64 +1053,96 @@ define(
}
MailPoet.Modal.loading(true);
var subscribers = {};
var columns = {},
queue = new jQuery.AsyncQueue(),
batch = 0,
subscribers = [],
importResults = {
'created': 0,
'updated': 0,
'errors': [],
'segments': []
},
splitSubscribers = function (subscribers, size) {
return subscribers.reduce(function (res, item, index) {
if (index % size === 0) {
res.push([]);
}
res[res.length - 1].push(item);
return res;
}, []);
},
subscribers = splitSubscribers(importData.step1.subscribers, 500);
_.each(jQuery('select.mailpoet_subscribers_column_data_match'),
function (column, index) {
function (column, columnIndex) {
var columnId = jQuery(column).data('column-id');
if (columnId === 'ignore') {
return;
}
subscribers[columnId] = [];
_.each(importData.step1.subscribers, function (subsciber) {
subscribers[columnId].push(
_.chain(subsciber)
.pick(index)
.toArray()
.flatten()
.value()
);
});
subscribers[columnId] = _.flatten(subscribers[columnId]);
columns[columnId] = columnIndex;
});
MailPoet.Ajax.post({
_.each(subscribers, function () {
queue.add(function (queue) {
queue.pause();
MailPoet.Ajax
.post({
endpoint: 'ImportExport',
action: 'processImport',
data: JSON.stringify({
subscribers: subscribers,
columns: columns,
subscribers: subscribers[batch],
length: subscribers[batch].length,
segments: segmentSelectElement.val(),
updateSubscribers: (jQuery(':radio[name="subscriber_update_option"]:checked').val() === 'yes') ? true : false
})
}).done(function (response) {
MailPoet.Modal.loading(false);
})
.done(function (response) {
if (response.result === false) {
MailPoet.Notice.error(response.errors, {
timeout: 3000,
});
importResults.errors.push(response.errors);
} else {
mailpoetSegments = response.data.segments;
response.data.segments = _.map(segmentSelectElement.select2('data'),
function (data) {
return data.name;
});
importData.step2 = response.data;
enableSegmentSelection(mailpoetSegments);
router.navigate('step3', {trigger: true});
importResults.created += response.data.created;
importResults.updated += response.data.updated;
importResults.segments = response.data.segments;
}
}).error(function (error) {
queue.run();
})
.error(function (error) {
importResults.errors.push(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.'
);
queue.run();
});
batch++;
})
});
queue.run();
queue.onComplete(function () {
MailPoet.Modal.loading(false);
MailPoet.Notice.error(
MailPoetI18n.serverError + error.statusText.toLowerCase() + '.', {
if (importResults.errors.length > 0 && !importResults.updated && !importResults.created) {
MailPoet.Notice.error(_.flatten(importResults.errors), {
timeout: 3000,
}
);
}
else {
mailpoetSegments = importResults.segments;
importResults.segments = _.map(segmentSelectElement.select2('data'),
function (data) {
return data.name;
});
importData.step2 = importResults;
enableSegmentSelection(mailpoetSegments);
router.navigate('step3', {trigger: true});
}
});
});
filterSubscribers();
enableSegmentSelection(mailpoetSegments);
});
router.on('route:step3', function () {
@ -1118,6 +1153,12 @@ define(
showCurrentStep();
if (importData.step2.errors.length > 0) {
MailPoet.Notice.error(_.flatten(importData.step2.errors), {
timeout: 3000,
});
}
// display statistics
var subscribersDataImportResultsTemplate =
Handlebars

1
lib/Config/Migrator.php
View File

@ -57,6 +57,7 @@ class Migrator {
'last_name tinytext NOT NULL,',
'email varchar(150) NOT NULL,',
'status varchar(12) NOT NULL DEFAULT "unconfirmed",',
'import_batch varchar(12) NULL,',
'created_at TIMESTAMP NOT NULL DEFAULT 0,',
'deleted_at TIMESTAMP NULL DEFAULT NULL,',
'updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,',

4
lib/Models/Subscriber.php
View File

@ -498,7 +498,7 @@ class Subscriber extends Model {
);
}
static function updateMultiple($columns, $subscribers, $currentTime = false) {
static function updateMultiple($columns, $subscribers, $import_batch = false) {
$ignoreColumnsOnUpdate = array(
'email',
'created_at'
@ -538,7 +538,7 @@ class Subscriber extends Model {
return self::rawExecute(
'UPDATE `' . self::$_table . '` ' .
'SET ' . implode(', ', $sql('statement')) . ' '.
(($currentTime) ? ', updated_at = "' . $currentTime . '" ' : '') .
(($import_batch) ? ', import_batch = "' . $import_batch . '" ' : '') .
'WHERE email IN ' .
'(' . rtrim(str_repeat('?,', count($subscribers)), ',') . ')',
array_merge(

51
lib/Subscribers/ImportExport/Import/Import.php
View File

@ -6,6 +6,7 @@ use MailPoet\Models\SubscriberCustomField;
use MailPoet\Models\SubscriberSegment;
use MailPoet\Subscribers\ImportExport\BootStrapMenu;
use MailPoet\Util\Helpers;
use MailPoet\Util\Security;
class Import {
public $subscribers_data;
@ -15,20 +16,25 @@ class Import {
public $subscriber_custom_fields;
public $subscribers_count;
public $import_time;
public $import_batch;
public $profiler_start;
public function __construct($data) {
$this->subscribers_data = $data['subscribers'];
$this->subscribers_data = $this->transformSubscribersData(
$data['subscribers'],
$data['columns']
);
$this->segments = $data['segments'];
$this->update_subscribers = $data['updateSubscribers'];
$this->subscriber_fields = $this->getSubscriberFields(
array_keys($this->subscribers_data)
array_keys($data['columns'])
);
$this->subscriber_custom_fields = $this->getCustomSubscriberFields(
array_keys($this->subscribers_data)
array_keys($data['columns'])
);
$this->subscribers_count = count(reset($this->subscribers_data));
$this->import_time = date('Y-m-d H:i:s');
$this->import_batch = Security::generateRandomString();
$this->profiler_start = microtime(true);
}
@ -63,14 +69,6 @@ class Import {
$subscriber_fields,
$subscriber_custom_fields
);
if($created_subscribers) {
// subtract added from updated subscribers when DB operation takes <1s
$updated_subscribers = array_diff_key(
$updated_subscribers,
$created_subscribers,
$subscriber_custom_fields
);
}
}
} catch(\PDOException $e) {
return array(
@ -86,10 +84,18 @@ class Import {
'updated' => count($updated_subscribers),
'segments' => $segments->getSegments()
),
'time' => date('Y-m-d H:i:s'),
'profiler' => $this->timeExecution()
);
}
function transformSubscribersData($subscribers, $columns) {
foreach($columns as $column => $index) {
$transformed_subscribers[$column] = Helpers::arrayColumn($subscribers, $index);
}
return $transformed_subscribers;
}
function filterExistingAndNewSubscribers($subscribers_data) {
$existing_records = array_filter(
array_map(function ($subscriber_emails) {
@ -163,8 +169,17 @@ class Import {
}
function extendSubscribersAndFields($subscribers_data, $subscriber_fields) {
$subscribers_data['created_at'] = $this->filterSubscriberCreatedAtDate();
$subscriber_fields[] = 'created_at';
$subscribers_data['created_at'] =
array_fill(0, $this->subscribers_count, $this->import_time);
$subscribers_data['import_batch'] =
array_fill(0, $this->subscribers_count, $this->import_batch);
$subscriber_fields = array_merge(
$subscriber_fields,
array(
'created_at',
'import_batch'
)
);
return array(
$subscribers_data,
$subscriber_fields
@ -191,10 +206,6 @@ class Import {
);
}
function filterSubscriberCreatedAtDate() {
return array_fill(0, $this->subscribers_count, $this->import_time);
}
function filterSubscriberStatus($subscribers_data, $subscriber_fields) {
if(!in_array('status', $subscriber_fields)) {
$subscribers_data['status'] =
@ -255,7 +266,7 @@ class Import {
return $subscribers_data[$field][$index];
}, $subscriber_fields);
}, range(0, $subscribers_count));
$import_time = ($action === 'update') ? date('Y-m-d H:i:s') : $this->import_time;
$batch = ($action === 'update') ? Security::generateRandomString() : $this->import_batch;
foreach(array_chunk($subscribers, 100) as $data) {
if($action == 'create') {
Subscriber::createMultiple(
@ -267,7 +278,7 @@ class Import {
Subscriber::updateMultiple(
$subscriber_fields,
$data,
$import_time
$batch
);
}
}
@ -277,7 +288,7 @@ class Import {
'id',
'email'
))
->where(($action === 'create') ? 'created_at' : 'updated_at', $import_time)
->where('import_batch', $batch)
->findArray(),
'email', 'id'
);

3
package.json
View File

@ -5,7 +5,8 @@
},
"napa": {
"blob": "eligrey/Blob.js.git",
"filesaver": "eligrey/FileSaver.js.git"
"filesaver": "eligrey/FileSaver.js.git",
"asyncqueue": "mjward/Jquery-Async-queue.git"
},
"dependencies": {
"backbone": "1.2.3",

100
tests/unit/Subscribers/ImportExport/Import/ImportCest.php
View File

@ -8,24 +8,31 @@ use MailPoet\Util\Helpers;
class ImportCest {
function __construct() {
$this->JSON_data = json_decode(file_get_contents(dirname(__FILE__) . '/ImportTestData.json'), true);
$this->subscribers_data = array(
'first_name' => array(
$this->data = array(
'subscribers' => array(
array(
'Adam',
'Mary'
),
'last_name' => array(
'Smith',
'Jane'
),
'email' => array(
'adam@smith.com',
'mary@jane.com'
'France'
),
777 => array(
'France',
array(
'Mary',
'Jane',
'mary@jane.com',
'Brazil'
)
),
'columns' => array(
'first_name' => 0,
'last_name' => 1,
'email' => 2,
777 => 3
),
'segments' => array(
195
),
'updateSubscribers' => true
);
$this->subscriber_fields = array(
'first_name',
@ -34,25 +41,37 @@ class ImportCest {
);
$this->segments = range(0, 1);
$this->subscriber_custom_fields = array(777);
$this->import = new Import($this->JSON_data);
$this->import = new Import($this->data);
$this->subscribers_data = $this->import->transformSubscribersData(
$this->data['subscribers'],
$this->data['columns']
);
}
function itCanConstruct() {
expect($this->import->subscribers_data)->equals($this->JSON_data['subscribers']);
expect($this->import->segments)->equals($this->JSON_data['segments']);
expect(is_array($this->import->subscribers_data))->true();
expect($this->import->segments)->equals($this->data['segments']);
expect(is_array($this->import->subscriber_fields))->true();
expect(is_array($this->import->subscriber_custom_fields))->true();
expect($this->import->subscribers_count)->equals(
count($this->JSON_data['subscribers']['email'])
);
expect(
preg_match(
'/\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}/',
$this->import->import_time)
)->equals(1);
expect($this->import->subscribers_count)->equals(2);
expect($this->import->import_batch)->notEmpty();
expect($this->import->import_time)->notEmpty();
expect($this->import->import_batch)->notEmpty();
}
function itCanTransformSubscribers() {
expect($this->import->subscribers_data['first_name'][0])
->equals($this->data['subscribers'][0][0]);
expect($this->import->subscribers_data['last_name'][0])
->equals($this->data['subscribers'][0][1]);
expect($this->import->subscribers_data['email'][0])
->equals($this->data['subscribers'][0][2]);
expect($this->import->subscribers_data['777'][0])
->equals($this->data['subscribers'][0][3]);
}
function itCanFilterExistingAndNewSubscribers() {
$subscribers_data = $this->subscribers_data;
$subscriber = Subscriber::create();
$subscriber->hydrate(
array(
@ -62,10 +81,10 @@ class ImportCest {
));
$subscriber->save();
list($existing, $new) = $this->import->filterExistingAndNewSubscribers(
$this->subscribers_data
$subscribers_data
);
expect($existing['email'][0])->equals($this->subscribers_data['email'][0]);
expect($new['email'][0])->equals($this->subscribers_data['email'][1]);
expect($existing['email'][0])->equals($subscribers_data['email'][0]);
expect($new['email'][0])->equals($subscribers_data['email'][1]);
}
function itCanExtendSubscribersAndFields() {
@ -106,17 +125,17 @@ class ImportCest {
}
function itCanFilterSubscriberStatus() {
$subscibers_data = $this->subscribers_data;
$subscribers_data = $this->subscribers_data;
$subscriber_fields = $this->subscriber_fields;
list($subscibers_data, $subsciber_fields) =
$this->import->filterSubscriberStatus($subscibers_data, $subscriber_fields);
list($subscribers_data, $subsciber_fields) =
$this->import->filterSubscriberStatus($subscribers_data, $subscriber_fields);
// subscribers' status was set to "subscribed" & status column was added
// to subscribers fields
expect(array_pop($subsciber_fields))->equals('status');
expect($subscibers_data['status'][0])->equals('subscribed');
expect(count($subscibers_data['status']))->equals(2);
expect($subscribers_data['status'][0])->equals('subscribed');
expect(count($subscribers_data['status']))->equals(2);
$subscriber_fields[] = 'status';
$subscibers_data = array(
$subscribers_data = array(
'status' => array(
#subscribed
'subscribed',
@ -135,9 +154,9 @@ class ImportCest {
'false'
),
);
list($subscibers_data, $subsciber_fields) =
$this->import->filterSubscriberStatus($subscibers_data, $subscriber_fields);
expect($subscibers_data)->equals(
list($subscribers_data, $subsciber_fields) =
$this->import->filterSubscriberStatus($subscribers_data, $subscriber_fields);
expect($subscribers_data)->equals(
array(
'status' => array(
'subscribed',
@ -255,6 +274,7 @@ class ImportCest {
->equals($subscribers_data[777][1]);
}
function itCanaddSubscribersToSegments() {
$subscribers_data = $this->subscribers_data;
$this->import->createOrUpdateSubscribers(
@ -296,19 +316,17 @@ class ImportCest {
function itCanProcess() {
$import = clone($this->import);
$result = $import->process();
expect($result['data']['created'])->equals(997);
expect($result['data']['created'])->equals(2);
expect($result['data']['updated'])->equals(0);
$result = $import->process();
expect($result['data']['created'])->equals(0);
expect($result['data']['updated'])->equals(997);
Subscriber::where('email', 'mbanks4@blinklist.com')
expect($result['data']['updated'])->equals(2);
Subscriber::where('email', 'mary@jane.com')
->findOne()
->delete();
// TODO: find a more elegant way to test this
$import->import_time = date('Y-m-d 12:i:s');
$result = $import->process();
expect($result['data']['created'])->equals(1);
expect($result['data']['updated'])->equals(996);
expect($result['data']['updated'])->equals(1);
$import->update_subscribers = false;
$result = $import->process();
expect($result['data']['created'])->equals(0);

5003
tests/unit/Subscribers/ImportExport/Import/ImportTestData.json
View File

File diff suppressed because it is too large Load Diff

2
views/subscribers/importExport/import/step1.html
View File

@ -60,8 +60,6 @@
</th>
<td>
<input type="file" id="file_local" accept=".csv" />
&nbsp;
<%= __('total max upload file size : %s')|format(maxPostSize) %>
</td>
</tr>
</tbody>

3
webpack.config.js
View File

@ -30,7 +30,8 @@ baseConfig = {
'filesaver$': 'filesaver/FileSaver.js',
'papaparse': 'papaparse/papaparse.min.js',
'helpscout': 'helpscout.js',
'html2canvas': 'html2canvas/dist/html2canvas.js'
'html2canvas': 'html2canvas/dist/html2canvas.js',
'asyncqueue': 'asyncqueue/jquery.asyncqueue.js'
},
},
node: {