Cavemanon/piratepoet
7
Fork 0
Code Pull Requests Actions Releases 16 Activity

Import the MP2 users lists

Browse Source 
Import the MP2 users
Add the table "imported_data_mapping"
Add the function Helpers::mysql_date
This commit is contained in:
fred
2017-04-26 11:22:56 +02:00
parent 37a9fd9e1b
commit 472be3b071
7 changed files with 508 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
assets/js/src/mp2migrator.js
View File

@ -74,7 +74,7 @@
dataType: 'json'
}).always(function (result) {
// Move the progress bar
var progress = Number(result.current) / Number(result.total) * 100;
var progress = Math.round(Number(result.current) / Number(result.total) * 100);
$('#progressbar').progressbar('option', 'value', progress);
$('#progresslabel').html(progress + '%');
if(that.is_logging) {

2
lib/API/Endpoints/MP2Migrator.php
View File

@ -18,7 +18,7 @@ class MP2Migrator extends APIEndpoint {
*/
public function import($data) {
try {
$process = $this->MP2Migrator->import(json_decode($data, true));
$process = $this->MP2Migrator->import($data);
return $this->successResponse($process);
} catch(\Exception $e) {
return $this->errorResponse(array(

2
lib/Config/Database.php
View File

@ -77,6 +77,7 @@ class Database {
$statistics_opens = Env::$db_prefix . 'statistics_opens';
$statistics_unsubscribes = Env::$db_prefix . 'statistics_unsubscribes';
$statistics_forms = Env::$db_prefix . 'statistics_forms';
$imported_data_mapping = Env::$db_prefix . 'imported_data_mapping';
define('MP_SETTINGS_TABLE', $settings);
define('MP_SEGMENTS_TABLE', $segments);
@ -98,6 +99,7 @@ class Database {
define('MP_STATISTICS_OPENS_TABLE', $statistics_opens);
define('MP_STATISTICS_UNSUBSCRIBES_TABLE', $statistics_unsubscribes);
define('MP_STATISTICS_FORMS_TABLE', $statistics_forms);
define('MP_IMPORTED_DATA_MAPPING_TABLE', $imported_data_mapping);
}
}
}

480
lib/Config/MP2Migrator.php
View File

@ -4,6 +4,14 @@ namespace MailPoet\Config;
use MailPoet\Util\ProgressBar;
use MailPoet\Models\Setting;
use MailPoet\Models\Segment;
use MailPoet\Models\Subscriber;
use MailPoet\Models\CustomField;
use MailPoet\Models\SubscriberSegment;
use MailPoet\Models\SubscriberCustomField;
use MailPoet\Models\ImportedDataMapping;
use MailPoet\Config\Activator;
use MailPoet\Util\Helpers;
if(!defined('ABSPATH')) exit;
@ -12,6 +20,7 @@ class MP2Migrator {
private $log_file;
public $log_file_url;
public $progressbar;
private $chunks_size = 10; // To import the data by batch
public function __construct() {
$log_filename = 'mp2migration.log';
@ -94,23 +103,18 @@ class MP2Migrator {
* @return boolean Result
*/
public function import() {
set_time_limit(7200); // Timeout = 2 hours
ob_start();
$this->emptyLog();
$this->log(sprintf("=== START IMPORT %s ===", date('Y-m-d H:i:s')));
Setting::setValue('mailpoet_stopImport', false); // Reset the stop import action
Setting::setValue('import_stopped', false); // Reset the stop import action
$this->eraseMP3Data();
$this->displayDataToMigrate();
// TODO to remove, for testing only
$this->progressbar->setTotalCount(0);
$this->progressbar->setTotalCount(10);
for($i = 0; $i < 10; $i++) {
$this->progressbar->incrementCurrentCount(1);
usleep(300000);
if($this->importStopped()) {
return;
}
}
$this->importSegments();
$this->importCustomFields();
$this->importSubscribers();
$this->log(sprintf("=== END IMPORT %s ===", date('Y-m-d H:i:s')));
$result = ob_get_contents();
@ -126,12 +130,32 @@ class MP2Migrator {
file_put_contents($this->log_file, '');
}
/**
* Erase all the MailPoet 3 data
*
*/
private function eraseMP3Data() {
Activator::deactivate();
Activator::activate();
$this->resetMigrationCounters();
$this->log(__("MailPoet data erased", Env::$plugin_name));
}
/**
* Reset the migration counters
*
*/
private function resetMigrationCounters() {
Setting::setValue('last_imported_user_id', 0);
Setting::setValue('last_imported_list_id', 0);
}
/**
* Stop the import
*
*/
public function stopImport() {
Setting::setValue('mailpoet_stopImport', true);
Setting::setValue('import_stopped', true);
$this->log(__('IMPORT STOPPED BY USER', Env::$plugin_name));
}
@ -141,7 +165,7 @@ class MP2Migrator {
* @return boolean Import must stop or not
*/
private function importStopped() {
return Setting::getValue('mailpoet_stopImport');
return Setting::getValue('import_stopped', false);
}
/**
@ -166,16 +190,16 @@ class MP2Migrator {
$result .= __('MailPoet 2 data found:', Env::$plugin_name) . "\n";
// User Lists
$usersListsCount = $this->rowsCount('wysija_list');
$totalCount += $usersListsCount;
$result .= sprintf(_n('%d subscribers list', '%d subscribers lists', $usersListsCount, Env::$plugin_name), $usersListsCount) . "\n";
// Users
$usersCount = $this->rowsCount('wysija_user');
$totalCount += $usersCount;
$result .= sprintf(_n('%d subscriber', '%d subscribers', $usersCount, Env::$plugin_name), $usersCount) . "\n";
// User Lists
$usersListsCount = $this->rowsCount('wysija_user_list');
$totalCount += $usersListsCount;
$result .= sprintf(_n('%d subscribers list', '%d subscribers lists', $usersListsCount, Env::$plugin_name), $usersListsCount) . "\n";
// Emails
$emailsCount = $this->rowsCount('wysija_email');
$totalCount += $emailsCount;
@ -194,6 +218,7 @@ class MP2Migrator {
/**
* Count the number of rows in a table
*
* @global object $wpdb
* @param string $table Table
* @return int Number of rows found
*/
@ -207,4 +232,423 @@ class MP2Migrator {
return $count;
}
/**
* Import the subscribers segments
*
*/
private function importSegments() {
$imported_segments_count = 0;
if($this->importStopped()) {
return;
}
$this->log(__("Importing segments...", Env::$plugin_name));
do {
if($this->importStopped()) {
break;
}
$lists = $this->getLists($this->chunks_size);
$lists_count = count($lists);
if(is_array($lists)) {
foreach($lists as $list) {
$segment = $this->importSegment($list);
if(!empty($segment)) {
$imported_segments_count++;
}
}
}
$this->progressbar->incrementCurrentCount($lists_count);
} while(($lists != null) && ($lists_count > 0));
$this->log(sprintf(_n("%d segment imported", "%d segments imported", $imported_segments_count, Env::$plugin_name), $imported_segments_count));
}
/**
* Get the Mailpoet 2 users lists
*
* @global object $wpdb
* @param int $limit Number of users max
* @return array Users Lists
*/
private function getLists($limit) {
global $wpdb;
$lists = array();
$last_id = Setting::getValue('last_imported_list_id', 0);
$table = $wpdb->prefix . 'wysija_list';
$sql = "
SELECT l.list_id, l.name, l.description, l.is_enabled, l.created_at
FROM `$table` l
WHERE l.list_id > '$last_id'
ORDER BY l.list_id
LIMIT $limit
";
$lists = $wpdb->get_results($sql, ARRAY_A);
return $lists;
}
/**
* Import a segment
*
* @param array $list_data List data
* @return Segment
*/
private function importSegment($list_data) {
$segment = Segment::createOrUpdate(array(
'id' => $list_data['list_id'],
'name' => $list_data['name'],
'type' => $list_data['is_enabled']? 'default': 'wp_users',
'description' => $list_data['description'],
'created_at' => Helpers::mysql_date($list_data['created_at']),
));
Setting::setValue('last_imported_list_id', $list_data['list_id']);
return $segment;
}
/**
* Import the custom fields
*
*/
private function importCustomFields() {
$imported_custom_fields_count = 0;
if($this->importStopped()) {
return;
}
$this->log(__("Importing custom fields...", Env::$plugin_name));
$custom_fields = $this->getCustomFields();
foreach($custom_fields as $custom_field) {
$result = $this->importCustomField($custom_field);
if (!empty($result)) {
$imported_custom_fields_count++;
}
}
$this->log(sprintf(_n("%d custom field imported", "%d custom fields imported", $imported_custom_fields_count, Env::$plugin_name), $imported_custom_fields_count));
}
/**
* Get the Mailpoet 2 custom fields
*
* @global object $wpdb
* @return array Custom fields
*/
private function getCustomFields() {
global $wpdb;
$custom_fields = array();
$table = $wpdb->prefix . 'wysija_custom_field';
$sql = "
SELECT cf.id, cf.name, cf.type, cf.required, cf.settings
FROM `$table` cf
";
$custom_fields = $wpdb->get_results($sql, ARRAY_A);
return $custom_fields;
}
/**
* Import a custom field
*
* @param array $custom_field MP2 custom field
* @return CustomField
*/
private function importCustomField($custom_field) {
$data = array(
'id' => $custom_field['id'],
'name' => $custom_field['name'],
'type' => $this->mapCustomFieldType($custom_field['type']),
'params' => $this->mapCustomFieldParams($custom_field),
);
$customField = new CustomField();
$customField->createOrUpdate($data);
return $customField;
}
/**
* Map the MailPoet 2 custom field type with the MailPoet custom field type
*
* @param string $MP2type MP2 custom field type
* @return string MP3 custom field type
*/
private function mapCustomFieldType($MP2type) {
$type = '';
switch($MP2type) {
case 'input':
$type = 'text';
break;
default:
$type = $MP2type;
}
return $type;
}
/**
* Map the MailPoet 2 custom field settings with the MailPoet custom field params
*
* @param array $custom_field MP2 custom field
* @return string serialized MP3 custom field params
*/
private function mapCustomFieldParams($custom_field) {
$params = unserialize($custom_field['settings']);
$params['label'] = $custom_field['name'];
if(isset($params['validate'])) {
$params['validate'] = $this->mapCustomFieldValidateValue($params['validate']);
}
if(isset($params['date_order'])) { // Convert the date_order field
$params['date_format'] = strtoupper($params['date_order']);
unset($params['date_order']);
}
return $params;
}
/**
* Map the validate value
*
* @param string $MP2value MP2 value
* @return string MP3 value
*/
private function mapCustomFieldValidateValue($MP2value) {
$value = '';
switch($MP2value) {
case 'onlyLetterSp':
case 'onlyLetterNumber':
$value = 'alphanum';
break;
case 'onlyNumberSp':
$value = 'number';
break;
case 'phone':
$value = 'phone';
break;
}
return $value;
}
/**
* Import the subscribers
*
*/
private function importSubscribers() {
$imported_subscribers_count = 0;
if($this->importStopped()) {
return;
}
$this->log(__("Importing subscribers...", Env::$plugin_name));
do {
if($this->importStopped()) {
break;
}
$users = $this->getUsers($this->chunks_size);
$users_count = count($users);
if(is_array($users)) {
foreach($users as $user) {
$subscriber = $this->importSubscriber($user);
if (!empty($subscriber)) {
$imported_subscribers_count++;
$this->importSubscriberSegments($subscriber, $user['user_id']);
$this->importSubscriberCustomFields($subscriber, $user);
}
}
}
$this->progressbar->incrementCurrentCount($users_count);
} while(($users != null) && ($users_count > 0));
$this->log(sprintf(_n("%d subscriber imported", "%d subscribers imported", $imported_subscribers_count, Env::$plugin_name), $imported_subscribers_count));
}
/**
* Get the Mailpoet 2 users
*
* @global object $wpdb
* @param int $limit Number of users max
* @return array Users
*/
private function getUsers($limit) {
global $wpdb;
$users = array();
$last_id = Setting::getValue('last_imported_user_id', 0);
$table = $wpdb->prefix . 'wysija_user';
$sql = "
SELECT u.*
FROM `$table` u
WHERE u.user_id > '$last_id'
ORDER BY u.user_id
LIMIT $limit
";
$users = $wpdb->get_results($sql, ARRAY_A);
return $users;
}
/**
* Import a subscriber
*
* @param array $user_data User data
* @return Subscriber
*/
private function importSubscriber($user_data) {
$subscriber = Subscriber::createOrUpdate(array(
'id' => $user_data['user_id'],
'wp_user_id' => !empty($user_data['wpuser_id'])? $user_data['wpuser_id'] : null,
'email' => $user_data['email'],
'first_name' => $user_data['firstname'],
'last_name' => $user_data['lastname'],
'status' => $this->mapUserStatus($user_data['status']),
'created_at' => Helpers::mysql_date($user_data['created_at']),
'subscribed_ip' => !empty($user_data['ip'])? $user_data['ip'] : null,
'confirmed_ip' => !empty($user_data['confirmed_ip'])? $user_data['confirmed_ip'] : null,
'confirmed_at' => !empty($user_data['confirmed_at'])? Helpers::mysql_date($user_data['confirmed_at']) : null,
));
Setting::setValue('last_imported_user_id', $user_data['user_id']);
return $subscriber;
}
/**
* Map the MailPoet 2 user status with MailPoet 3
*
* @param int $mp2_user_status MP2 user status
* @return string MP3 user status
*/
private function mapUserStatus($mp2_user_status) {
switch($mp2_user_status) {
case 0: $status = 'unconfirmed'; break;
case 1: $status = 'subscribed'; break;
case -1: $status = 'unsubscribed'; break;
default: $status = 'unconfirmed';
}
return $status;
}
/**
* Import the segments for a subscriber
*
* @param Subscriber $subscriber MP3 subscriber
* @param int $user_id MP2 user ID
*/
private function importSubscriberSegments($subscriber, $user_id) {
$user_lists = $this->getUserLists($user_id);
foreach($user_lists as $user_list) {
$this->importSubscriberSegment($subscriber->id, $user_list);
}
}
/**
* Get the lists for a user
*
* @global object $wpdb
* @param int $user_id User ID
* @return array Users Lists
*/
private function getUserLists($user_id) {
global $wpdb;
$user_lists = array();
$table = $wpdb->prefix . 'wysija_user_list';
$sql = "
SELECT ul.list_id, ul.sub_date, ul.unsub_date
FROM `$table` ul
WHERE ul.user_id = '$user_id'
";
$user_lists = $wpdb->get_results($sql, ARRAY_A);
return $user_lists;
}
/**
* Import a subscriber segment
*
* @param int $subscriber_id
* @param array $user_list
* @return SubscriberSegment
*/
private function importSubscriberSegment($subscriber_id, $user_list) {
$data = array(
'subscriber_id' => $subscriber_id,
'segment_id' => $user_list['list_id'],
'status' => empty($user_list['unsub_date'])? 'subscribed' : 'unsubscribed',
'created_at' => Helpers::mysql_date($user_list['sub_date']),
'updated_at' => !empty($user_list['unsub_date'])? Helpers::mysql_date($user_list['unsub_date']) : null,
);
$subscriberSegment = new SubscriberSegment();
$subscriberSegment->createOrUpdate($data);
return $subscriberSegment;
}
/**
* Import the custom fields values for a subscriber
*
* @param Subscriber $subscriber MP3 subscriber
* @param array $user MP2 user
*/
private function importSubscriberCustomFields($subscriber, $user) {
$imported_custom_fields = $this->getImportedCustomFields();
foreach($imported_custom_fields as $custom_field) {
$custom_field_column = 'cf_' . $custom_field['id'];
if(isset($custom_field_column)) {
$this->importSubscriberCustomField($subscriber->id, $custom_field, $user[$custom_field_column]);
}
}
}
/**
* Get the imported custom fields
*
* @global object $wpdb
* @return array Imported custom fields
*
*/
private function getImportedCustomFields() {
global $wpdb;
$table = MP_CUSTOM_FIELDS_TABLE;
$sql = "
SELECT cf.id, cf.name, cf.type
FROM `$table` cf
";
$custom_fields = $wpdb->get_results($sql, ARRAY_A);
return $custom_fields;
}
/**
* Import a subscriber custom field
*
* @param int $subscriber_id Subscriber ID
* @param int $custom_field Custom field
* @param string $custom_field_value Custom field value
* @return SubscriberCustomField
*/
private function importSubscriberCustomField($subscriber_id, $custom_field, $custom_field_value) {
if($custom_field['type'] == 'date') {
$value = Helpers::mysql_date($custom_field_value); // Convert the date field
} else {
$value = $custom_field_value;
}
$data = array(
'subscriber_id' => $subscriber_id,
'custom_field_id' => $custom_field['id'],
'value' => isset($value)? $value : '',
);
$subscriberCustomField = new SubscriberCustomField();
$subscriberCustomField->createOrUpdate($data);
return $subscriberCustomField;
}
/**
* Get the mapping between the MP2 and the imported MP3 IDs
*
* @param string $model Model (segment,...)
* @return array Mapping
*/
private function getImportedMapping($model) {
$mappings = array();
$mapping_relations = ImportedDataMapping::where('type', $model)->findArray();
foreach($mapping_relations as $relation) {
$mappings[$relation['old_id']] = $relation['new_id'];
}
return $mappings;
}
}

17
lib/Config/Migrator.php
View File

@ -33,7 +33,8 @@ class Migrator {
'statistics_clicks',
'statistics_opens',
'statistics_unsubscribes',
'statistics_forms'
'statistics_forms',
'imported_data_mapping'
);
}
@ -79,7 +80,7 @@ class Migrator {
function settings() {
$attributes = array(
'id mediumint(9) NOT NULL AUTO_INCREMENT,',
'name varchar(20) NOT NULL,',
'name varchar(50) NOT NULL,',
'value longtext,',
'created_at TIMESTAMP NULL,',
'updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,',
@ -363,6 +364,18 @@ class Migrator {
return $this->sqlify(__FUNCTION__, $attributes);
}
function importedDataMapping() {
$attributes = array(
'old_id mediumint(9) NOT NULL,',
'type varchar(50) NOT NULL,',
'new_id mediumint(9) NOT NULL,',
'created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,',
'PRIMARY KEY (old_id, type),',
'KEY new_id (new_id)'
);
return $this->sqlify(__FUNCTION__, $attributes);
}
private function sqlify($model, $attributes) {
$table = $this->prefix . Helpers::camelCaseToUnderscore($model);

15
lib/Models/ImportedDataMapping.php Normal file
View File

@ -0,0 +1,15 @@
<?php
namespace MailPoet\Models;
if(!defined('ABSPATH')) exit;
class ImportedDataMapping extends Model {
public static $_table = MP_IMPORTED_DATA_MAPPING_TABLE;
static function create($data = array()) {
$relation = parent::create();
$relation->hydrate($data);
return $relation->save();
}
}

11
lib/Util/Helpers.php
View File

@ -136,4 +136,15 @@ class Helpers {
static function splitObject($object = array()) {
return explode(self::DIVIDER, $object);
}
/**
* Convert a timestamp to a Mysql datetime
*
* @param int $timestamp Timestamp
* @return string Datetime
*/
static function mysql_date($timestamp) {
return date('Y-m-d H:i:s', $timestamp);
}
}