diff --git a/assets/js/src/mp2migrator.js b/assets/js/src/mp2migrator.js index 4695b74b32..900017d124 100644 --- a/assets/js/src/mp2migrator.js +++ b/assets/js/src/mp2migrator.js @@ -74,7 +74,7 @@ dataType: 'json' }).always(function (result) { // Move the progress bar - var progress = Number(result.current) / Number(result.total) * 100; + var progress = Math.round(Number(result.current) / Number(result.total) * 100); $('#progressbar').progressbar('option', 'value', progress); $('#progresslabel').html(progress + '%'); if(that.is_logging) { diff --git a/lib/API/Endpoints/MP2Migrator.php b/lib/API/Endpoints/MP2Migrator.php index 054e62a7ec..49c6370bdf 100644 --- a/lib/API/Endpoints/MP2Migrator.php +++ b/lib/API/Endpoints/MP2Migrator.php @@ -18,7 +18,7 @@ class MP2Migrator extends APIEndpoint { */ public function import($data) { try { - $process = $this->MP2Migrator->import(json_decode($data, true)); + $process = $this->MP2Migrator->import($data); return $this->successResponse($process); } catch(\Exception $e) { return $this->errorResponse(array( diff --git a/lib/Config/Database.php b/lib/Config/Database.php index cee44d7339..133a9a3132 100644 --- a/lib/Config/Database.php +++ b/lib/Config/Database.php @@ -77,6 +77,7 @@ class Database { $statistics_opens = Env::$db_prefix . 'statistics_opens'; $statistics_unsubscribes = Env::$db_prefix . 'statistics_unsubscribes'; $statistics_forms = Env::$db_prefix . 'statistics_forms'; + $imported_data_mapping = Env::$db_prefix . 'imported_data_mapping'; define('MP_SETTINGS_TABLE', $settings); define('MP_SEGMENTS_TABLE', $segments); @@ -98,6 +99,7 @@ class Database { define('MP_STATISTICS_OPENS_TABLE', $statistics_opens); define('MP_STATISTICS_UNSUBSCRIBES_TABLE', $statistics_unsubscribes); define('MP_STATISTICS_FORMS_TABLE', $statistics_forms); + define('MP_IMPORTED_DATA_MAPPING_TABLE', $imported_data_mapping); } } } diff --git a/lib/Config/MP2Migrator.php b/lib/Config/MP2Migrator.php index 6aff8c44a7..683894abff 100644 --- a/lib/Config/MP2Migrator.php +++ b/lib/Config/MP2Migrator.php @@ -4,6 +4,14 @@ namespace MailPoet\Config; use MailPoet\Util\ProgressBar; use MailPoet\Models\Setting; +use MailPoet\Models\Segment; +use MailPoet\Models\Subscriber; +use MailPoet\Models\CustomField; +use MailPoet\Models\SubscriberSegment; +use MailPoet\Models\SubscriberCustomField; +use MailPoet\Models\ImportedDataMapping; +use MailPoet\Config\Activator; +use MailPoet\Util\Helpers; if(!defined('ABSPATH')) exit; @@ -12,6 +20,7 @@ class MP2Migrator { private $log_file; public $log_file_url; public $progressbar; + private $chunks_size = 10; // To import the data by batch public function __construct() { $log_filename = 'mp2migration.log'; @@ -94,24 +103,19 @@ class MP2Migrator { * @return boolean Result */ public function import() { + set_time_limit(7200); // Timeout = 2 hours ob_start(); $this->emptyLog(); $this->log(sprintf("=== START IMPORT %s ===", date('Y-m-d H:i:s'))); - Setting::setValue('mailpoet_stopImport', false); // Reset the stop import action + Setting::setValue('import_stopped', false); // Reset the stop import action + $this->eraseMP3Data(); $this->displayDataToMigrate(); - // TODO to remove, for testing only - $this->progressbar->setTotalCount(0); - $this->progressbar->setTotalCount(10); - for($i = 0; $i < 10; $i++) { - $this->progressbar->incrementCurrentCount(1); - usleep(300000); - if($this->importStopped()) { - return; - } - } - + $this->importSegments(); + $this->importCustomFields(); + $this->importSubscribers(); + $this->log(sprintf("=== END IMPORT %s ===", date('Y-m-d H:i:s'))); $result = ob_get_contents(); ob_clean(); @@ -126,12 +130,32 @@ class MP2Migrator { file_put_contents($this->log_file, ''); } + /** + * Erase all the MailPoet 3 data + * + */ + private function eraseMP3Data() { + Activator::deactivate(); + Activator::activate(); + $this->resetMigrationCounters(); + $this->log(__("MailPoet data erased", Env::$plugin_name)); + } + + /** + * Reset the migration counters + * + */ + private function resetMigrationCounters() { + Setting::setValue('last_imported_user_id', 0); + Setting::setValue('last_imported_list_id', 0); + } + /** * Stop the import * */ public function stopImport() { - Setting::setValue('mailpoet_stopImport', true); + Setting::setValue('import_stopped', true); $this->log(__('IMPORT STOPPED BY USER', Env::$plugin_name)); } @@ -141,7 +165,7 @@ class MP2Migrator { * @return boolean Import must stop or not */ private function importStopped() { - return Setting::getValue('mailpoet_stopImport'); + return Setting::getValue('import_stopped', false); } /** @@ -166,16 +190,16 @@ class MP2Migrator { $result .= __('MailPoet 2 data found:', Env::$plugin_name) . "\n"; + // User Lists + $usersListsCount = $this->rowsCount('wysija_list'); + $totalCount += $usersListsCount; + $result .= sprintf(_n('%d subscribers list', '%d subscribers lists', $usersListsCount, Env::$plugin_name), $usersListsCount) . "\n"; + // Users $usersCount = $this->rowsCount('wysija_user'); $totalCount += $usersCount; $result .= sprintf(_n('%d subscriber', '%d subscribers', $usersCount, Env::$plugin_name), $usersCount) . "\n"; - // User Lists - $usersListsCount = $this->rowsCount('wysija_user_list'); - $totalCount += $usersListsCount; - $result .= sprintf(_n('%d subscribers list', '%d subscribers lists', $usersListsCount, Env::$plugin_name), $usersListsCount) . "\n"; - // Emails $emailsCount = $this->rowsCount('wysija_email'); $totalCount += $emailsCount; @@ -194,6 +218,7 @@ class MP2Migrator { /** * Count the number of rows in a table * + * @global object $wpdb * @param string $table Table * @return int Number of rows found */ @@ -207,4 +232,423 @@ class MP2Migrator { return $count; } + /** + * Import the subscribers segments + * + */ + private function importSegments() { + $imported_segments_count = 0; + if($this->importStopped()) { + return; + } + $this->log(__("Importing segments...", Env::$plugin_name)); + do { + if($this->importStopped()) { + break; + } + $lists = $this->getLists($this->chunks_size); + $lists_count = count($lists); + + if(is_array($lists)) { + foreach($lists as $list) { + $segment = $this->importSegment($list); + if(!empty($segment)) { + $imported_segments_count++; + } + } + } + $this->progressbar->incrementCurrentCount($lists_count); + } while(($lists != null) && ($lists_count > 0)); + + $this->log(sprintf(_n("%d segment imported", "%d segments imported", $imported_segments_count, Env::$plugin_name), $imported_segments_count)); + } + + /** + * Get the Mailpoet 2 users lists + * + * @global object $wpdb + * @param int $limit Number of users max + * @return array Users Lists + */ + private function getLists($limit) { + global $wpdb; + $lists = array(); + + $last_id = Setting::getValue('last_imported_list_id', 0); + $table = $wpdb->prefix . 'wysija_list'; + $sql = " + SELECT l.list_id, l.name, l.description, l.is_enabled, l.created_at + FROM `$table` l + WHERE l.list_id > '$last_id' + ORDER BY l.list_id + LIMIT $limit + "; + $lists = $wpdb->get_results($sql, ARRAY_A); + + return $lists; + } + + /** + * Import a segment + * + * @param array $list_data List data + * @return Segment + */ + private function importSegment($list_data) { + $segment = Segment::createOrUpdate(array( + 'id' => $list_data['list_id'], + 'name' => $list_data['name'], + 'type' => $list_data['is_enabled']? 'default': 'wp_users', + 'description' => $list_data['description'], + 'created_at' => Helpers::mysql_date($list_data['created_at']), + )); + Setting::setValue('last_imported_list_id', $list_data['list_id']); + return $segment; + } + + /** + * Import the custom fields + * + */ + private function importCustomFields() { + $imported_custom_fields_count = 0; + if($this->importStopped()) { + return; + } + $this->log(__("Importing custom fields...", Env::$plugin_name)); + $custom_fields = $this->getCustomFields(); + + foreach($custom_fields as $custom_field) { + $result = $this->importCustomField($custom_field); + if (!empty($result)) { + $imported_custom_fields_count++; + } + } + + $this->log(sprintf(_n("%d custom field imported", "%d custom fields imported", $imported_custom_fields_count, Env::$plugin_name), $imported_custom_fields_count)); + } + + /** + * Get the Mailpoet 2 custom fields + * + * @global object $wpdb + * @return array Custom fields + */ + private function getCustomFields() { + global $wpdb; + $custom_fields = array(); + + $table = $wpdb->prefix . 'wysija_custom_field'; + $sql = " + SELECT cf.id, cf.name, cf.type, cf.required, cf.settings + FROM `$table` cf + "; + $custom_fields = $wpdb->get_results($sql, ARRAY_A); + + return $custom_fields; + } + + /** + * Import a custom field + * + * @param array $custom_field MP2 custom field + * @return CustomField + */ + private function importCustomField($custom_field) { + $data = array( + 'id' => $custom_field['id'], + 'name' => $custom_field['name'], + 'type' => $this->mapCustomFieldType($custom_field['type']), + 'params' => $this->mapCustomFieldParams($custom_field), + ); + $customField = new CustomField(); + $customField->createOrUpdate($data); + return $customField; + } + + /** + * Map the MailPoet 2 custom field type with the MailPoet custom field type + * + * @param string $MP2type MP2 custom field type + * @return string MP3 custom field type + */ + private function mapCustomFieldType($MP2type) { + $type = ''; + switch($MP2type) { + case 'input': + $type = 'text'; + break; + default: + $type = $MP2type; + } + return $type; + } + + /** + * Map the MailPoet 2 custom field settings with the MailPoet custom field params + * + * @param array $custom_field MP2 custom field + * @return string serialized MP3 custom field params + */ + private function mapCustomFieldParams($custom_field) { + $params = unserialize($custom_field['settings']); + $params['label'] = $custom_field['name']; + if(isset($params['validate'])) { + $params['validate'] = $this->mapCustomFieldValidateValue($params['validate']); + } + if(isset($params['date_order'])) { // Convert the date_order field + $params['date_format'] = strtoupper($params['date_order']); + unset($params['date_order']); + } + return $params; + } + + /** + * Map the validate value + * + * @param string $MP2value MP2 value + * @return string MP3 value + */ + private function mapCustomFieldValidateValue($MP2value) { + $value = ''; + switch($MP2value) { + case 'onlyLetterSp': + case 'onlyLetterNumber': + $value = 'alphanum'; + break; + case 'onlyNumberSp': + $value = 'number'; + break; + case 'phone': + $value = 'phone'; + break; + } + return $value; + } + + /** + * Import the subscribers + * + */ + private function importSubscribers() { + $imported_subscribers_count = 0; + if($this->importStopped()) { + return; + } + $this->log(__("Importing subscribers...", Env::$plugin_name)); + do { + if($this->importStopped()) { + break; + } + $users = $this->getUsers($this->chunks_size); + $users_count = count($users); + + if(is_array($users)) { + foreach($users as $user) { + $subscriber = $this->importSubscriber($user); + if (!empty($subscriber)) { + $imported_subscribers_count++; + $this->importSubscriberSegments($subscriber, $user['user_id']); + $this->importSubscriberCustomFields($subscriber, $user); + } + } + } + $this->progressbar->incrementCurrentCount($users_count); + } while(($users != null) && ($users_count > 0)); + + $this->log(sprintf(_n("%d subscriber imported", "%d subscribers imported", $imported_subscribers_count, Env::$plugin_name), $imported_subscribers_count)); + } + + /** + * Get the Mailpoet 2 users + * + * @global object $wpdb + * @param int $limit Number of users max + * @return array Users + */ + private function getUsers($limit) { + global $wpdb; + $users = array(); + + $last_id = Setting::getValue('last_imported_user_id', 0); + $table = $wpdb->prefix . 'wysija_user'; + $sql = " + SELECT u.* + FROM `$table` u + WHERE u.user_id > '$last_id' + ORDER BY u.user_id + LIMIT $limit + "; + $users = $wpdb->get_results($sql, ARRAY_A); + + return $users; + } + + /** + * Import a subscriber + * + * @param array $user_data User data + * @return Subscriber + */ + private function importSubscriber($user_data) { + $subscriber = Subscriber::createOrUpdate(array( + 'id' => $user_data['user_id'], + 'wp_user_id' => !empty($user_data['wpuser_id'])? $user_data['wpuser_id'] : null, + 'email' => $user_data['email'], + 'first_name' => $user_data['firstname'], + 'last_name' => $user_data['lastname'], + 'status' => $this->mapUserStatus($user_data['status']), + 'created_at' => Helpers::mysql_date($user_data['created_at']), + 'subscribed_ip' => !empty($user_data['ip'])? $user_data['ip'] : null, + 'confirmed_ip' => !empty($user_data['confirmed_ip'])? $user_data['confirmed_ip'] : null, + 'confirmed_at' => !empty($user_data['confirmed_at'])? Helpers::mysql_date($user_data['confirmed_at']) : null, + )); + Setting::setValue('last_imported_user_id', $user_data['user_id']); + return $subscriber; + } + + /** + * Map the MailPoet 2 user status with MailPoet 3 + * + * @param int $mp2_user_status MP2 user status + * @return string MP3 user status + */ + private function mapUserStatus($mp2_user_status) { + switch($mp2_user_status) { + case 0: $status = 'unconfirmed'; break; + case 1: $status = 'subscribed'; break; + case -1: $status = 'unsubscribed'; break; + default: $status = 'unconfirmed'; + } + return $status; + } + + /** + * Import the segments for a subscriber + * + * @param Subscriber $subscriber MP3 subscriber + * @param int $user_id MP2 user ID + */ + private function importSubscriberSegments($subscriber, $user_id) { + $user_lists = $this->getUserLists($user_id); + foreach($user_lists as $user_list) { + $this->importSubscriberSegment($subscriber->id, $user_list); + } + } + + /** + * Get the lists for a user + * + * @global object $wpdb + * @param int $user_id User ID + * @return array Users Lists + */ + private function getUserLists($user_id) { + global $wpdb; + $user_lists = array(); + + $table = $wpdb->prefix . 'wysija_user_list'; + $sql = " + SELECT ul.list_id, ul.sub_date, ul.unsub_date + FROM `$table` ul + WHERE ul.user_id = '$user_id' + "; + $user_lists = $wpdb->get_results($sql, ARRAY_A); + + return $user_lists; + } + + /** + * Import a subscriber segment + * + * @param int $subscriber_id + * @param array $user_list + * @return SubscriberSegment + */ + private function importSubscriberSegment($subscriber_id, $user_list) { + $data = array( + 'subscriber_id' => $subscriber_id, + 'segment_id' => $user_list['list_id'], + 'status' => empty($user_list['unsub_date'])? 'subscribed' : 'unsubscribed', + 'created_at' => Helpers::mysql_date($user_list['sub_date']), + 'updated_at' => !empty($user_list['unsub_date'])? Helpers::mysql_date($user_list['unsub_date']) : null, + ); + $subscriberSegment = new SubscriberSegment(); + $subscriberSegment->createOrUpdate($data); + return $subscriberSegment; + } + + /** + * Import the custom fields values for a subscriber + * + * @param Subscriber $subscriber MP3 subscriber + * @param array $user MP2 user + */ + private function importSubscriberCustomFields($subscriber, $user) { + $imported_custom_fields = $this->getImportedCustomFields(); + foreach($imported_custom_fields as $custom_field) { + $custom_field_column = 'cf_' . $custom_field['id']; + if(isset($custom_field_column)) { + $this->importSubscriberCustomField($subscriber->id, $custom_field, $user[$custom_field_column]); + } + } + } + + /** + * Get the imported custom fields + * + * @global object $wpdb + * @return array Imported custom fields + * + */ + private function getImportedCustomFields() { + global $wpdb; + $table = MP_CUSTOM_FIELDS_TABLE; + $sql = " + SELECT cf.id, cf.name, cf.type + FROM `$table` cf + "; + $custom_fields = $wpdb->get_results($sql, ARRAY_A); + return $custom_fields; + } + + /** + * Import a subscriber custom field + * + * @param int $subscriber_id Subscriber ID + * @param int $custom_field Custom field + * @param string $custom_field_value Custom field value + * @return SubscriberCustomField + */ + private function importSubscriberCustomField($subscriber_id, $custom_field, $custom_field_value) { + if($custom_field['type'] == 'date') { + $value = Helpers::mysql_date($custom_field_value); // Convert the date field + } else { + $value = $custom_field_value; + } + $data = array( + 'subscriber_id' => $subscriber_id, + 'custom_field_id' => $custom_field['id'], + 'value' => isset($value)? $value : '', + ); + $subscriberCustomField = new SubscriberCustomField(); + $subscriberCustomField->createOrUpdate($data); + return $subscriberCustomField; + } + + /** + * Get the mapping between the MP2 and the imported MP3 IDs + * + * @param string $model Model (segment,...) + * @return array Mapping + */ + private function getImportedMapping($model) { + $mappings = array(); + $mapping_relations = ImportedDataMapping::where('type', $model)->findArray(); + foreach($mapping_relations as $relation) { + $mappings[$relation['old_id']] = $relation['new_id']; + } + return $mappings; + } + } diff --git a/lib/Config/Migrator.php b/lib/Config/Migrator.php index 10e22d0e1d..1b9f1f2224 100644 --- a/lib/Config/Migrator.php +++ b/lib/Config/Migrator.php @@ -33,7 +33,8 @@ class Migrator { 'statistics_clicks', 'statistics_opens', 'statistics_unsubscribes', - 'statistics_forms' + 'statistics_forms', + 'imported_data_mapping' ); } @@ -79,7 +80,7 @@ class Migrator { function settings() { $attributes = array( 'id mediumint(9) NOT NULL AUTO_INCREMENT,', - 'name varchar(20) NOT NULL,', + 'name varchar(50) NOT NULL,', 'value longtext,', 'created_at TIMESTAMP NULL,', 'updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,', @@ -363,6 +364,18 @@ class Migrator { return $this->sqlify(__FUNCTION__, $attributes); } + function importedDataMapping() { + $attributes = array( + 'old_id mediumint(9) NOT NULL,', + 'type varchar(50) NOT NULL,', + 'new_id mediumint(9) NOT NULL,', + 'created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,', + 'PRIMARY KEY (old_id, type),', + 'KEY new_id (new_id)' + ); + return $this->sqlify(__FUNCTION__, $attributes); + } + private function sqlify($model, $attributes) { $table = $this->prefix . Helpers::camelCaseToUnderscore($model); diff --git a/lib/Models/ImportedDataMapping.php b/lib/Models/ImportedDataMapping.php new file mode 100644 index 0000000000..550dbccba4 --- /dev/null +++ b/lib/Models/ImportedDataMapping.php @@ -0,0 +1,15 @@ +hydrate($data); + return $relation->save(); + } + +} \ No newline at end of file diff --git a/lib/Util/Helpers.php b/lib/Util/Helpers.php index fb9922a7d0..2888c8f4e4 100644 --- a/lib/Util/Helpers.php +++ b/lib/Util/Helpers.php @@ -136,4 +136,15 @@ class Helpers { static function splitObject($object = array()) { return explode(self::DIVIDER, $object); } + + /** + * Convert a timestamp to a Mysql datetime + * + * @param int $timestamp Timestamp + * @return string Datetime + */ + static function mysql_date($timestamp) { + return date('Y-m-d H:i:s', $timestamp); + } + } \ No newline at end of file