Import the MP2 users lists

Import the MP2 users
Add the table "imported_data_mapping"
Add the function Helpers::mysql_date

assets/js/src/mp2migrator.js

@@ -74,7 +74,7 @@
         dataType: 'json'
       }).always(function (result) {
         // Move the progress bar
-        var progress = Number(result.current) / Number(result.total) * 100;
+        var progress = Math.round(Number(result.current) / Number(result.total) * 100);
         $('#progressbar').progressbar('option', 'value', progress);
         $('#progresslabel').html(progress + '%');
         if(that.is_logging) {

lib/API/Endpoints/MP2Migrator.php

@@ -18,7 +18,7 @@ class MP2Migrator extends APIEndpoint {
    */
   public function import($data) {
     try {
-      $process = $this->MP2Migrator->import(json_decode($data, true));
+      $process = $this->MP2Migrator->import($data);
       return $this->successResponse($process);
     } catch(\Exception $e) {
       return $this->errorResponse(array(

lib/Config/Database.php

@@ -77,6 +77,7 @@ class Database {
       $statistics_opens = Env::$db_prefix . 'statistics_opens';
       $statistics_unsubscribes = Env::$db_prefix . 'statistics_unsubscribes';
       $statistics_forms = Env::$db_prefix . 'statistics_forms';
+      $imported_data_mapping = Env::$db_prefix . 'imported_data_mapping';
 
       define('MP_SETTINGS_TABLE', $settings);
       define('MP_SEGMENTS_TABLE', $segments);
@@ -98,6 +99,7 @@ class Database {
       define('MP_STATISTICS_OPENS_TABLE', $statistics_opens);
       define('MP_STATISTICS_UNSUBSCRIBES_TABLE', $statistics_unsubscribes);
       define('MP_STATISTICS_FORMS_TABLE', $statistics_forms);
+      define('MP_IMPORTED_DATA_MAPPING_TABLE', $imported_data_mapping);
     }
   }
 }

lib/Config/MP2Migrator.php

@@ -4,6 +4,14 @@ namespace MailPoet\Config;
 
 use MailPoet\Util\ProgressBar;
 use MailPoet\Models\Setting;
+use MailPoet\Models\Segment;
+use MailPoet\Models\Subscriber;
+use MailPoet\Models\CustomField;
+use MailPoet\Models\SubscriberSegment;
+use MailPoet\Models\SubscriberCustomField;
+use MailPoet\Models\ImportedDataMapping;
+use MailPoet\Config\Activator;
+use MailPoet\Util\Helpers;
 
 if(!defined('ABSPATH')) exit;
 
@@ -12,6 +20,7 @@ class MP2Migrator {
   private $log_file;
   public $log_file_url;
   public $progressbar;
+  private $chunks_size = 10; // To import the data by batch
 
   public function __construct() {
     $log_filename = 'mp2migration.log';
@@ -94,23 +103,18 @@ class MP2Migrator {
    * @return boolean Result
    */
   public function import() {
+    set_time_limit(7200); // Timeout = 2 hours
     ob_start();
     $this->emptyLog();
     $this->log(sprintf("=== START IMPORT %s ===", date('Y-m-d H:i:s')));
-    Setting::setValue('mailpoet_stopImport', false); // Reset the stop import action
+    Setting::setValue('import_stopped', false); // Reset the stop import action
+    $this->eraseMP3Data();
     
     $this->displayDataToMigrate();
     
-    // TODO to remove, for testing only
+    $this->importSegments();
-    $this->progressbar->setTotalCount(0);
+    $this->importCustomFields();
-    $this->progressbar->setTotalCount(10);
+    $this->importSubscribers();
-    for($i = 0; $i < 10; $i++) {
-      $this->progressbar->incrementCurrentCount(1);
-      usleep(300000);
-      if($this->importStopped()) {
-        return;
-      }
-    }
     
     $this->log(sprintf("=== END IMPORT %s ===", date('Y-m-d H:i:s')));
     $result = ob_get_contents();
@@ -126,12 +130,32 @@ class MP2Migrator {
     file_put_contents($this->log_file, '');
   }
   
+  /**
+   * Erase all the MailPoet 3 data
+   * 
+   */
+  private function eraseMP3Data() {
+    Activator::deactivate();
+    Activator::activate();
+    $this->resetMigrationCounters();
+    $this->log(__("MailPoet data erased", Env::$plugin_name));
+  }
+  
+  /**
+   * Reset the migration counters
+   * 
+   */
+  private function resetMigrationCounters() {
+    Setting::setValue('last_imported_user_id', 0);
+    Setting::setValue('last_imported_list_id', 0);
+  }
+  
   /**
    * Stop the import
    * 
    */
   public function stopImport() {
-    Setting::setValue('mailpoet_stopImport', true);
+    Setting::setValue('import_stopped', true);
     $this->log(__('IMPORT STOPPED BY USER', Env::$plugin_name));
   }
 
@@ -141,7 +165,7 @@ class MP2Migrator {
    * @return boolean Import must stop or not
    */
   private function importStopped() {
-    return Setting::getValue('mailpoet_stopImport');
+    return Setting::getValue('import_stopped', false);
   }
 
   /**
@@ -166,16 +190,16 @@ class MP2Migrator {
     
     $result .= __('MailPoet 2 data found:', Env::$plugin_name) . "\n";
     
+    // User Lists
+    $usersListsCount = $this->rowsCount('wysija_list');
+    $totalCount += $usersListsCount;
+    $result .= sprintf(_n('%d subscribers list', '%d subscribers lists', $usersListsCount, Env::$plugin_name), $usersListsCount) . "\n";
+    
     // Users
     $usersCount = $this->rowsCount('wysija_user');
     $totalCount += $usersCount;
     $result .= sprintf(_n('%d subscriber', '%d subscribers', $usersCount, Env::$plugin_name), $usersCount) . "\n";
     
-    // User Lists
-    $usersListsCount = $this->rowsCount('wysija_user_list');
-    $totalCount += $usersListsCount;
-    $result .= sprintf(_n('%d subscribers list', '%d subscribers lists', $usersListsCount, Env::$plugin_name), $usersListsCount) . "\n";
-    
     // Emails
     $emailsCount = $this->rowsCount('wysija_email');
     $totalCount += $emailsCount;
@@ -194,6 +218,7 @@ class MP2Migrator {
   /**
    * Count the number of rows in a table
    * 
+   * @global object $wpdb
    * @param string $table Table
    * @return int Number of rows found
    */
@@ -207,4 +232,423 @@ class MP2Migrator {
     return $count;
   }
   
+  /**
+   * Import the subscribers segments
+   * 
+   */
+  private function importSegments() {
+    $imported_segments_count = 0;
+    if($this->importStopped()) {
+      return;
+    }
+    $this->log(__("Importing segments...", Env::$plugin_name));
+    do {
+      if($this->importStopped()) {
+        break;
+      }
+      $lists = $this->getLists($this->chunks_size);
+      $lists_count = count($lists);
+
+      if(is_array($lists)) {
+        foreach($lists as $list) {
+          $segment = $this->importSegment($list);
+          if(!empty($segment)) {
+            $imported_segments_count++;
+          }
+        }
+      }
+      $this->progressbar->incrementCurrentCount($lists_count);
+    } while(($lists != null) && ($lists_count > 0));
+    
+    $this->log(sprintf(_n("%d segment imported", "%d segments imported", $imported_segments_count, Env::$plugin_name), $imported_segments_count));
+  }
+  
+  /**
+   * Get the Mailpoet 2 users lists
+   * 
+   * @global object $wpdb
+   * @param int $limit Number of users max
+   * @return array Users Lists
+   */
+  private function getLists($limit) {
+    global $wpdb;
+    $lists = array();
+
+    $last_id = Setting::getValue('last_imported_list_id', 0);
+    $table = $wpdb->prefix . 'wysija_list';
+    $sql = "
+      SELECT l.list_id, l.name, l.description, l.is_enabled, l.created_at
+      FROM `$table` l
+      WHERE l.list_id > '$last_id'
+      ORDER BY l.list_id
+      LIMIT $limit
+      ";
+    $lists = $wpdb->get_results($sql, ARRAY_A);
+    
+    return $lists;
+  }
+  
+  /**
+   * Import a segment
+   * 
+   * @param array $list_data List data
+   * @return Segment
+   */
+  private function importSegment($list_data) {
+    $segment = Segment::createOrUpdate(array(
+      'id' => $list_data['list_id'],
+      'name' => $list_data['name'],
+      'type' => $list_data['is_enabled']? 'default': 'wp_users',
+      'description' => $list_data['description'],
+      'created_at' => Helpers::mysql_date($list_data['created_at']),
+    ));
+    Setting::setValue('last_imported_list_id', $list_data['list_id']);
+    return $segment;
+  }
+  
+  /**
+   * Import the custom fields
+   * 
+   */
+  private function importCustomFields() {
+    $imported_custom_fields_count = 0;
+    if($this->importStopped()) {
+      return;
+    }
+    $this->log(__("Importing custom fields...", Env::$plugin_name));
+    $custom_fields = $this->getCustomFields();
+ 
+    foreach($custom_fields as $custom_field) {
+      $result = $this->importCustomField($custom_field);
+      if (!empty($result)) {
+        $imported_custom_fields_count++;
+      }
+    }
+    
+    $this->log(sprintf(_n("%d custom field imported", "%d custom fields imported", $imported_custom_fields_count, Env::$plugin_name), $imported_custom_fields_count));
+  }
+  
+  /**
+   * Get the Mailpoet 2 custom fields
+   * 
+   * @global object $wpdb
+   * @return array Custom fields
+   */
+  private function getCustomFields() {
+    global $wpdb;
+    $custom_fields = array();
+
+    $table = $wpdb->prefix . 'wysija_custom_field';
+    $sql = "
+      SELECT cf.id, cf.name, cf.type, cf.required, cf.settings
+      FROM `$table` cf
+      ";
+    $custom_fields = $wpdb->get_results($sql, ARRAY_A);
+    
+    return $custom_fields;
+  }
+  
+  /**
+   * Import a custom field
+   * 
+   * @param array $custom_field MP2 custom field
+   * @return CustomField
+   */
+  private function importCustomField($custom_field) {
+    $data = array(
+      'id' => $custom_field['id'],
+      'name' => $custom_field['name'],
+      'type' => $this->mapCustomFieldType($custom_field['type']),
+      'params' => $this->mapCustomFieldParams($custom_field),
+    );
+    $customField = new CustomField();
+    $customField->createOrUpdate($data);
+    return $customField;
+  }
+  
+  /**
+   * Map the MailPoet 2 custom field type with the MailPoet custom field type
+   * 
+   * @param string $MP2type MP2 custom field type
+   * @return string MP3 custom field type
+   */
+  private function mapCustomFieldType($MP2type) {
+    $type = '';
+    switch($MP2type) {
+      case 'input':
+        $type = 'text';
+        break;
+      default:
+        $type = $MP2type;
+    }
+    return $type;
+  }
+  
+  /**
+   * Map the MailPoet 2 custom field settings with the MailPoet custom field params
+   * 
+   * @param array $custom_field MP2 custom field
+   * @return string serialized MP3 custom field params
+   */
+  private function mapCustomFieldParams($custom_field) {
+    $params = unserialize($custom_field['settings']);
+    $params['label'] = $custom_field['name'];
+    if(isset($params['validate'])) {
+      $params['validate'] = $this->mapCustomFieldValidateValue($params['validate']);
+    }
+    if(isset($params['date_order'])) { // Convert the date_order field
+      $params['date_format'] = strtoupper($params['date_order']);
+      unset($params['date_order']);
+    }
+    return $params;
+  }
+  
+  /**
+   * Map the validate value
+   * 
+   * @param string $MP2value MP2 value
+   * @return string MP3 value
+   */
+  private function mapCustomFieldValidateValue($MP2value) {
+    $value = '';
+    switch($MP2value) {
+      case 'onlyLetterSp':
+      case 'onlyLetterNumber':
+        $value = 'alphanum';
+        break;
+      case 'onlyNumberSp':
+        $value = 'number';
+        break;
+      case 'phone':
+        $value = 'phone';
+        break;
+    }
+    return $value;
+  }
+  
+  /**
+   * Import the subscribers
+   * 
+   */
+  private function importSubscribers() {
+    $imported_subscribers_count = 0;
+    if($this->importStopped()) {
+      return;
+    }
+    $this->log(__("Importing subscribers...", Env::$plugin_name));
+    do {
+      if($this->importStopped()) {
+        break;
+      }
+      $users = $this->getUsers($this->chunks_size);
+      $users_count = count($users);
+
+      if(is_array($users)) {
+        foreach($users as $user) {
+          $subscriber = $this->importSubscriber($user);
+          if (!empty($subscriber)) {
+            $imported_subscribers_count++;
+            $this->importSubscriberSegments($subscriber, $user['user_id']);
+            $this->importSubscriberCustomFields($subscriber, $user);
+          }
+        }
+      }
+      $this->progressbar->incrementCurrentCount($users_count);
+    } while(($users != null) && ($users_count > 0));
+    
+    $this->log(sprintf(_n("%d subscriber imported", "%d subscribers imported", $imported_subscribers_count, Env::$plugin_name), $imported_subscribers_count));
+  }
+  
+  /**
+   * Get the Mailpoet 2 users
+   * 
+   * @global object $wpdb
+   * @param int $limit Number of users max
+   * @return array Users
+   */
+  private function getUsers($limit) {
+    global $wpdb;
+    $users = array();
+
+    $last_id = Setting::getValue('last_imported_user_id', 0);
+    $table = $wpdb->prefix . 'wysija_user';
+    $sql = "
+      SELECT u.*
+      FROM `$table` u
+      WHERE u.user_id > '$last_id'
+      ORDER BY u.user_id
+      LIMIT $limit
+      ";
+    $users = $wpdb->get_results($sql, ARRAY_A);
+    
+    return $users;
+  }
+  
+  /**
+   * Import a subscriber
+   * 
+   * @param array $user_data User data
+   * @return Subscriber
+   */
+  private function importSubscriber($user_data) {
+    $subscriber = Subscriber::createOrUpdate(array(
+      'id' => $user_data['user_id'],
+      'wp_user_id' => !empty($user_data['wpuser_id'])? $user_data['wpuser_id'] : null,
+      'email' => $user_data['email'],
+      'first_name' => $user_data['firstname'],
+      'last_name' => $user_data['lastname'],
+      'status' => $this->mapUserStatus($user_data['status']),
+      'created_at' => Helpers::mysql_date($user_data['created_at']),
+      'subscribed_ip' => !empty($user_data['ip'])? $user_data['ip'] : null,
+      'confirmed_ip' => !empty($user_data['confirmed_ip'])? $user_data['confirmed_ip'] : null,
+      'confirmed_at' => !empty($user_data['confirmed_at'])? Helpers::mysql_date($user_data['confirmed_at']) : null,
+    ));
+    Setting::setValue('last_imported_user_id', $user_data['user_id']);
+    return $subscriber;
+  }
+  
+  /**
+   * Map the MailPoet 2 user status with MailPoet 3
+   * 
+   * @param int $mp2_user_status MP2 user status
+   * @return string MP3 user status
+   */
+  private function mapUserStatus($mp2_user_status) {
+    switch($mp2_user_status) {
+      case 0: $status = 'unconfirmed'; break;
+      case 1: $status = 'subscribed'; break;
+      case -1: $status = 'unsubscribed'; break;
+      default: $status = 'unconfirmed';
+    }
+    return $status;
+  }
+  
+  /**
+   * Import the segments for a subscriber
+   * 
+   * @param Subscriber $subscriber MP3 subscriber
+   * @param int $user_id MP2 user ID
+   */
+  private function importSubscriberSegments($subscriber, $user_id) {
+    $user_lists = $this->getUserLists($user_id);
+    foreach($user_lists as $user_list) {
+      $this->importSubscriberSegment($subscriber->id, $user_list);
+    }
+  }
+  
+  /**
+   * Get the lists for a user
+   * 
+   * @global object $wpdb
+   * @param int $user_id User ID
+   * @return array Users Lists
+   */
+  private function getUserLists($user_id) {
+    global $wpdb;
+    $user_lists = array();
+
+    $table = $wpdb->prefix . 'wysija_user_list';
+    $sql = "
+      SELECT ul.list_id, ul.sub_date, ul.unsub_date
+      FROM `$table` ul
+      WHERE ul.user_id = '$user_id'
+      ";
+    $user_lists = $wpdb->get_results($sql, ARRAY_A);
+    
+    return $user_lists;
+  }
+  
+  /**
+   * Import a subscriber segment
+   * 
+   * @param int $subscriber_id
+   * @param array $user_list
+   * @return SubscriberSegment
+   */
+  private function importSubscriberSegment($subscriber_id, $user_list) {
+    $data = array(
+      'subscriber_id' => $subscriber_id,
+      'segment_id' => $user_list['list_id'],
+      'status' => empty($user_list['unsub_date'])? 'subscribed' : 'unsubscribed',
+      'created_at' => Helpers::mysql_date($user_list['sub_date']),
+      'updated_at' => !empty($user_list['unsub_date'])? Helpers::mysql_date($user_list['unsub_date']) : null,
+    );
+    $subscriberSegment = new SubscriberSegment();
+    $subscriberSegment->createOrUpdate($data);
+    return $subscriberSegment;
+  }
+  
+  /**
+   * Import the custom fields values for a subscriber
+   * 
+   * @param Subscriber $subscriber MP3 subscriber
+   * @param array $user MP2 user
+   */
+  private function importSubscriberCustomFields($subscriber, $user) {
+    $imported_custom_fields = $this->getImportedCustomFields();
+    foreach($imported_custom_fields as $custom_field) {
+      $custom_field_column = 'cf_' . $custom_field['id'];
+      if(isset($custom_field_column)) {
+        $this->importSubscriberCustomField($subscriber->id, $custom_field, $user[$custom_field_column]);
+      }
+    }
+  }
+  
+  /**
+   * Get the imported custom fields
+   * 
+   * @global object $wpdb
+   * @return array Imported custom fields
+   * 
+   */
+  private function getImportedCustomFields() {
+    global $wpdb;
+    $table = MP_CUSTOM_FIELDS_TABLE;
+    $sql = "
+      SELECT cf.id, cf.name, cf.type
+      FROM `$table` cf
+      ";
+    $custom_fields = $wpdb->get_results($sql, ARRAY_A);
+    return $custom_fields;
+  }
+  
+  /**
+   * Import a subscriber custom field
+   * 
+   * @param int $subscriber_id Subscriber ID
+   * @param int $custom_field Custom field
+   * @param string $custom_field_value Custom field value
+   * @return SubscriberCustomField
+   */
+  private function importSubscriberCustomField($subscriber_id, $custom_field, $custom_field_value) {
+    if($custom_field['type'] == 'date') {
+      $value = Helpers::mysql_date($custom_field_value); // Convert the date field
+    } else {
+      $value = $custom_field_value;
+    }
+    $data = array(
+      'subscriber_id' => $subscriber_id,
+      'custom_field_id' => $custom_field['id'],
+      'value' => isset($value)? $value : '',
+    );
+    $subscriberCustomField = new SubscriberCustomField();
+    $subscriberCustomField->createOrUpdate($data);
+    return $subscriberCustomField;
+  }
+  
+  /**
+   * Get the mapping between the MP2 and the imported MP3 IDs
+   * 
+   * @param string $model Model (segment,...)
+   * @return array Mapping
+   */
+  private function getImportedMapping($model) {
+    $mappings = array();
+    $mapping_relations = ImportedDataMapping::where('type', $model)->findArray();
+    foreach($mapping_relations as $relation) {
+      $mappings[$relation['old_id']] = $relation['new_id'];
+    }
+    return $mappings;
+  }
+  
 }

lib/Config/Migrator.php

@@ -33,7 +33,8 @@ class Migrator {
       'statistics_clicks',
       'statistics_opens',
       'statistics_unsubscribes',
-      'statistics_forms'
+      'statistics_forms',
+      'imported_data_mapping'
     );
   }
 
@@ -79,7 +80,7 @@ class Migrator {
   function settings() {
     $attributes = array(
       'id mediumint(9) NOT NULL AUTO_INCREMENT,',
-      'name varchar(20) NOT NULL,',
+      'name varchar(50) NOT NULL,',
       'value longtext,',
       'created_at TIMESTAMP NULL,',
       'updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,',
@@ -363,6 +364,18 @@ class Migrator {
     return $this->sqlify(__FUNCTION__, $attributes);
   }
 
+  function importedDataMapping() {
+    $attributes = array(
+      'old_id mediumint(9) NOT NULL,',
+      'type varchar(50) NOT NULL,',
+      'new_id mediumint(9) NOT NULL,',
+      'created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,',
+      'PRIMARY KEY (old_id, type),',
+      'KEY new_id (new_id)'
+    );
+    return $this->sqlify(__FUNCTION__, $attributes);
+  }
+
   private function sqlify($model, $attributes) {
     $table = $this->prefix . Helpers::camelCaseToUnderscore($model);
 

lib/Models/ImportedDataMapping.php

@@ -0,0 +1,15 @@
+<?php
+namespace MailPoet\Models;
+
+if(!defined('ABSPATH')) exit;
+
+class ImportedDataMapping extends Model {
+  public static $_table = MP_IMPORTED_DATA_MAPPING_TABLE;
+
+  static function create($data = array()) {
+    $relation = parent::create();
+    $relation->hydrate($data);
+    return $relation->save();
+  }
+
+}

lib/Util/Helpers.php

@@ -136,4 +136,15 @@ class Helpers {
   static function splitObject($object = array()) {
     return explode(self::DIVIDER, $object);
   }
+  
+  /**
+   * Convert a timestamp to a Mysql datetime
+   * 
+   * @param int $timestamp Timestamp
+   * @return string Datetime
+   */
+  static function mysql_date($timestamp) {
+    return date('Y-m-d H:i:s', $timestamp);
+  }
+  
 }