Use Doctrine instead of Paris in Import

[MAILPOET-3378]

lib/API/JSON/v1/ImportExport.php

@@ -5,10 +5,16 @@ namespace MailPoet\API\JSON\v1;
 use MailPoet\API\JSON\Endpoint as APIEndpoint;
 use MailPoet\Config\AccessControl;
 use MailPoet\Cron\Workers\WooCommerceSync;
+use MailPoet\CustomFields\CustomFieldsRepository;
 use MailPoet\Models\ScheduledTask;
 use MailPoet\Models\Segment;
+use MailPoet\Newsletter\Options\NewsletterOptionsRepository;
 use MailPoet\Segments\WP;
+use MailPoet\Subscribers\ImportExport\Import\Import;
 use MailPoet\Subscribers\ImportExport\Import\MailChimp;
+use MailPoet\Subscribers\SubscriberCustomFieldRepository;
+use MailPoet\Subscribers\SubscriberSegmentRepository;
+use MailPoet\Subscribers\SubscribersRepository;
 use MailPoetVendor\Carbon\Carbon;
 
 class ImportExport extends APIEndpoint {
@@ -16,12 +22,39 @@ class ImportExport extends APIEndpoint {
   /** @var WP */
   private $wpSegment;
 
+  /** @var CustomFieldsRepository */
+  private $customFieldsRepository;
+
+  /** @var NewsletterOptionsRepository */
+  private $newsletterOptionsRepository;
+
+  /** @var SubscriberCustomFieldRepository */
+  private $subscriberCustomFieldRepository;
+
+  /** @var SubscribersRepository */
+  private $subscriberRepository;
+
+  /** @var SubscriberSegmentRepository */
+  private $subscriberSegmentRepository;
+
   public $permissions = [
     'global' => AccessControl::PERMISSION_MANAGE_SUBSCRIBERS,
   ];
 
-  public function __construct(WP $wpSegment) {
+  public function __construct(
+    WP $wpSegment,
+    CustomFieldsRepository $customFieldsRepository,
+    NewsletterOptionsRepository $newsletterOptionsRepository,
+    SubscriberCustomFieldRepository $subscriberCustomFieldRepository,
+    SubscribersRepository $subscribersRepository,
+    SubscriberSegmentRepository $subscriberSegmentRepository
+  ) {
     $this->wpSegment = $wpSegment;
+    $this->customFieldsRepository = $customFieldsRepository;
+    $this->newsletterOptionsRepository = $newsletterOptionsRepository;
+    $this->subscriberCustomFieldRepository = $subscriberCustomFieldRepository;
+    $this->subscriberRepository = $subscribersRepository;
+    $this->subscriberSegmentRepository = $subscriberSegmentRepository;
   }
 
   public function getMailChimpLists($data) {
@@ -63,8 +96,13 @@ class ImportExport extends APIEndpoint {
 
   public function processImport($data) {
     try {
-      $import = new \MailPoet\Subscribers\ImportExport\Import\Import(
+      $import = new Import(
         $this->wpSegment,
+        $this->customFieldsRepository,
+        $this->newsletterOptionsRepository,
+        $this->subscriberCustomFieldRepository,
+        $this->subscriberRepository,
+        $this->subscriberSegmentRepository,
         json_decode($data, true)
       );
       $process = $import->process();

lib/Entities/SubscriberEntity.php

@@ -24,6 +24,9 @@ class SubscriberEntity {
   const STATUS_UNCONFIRMED = 'unconfirmed';
   const STATUS_UNSUBSCRIBED = 'unsubscribed';
 
+  public const OBSOLETE_LINK_TOKEN_LENGTH = 6;
+  public const LINK_TOKEN_LENGTH = 32;
+
   use AutoincrementedIdTrait;
   use CreatedAtTrait;
   use UpdatedAtTrait;

lib/Newsletter/Options/NewsletterOptionsRepository.php

@@ -3,7 +3,9 @@
 namespace MailPoet\Newsletter\Options;
 
 use MailPoet\Doctrine\Repository;
+use MailPoet\Entities\NewsletterEntity;
 use MailPoet\Entities\NewsletterOptionEntity;
+use MailPoet\Entities\NewsletterOptionFieldEntity;
 
 /**
  * @extends Repository<NewsletterOptionEntity>
@@ -12,4 +14,20 @@ class NewsletterOptionsRepository extends Repository {
   protected function getEntityClassName() {
     return NewsletterOptionEntity::class;
   }
+
+  public function findWelcomeNotificationsForSegments(array $segmentIds): array {
+    return $this->entityManager->createQueryBuilder()
+      ->select('no')
+      ->from(NewsletterOptionEntity::class, 'no')
+      ->join('no.newsletter', 'n')
+      ->join('no.optionField', 'nof')
+      ->where('n.deletedAt IS NULL')
+      ->andWhere('n.type = :typeWelcome')
+      ->andWhere('nof.name = :nameSegment')
+      ->andWhere('no.value IN (:segmentIds)')
+      ->setParameter('typeWelcome', NewsletterEntity::TYPE_WELCOME)
+      ->setParameter('nameSegment', NewsletterOptionFieldEntity::NAME_SEGMENT)
+      ->setParameter('segmentIds', $segmentIds)
+      ->getQuery()->getResult();
+  }
 }

lib/Subscribers/ImportExport/Import/Import.php

@@ -2,15 +2,17 @@
 
 namespace MailPoet\Subscribers\ImportExport\Import;
 
-use MailPoet\Models\CustomField;
+use MailPoet\CustomFields\CustomFieldsRepository;
+use MailPoet\Entities\CustomFieldEntity;
+use MailPoet\Entities\SubscriberEntity;
 use MailPoet\Models\ModelValidator;
-use MailPoet\Models\Newsletter;
+use MailPoet\Newsletter\Options\NewsletterOptionsRepository;
-use MailPoet\Models\Subscriber;
-use MailPoet\Models\SubscriberCustomField;
-use MailPoet\Models\SubscriberSegment;
 use MailPoet\Segments\WP;
 use MailPoet\Subscribers\ImportExport\ImportExportFactory;
 use MailPoet\Subscribers\Source;
+use MailPoet\Subscribers\SubscriberCustomFieldRepository;
+use MailPoet\Subscribers\SubscriberSegmentRepository;
+use MailPoet\Subscribers\SubscribersRepository;
 use MailPoet\Util\DateConverter;
 use MailPoet\Util\Helpers;
 use MailPoet\Util\Security;
@@ -35,8 +37,36 @@ class Import {
   /** @var WP */
   private $wpSegment;
 
-  public function __construct(WP $wpSegment, $data) {
+  /** @var CustomFieldsRepository */
+  private $customFieldsRepository;
+
+  /** @var NewsletterOptionsRepository */
+  private $newsletterOptionsRepository;
+
+  /** @var SubscriberCustomFieldRepository */
+  private $subscriberCustomFieldRepository;
+
+  /** @var SubscribersRepository */
+  private $subscriberRepository;
+
+  /** @var SubscriberSegmentRepository */
+  private $subscriberSegmentRepository;
+
+  public function __construct(
+    WP $wpSegment,
+    CustomFieldsRepository $customFieldsRepository,
+    NewsletterOptionsRepository $newsletterOptionsRepository,
+    SubscriberCustomFieldRepository $subscriberCustomFieldRepository,
+    SubscribersRepository $subscriberRepository,
+    SubscriberSegmentRepository $subscriberSegmentRepository,
+    array $data
+  ) {
     $this->wpSegment = $wpSegment;
+    $this->customFieldsRepository = $customFieldsRepository;
+    $this->newsletterOptionsRepository = $newsletterOptionsRepository;
+    $this->subscriberCustomFieldRepository = $subscriberCustomFieldRepository;
+    $this->subscriberRepository = $subscriberRepository;
+    $this->subscriberSegmentRepository = $subscriberSegmentRepository;
     $this->validateImportData($data);
     $this->subscribersData = $this->transformSubscribersData(
       $data['subscribers'],
@@ -56,7 +86,7 @@ class Import {
     $this->createdAt = Carbon::createFromTimestamp(WPFunctions::get()->currentTime('timestamp'));
     $this->updatedAt = Carbon::createFromTimestamp(WPFunctions::get()->currentTime('timestamp') + 1);
     $this->requiredSubscribersFields = [
-      'status' => Subscriber::STATUS_SUBSCRIBED,
+      'status' => SubscriberEntity::STATUS_SUBSCRIBED,
       'first_name' => '',
       'last_name' => '',
       'created_at' => $this->createdAt,
@@ -111,23 +141,22 @@ class Import {
         $newSubscribers = $this->setLinkToken($newSubscribers);
         $createdSubscribers =
           $this->createOrUpdateSubscribers(
-            'create',
             $newSubscribers,
             $this->subscribersCustomFields
           );
       }
+
       if ($existingSubscribers['data'] && $this->updateSubscribers) {
         $allowedStatuses = [
-          Subscriber::STATUS_SUBSCRIBED,
+          SubscriberEntity::STATUS_SUBSCRIBED,
-          Subscriber::STATUS_UNSUBSCRIBED,
+          SubscriberEntity::STATUS_UNSUBSCRIBED,
-          Subscriber::STATUS_INACTIVE,
+          SubscriberEntity::STATUS_INACTIVE,
         ];
         if (in_array($this->existingSubscribersStatus, $allowedStatuses, true)) {
           $existingSubscribers = $this->addField($existingSubscribers, 'status', $this->existingSubscribersStatus);
         }
         $updatedSubscribers =
           $this->createOrUpdateSubscribers(
-            'update',
             $existingSubscribers,
             $this->subscribersCustomFields
           );
@@ -144,7 +173,7 @@ class Import {
     $segments = $importFactory->getSegments();
     $welcomeNotificationsInSegments =
       ($createdSubscribers || $updatedSubscribers) ?
-        Newsletter::getWelcomeNotificationsForSegments($this->segmentsIds) :
+        $this->newsletterOptionsRepository->findWelcomeNotificationsForSegments($this->segmentsIds) :
         false;
 
     return [
@@ -172,12 +201,12 @@ class Import {
       }
       // if this is a custom column
       if (in_array($column, $this->subscribersCustomFields)) {
-        $customField = CustomField::findOne($column);
+        $customField = $this->customFieldsRepository->findOneById($column);
-        if (!$customField instanceof CustomField) {
+        if (!$customField instanceof CustomFieldEntity) {
           continue;
         }
         // validate date type
-        if ($customField->type === 'date') {
+        if ($customField->getType() === CustomFieldEntity::TYPE_DATE) {
           $validationRule = 'datetime';
           $data = array_map(
             function($index, $date) use($validationRule, &$invalidRecords) {
@@ -219,11 +248,7 @@ class Import {
       // with just wp_user_id and email fields: [[wp_user_id, email], [wp_user_id, email], ...]
       $tempExistingSubscribers = array_merge(
         $tempExistingSubscribers,
-        Subscriber::select('wp_user_id')
+        $this->subscriberRepository->findWpUserIdAndEmailByEmails($subscribersEmails)
-          ->selectExpr('LOWER(email)', 'email')
-          ->whereIn('email', $subscribersEmails)
-          ->whereNull('deleted_at')
-          ->findArray()
       );
     }
     if (!$tempExistingSubscribers) {
@@ -261,19 +286,15 @@ class Import {
   public function deleteExistingTrashedSubscribers($subscribersData) {
     $existingTrashedRecords = array_filter(
       array_map(function($subscriberEmails) {
-        return Subscriber::selectMany(['id'])
+        return $this->subscriberRepository->findIdsOfDeletedByEmails($subscriberEmails);
-          ->whereIn('email', $subscriberEmails)
-          ->whereNotNull('deleted_at')
-          ->findArray();
       }, array_chunk($subscribersData['email'], self::DB_QUERY_CHUNK_SIZE))
     );
-    if (!$existingTrashedRecords) return;
     $existingTrashedRecords = Helpers::flattenArray($existingTrashedRecords);
+    if (!$existingTrashedRecords) {
+      return;
+    }
     foreach (array_chunk($existingTrashedRecords, self::DB_QUERY_CHUNK_SIZE) as $subscriberIds) {
-      Subscriber::whereIn('id', $subscriberIds)
+      $this->subscriberRepository->bulkDelete($subscriberIds);
-        ->deleteMany();
-      SubscriberSegment::whereIn('subscriber_id', $subscriberIds)
-        ->deleteMany();
     }
   }
 
@@ -305,7 +326,7 @@ class Import {
       return $defaultStatus;
     }, $subscribersData['data']['status']);
 
-    if ($defaultStatus === Subscriber::STATUS_SUBSCRIBED) {
+    if ($defaultStatus === SubscriberEntity::STATUS_SUBSCRIBED) {
       if (!in_array('last_subscribed_at', $subscribersData['fields'])) {
         $subscribersData['fields'][] = 'last_subscribed_at';
       }
@@ -332,7 +353,7 @@ class Import {
     $subscribersData['fields'][] = 'link_token';
     $subscribersData['data']['link_token'] = array_map(
       function () {
-        return Security::generateRandomString(Subscriber::LINK_TOKEN_LENGTH);
+        return Security::generateRandomString(SubscriberEntity::LINK_TOKEN_LENGTH);
       }, array_fill(0, $subscribersCount, null)
     );
     return $subscribersData;
@@ -359,9 +380,8 @@ class Import {
   }
 
   public function createOrUpdateSubscribers(
-    $action,
+    array $subscribersData,
-    $subscribersData,
+    array $subscribersCustomFields = []
-    $subscribersCustomFields = false
   ) {
     $subscribersCount = count($subscribersData['data'][key($subscribersData['data'])]);
     $subscribers = array_map(function($index) use ($subscribersData) {
@@ -370,23 +390,15 @@ class Import {
       }, $subscribersData['fields']);
     }, range(0, $subscribersCount - 1));
     foreach (array_chunk($subscribers, self::DB_QUERY_CHUNK_SIZE) as $data) {
-      if ($action == 'create') {
+      $this->subscriberRepository->insertOrUpdateMultiple(
-        Subscriber::createMultiple(
-          $subscribersData['fields'],
-          $data
-        );
-      }
-      if ($action == 'update') {
-        Subscriber::updateMultiple(
         $subscribersData['fields'],
         $data,
         $this->updatedAt
       );
     }
-    }
     $createdOrUpdatedSubscribers = [];
-    foreach (array_chunk($subscribersData['data']['email'], self::DB_QUERY_CHUNK_SIZE) as $data) {
+    foreach (array_chunk($subscribersData['data']['email'], self::DB_QUERY_CHUNK_SIZE) as $emails) {
-      foreach (Subscriber::selectMany(['id', 'email'])->whereIn('email', $data)->findArray() as $createdOrUpdatedSubscriber) {
+      foreach ($this->subscriberRepository->findIdAndEmailByEmails($emails) as $createdOrUpdatedSubscriber) {
         // ensure emails loaded from the DB are lowercased (imported emails are lowercased as well)
         $createdOrUpdatedSubscriber['email'] = mb_strtolower($createdOrUpdatedSubscriber['email']);
         $createdOrUpdatedSubscribers[] = $createdOrUpdatedSubscriber;
@@ -396,7 +408,6 @@ class Import {
     $createdOrUpdatedSubscribersIds = array_column($createdOrUpdatedSubscribers, 'id');
     if ($subscribersCustomFields) {
       $this->createOrUpdateCustomFields(
-        $action,
         $createdOrUpdatedSubscribers,
         $subscribersData,
         $subscribersCustomFields
@@ -410,18 +421,17 @@ class Import {
   }
 
   public function createOrUpdateCustomFields(
-    $action,
+    array $createdOrUpdatedSubscribers,
-    $createdOrUpdatedSubscribers,
+    array $subscribersData,
-    $subscribersData,
+    array $subscribersCustomFieldsIds
-    $subscribersCustomFieldsIds
   ) {
     // check if custom fields exist in the database
-    $subscribersCustomFieldsIds = Helpers::flattenArray(
+    $subscribersCustomFieldsIds = array_map(function(CustomFieldEntity $customField): int {
-      CustomField::whereIn('id', $subscribersCustomFieldsIds)
+      return (int)$customField->getId();
-        ->select('id')
+    }, $this->customFieldsRepository->findBy(['id' => $subscribersCustomFieldsIds]));
-        ->findArray()
+    if (!$subscribersCustomFieldsIds) {
-    );
+      return;
-    if (!$subscribersCustomFieldsIds) return;
+    }
     // assemble a two-dimensional array: [[custom_field_id, subscriber_id, value], [custom_field_id, subscriber_id, value], ...]
     $subscribersCustomFieldsData = [];
     $subscribersEmails = array_flip($subscribersData['data']['email']);
@@ -434,18 +444,22 @@ class Import {
           (int)$field,
           $createdOrUpdatedSubscriber['id'],
           $values[$subscriberIndex],
+          $this->createdAt,
         ];
       }
     }
+    $columns = [
+      'custom_field_id',
+      'subscriber_id',
+      'value',
+      'created_at',
+    ];
     foreach (array_chunk($subscribersCustomFieldsData, self::DB_QUERY_CHUNK_SIZE) as $subscribersCustomFieldsDataChunk) {
-      SubscriberCustomField::createMultiple(
+      $this->subscriberCustomFieldRepository->insertOrUpdateMultiple(
-        $subscribersCustomFieldsDataChunk
+        $columns,
+        $subscribersCustomFieldsDataChunk,
+        $this->updatedAt
       );
-      if ($action === 'update') {
-        SubscriberCustomField::updateMultiple(
-          $subscribersCustomFieldsDataChunk
-        );
-      }
     }
   }
 
@@ -454,10 +468,23 @@ class Import {
   }
 
   public function addSubscribersToSegments($subscribersIds, $segmentsIds) {
+    $columns = [
+      'subscriber_id',
+      'segment_id',
+      'created_at',
+    ];
     foreach (array_chunk($subscribersIds, self::DB_QUERY_CHUNK_SIZE) as $subscriberIdsChunk) {
-      SubscriberSegment::subscribeManyToSegments(
+      $data = [];
-        $subscriberIdsChunk, $segmentsIds
+      foreach ($segmentsIds as $segmentId) {
-      );
+        $data = array_merge($data, array_map(function ($subscriberId) use ($segmentId): array {
+          return [
+            $subscriberId,
+            $segmentId,
+            $this->createdAt,
+          ];
+        }, $subscriberIdsChunk));
+      }
+      $this->subscriberSegmentRepository->insertOrUpdateMultiple($columns, $data, $this->updatedAt);
     }
   }
 }

lib/Subscribers/SubscribersRepository.php

@@ -15,6 +15,14 @@ use MailPoetVendor\Doctrine\ORM\Query\Expr\Join;
  * @extends Repository<SubscriberEntity>
  */
 class SubscribersRepository extends Repository {
+  protected $ignoreColumnsForUpdate = [
+    'wp_user_id',
+    'is_woocommerce_user',
+    'email',
+    'created_at',
+    'last_subscribed_at',
+  ];
+
   protected function getEntityClassName() {
     return SubscriberEntity::class;
   }
@@ -244,4 +252,35 @@ class SubscribersRepository extends Repository {
 
     return count($ids);
   }
+  
+  public function findWpUserIdAndEmailByEmails(array $emails): array {
+    return $this->entityManager->createQueryBuilder()
+      ->select('s.wpUserId AS wp_user_id, LOWER(s.email) AS email')
+      ->from(SubscriberEntity::class, 's')
+      ->where('s.email IN (:emails)')
+      ->setParameter('emails', $emails)
+      ->getQuery()->getResult();
+  }
+
+  public function findIdAndEmailByEmails(array $emails): array {
+    return $this->entityManager->createQueryBuilder()
+      ->select('s.id, s.email')
+      ->from(SubscriberEntity::class, 's')
+      ->where('s.email IN (:emails)')
+      ->setParameter('emails', $emails)
+      ->getQuery()->getResult();
+  }
+
+  /**
+   * @return int[]
+   */
+  public function findIdsOfDeletedByEmails(array $emails): array {
+    return $this->entityManager->createQueryBuilder()
+    ->select('s.id')
+    ->from(SubscriberEntity::class, 's')
+    ->where('s.email IN (:emails)')
+    ->andWhere('s.deletedAt IS NOT NULL')
+    ->setParameter('emails', $emails)
+    ->getQuery()->getResult();
+  }
 }