Cavemanon/piratepoet
7
Fork 0
Code Pull Requests Actions Releases 16 Activity

Use Doctrine instead of Paris in Import

Browse Source 
[MAILPOET-3378]
This commit is contained in:
Jan Lysý
2021-02-05 16:07:03 +01:00
committed by Veljko V
parent 78d87120f3
commit 09acbb0d07
5 changed files with 198 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
lib/API/JSON/v1/ImportExport.php
View File

@ -5,10 +5,16 @@ namespace MailPoet\API\JSON\v1;
use MailPoet\API\JSON\Endpoint as APIEndpoint;
use MailPoet\Config\AccessControl;
use MailPoet\Cron\Workers\WooCommerceSync;
use MailPoet\CustomFields\CustomFieldsRepository;
use MailPoet\Models\ScheduledTask;
use MailPoet\Models\Segment;
use MailPoet\Newsletter\Options\NewsletterOptionsRepository;
use MailPoet\Segments\WP;
use MailPoet\Subscribers\ImportExport\Import\Import;
use MailPoet\Subscribers\ImportExport\Import\MailChimp;
use MailPoet\Subscribers\SubscriberCustomFieldRepository;
use MailPoet\Subscribers\SubscriberSegmentRepository;
use MailPoet\Subscribers\SubscribersRepository;
use MailPoetVendor\Carbon\Carbon;
class ImportExport extends APIEndpoint {
@ -16,12 +22,39 @@ class ImportExport extends APIEndpoint {
/** @var WP */
private $wpSegment;
/** @var CustomFieldsRepository */
private $customFieldsRepository;
/** @var NewsletterOptionsRepository */
private $newsletterOptionsRepository;
/** @var SubscriberCustomFieldRepository */
private $subscriberCustomFieldRepository;
/** @var SubscribersRepository */
private $subscriberRepository;
/** @var SubscriberSegmentRepository */
private $subscriberSegmentRepository;
public $permissions = [
'global' => AccessControl::PERMISSION_MANAGE_SUBSCRIBERS,
];
public function __construct(WP $wpSegment) {
public function __construct(
WP $wpSegment,
CustomFieldsRepository $customFieldsRepository,
NewsletterOptionsRepository $newsletterOptionsRepository,
SubscriberCustomFieldRepository $subscriberCustomFieldRepository,
SubscribersRepository $subscribersRepository,
SubscriberSegmentRepository $subscriberSegmentRepository
) {
$this->wpSegment = $wpSegment;
$this->customFieldsRepository = $customFieldsRepository;
$this->newsletterOptionsRepository = $newsletterOptionsRepository;
$this->subscriberCustomFieldRepository = $subscriberCustomFieldRepository;
$this->subscriberRepository = $subscribersRepository;
$this->subscriberSegmentRepository = $subscriberSegmentRepository;
}
public function getMailChimpLists($data) {
@ -63,8 +96,13 @@ class ImportExport extends APIEndpoint {
public function processImport($data) {
try {
$import = new \MailPoet\Subscribers\ImportExport\Import\Import(
$import = new Import(
$this->wpSegment,
$this->customFieldsRepository,
$this->newsletterOptionsRepository,
$this->subscriberCustomFieldRepository,
$this->subscriberRepository,
$this->subscriberSegmentRepository,
json_decode($data, true)
);
$process = $import->process();

3
lib/Entities/SubscriberEntity.php
View File

@ -24,6 +24,9 @@ class SubscriberEntity {
const STATUS_UNCONFIRMED = 'unconfirmed';
const STATUS_UNSUBSCRIBED = 'unsubscribed';
public const OBSOLETE_LINK_TOKEN_LENGTH = 6;
public const LINK_TOKEN_LENGTH = 32;
use AutoincrementedIdTrait;
use CreatedAtTrait;
use UpdatedAtTrait;

18
lib/Newsletter/Options/NewsletterOptionsRepository.php
View File

@ -3,7 +3,9 @@
namespace MailPoet\Newsletter\Options;
use MailPoet\Doctrine\Repository;
use MailPoet\Entities\NewsletterEntity;
use MailPoet\Entities\NewsletterOptionEntity;
use MailPoet\Entities\NewsletterOptionFieldEntity;
/**
* @extends Repository<NewsletterOptionEntity>
@ -12,4 +14,20 @@ class NewsletterOptionsRepository extends Repository {
protected function getEntityClassName() {
return NewsletterOptionEntity::class;
}
public function findWelcomeNotificationsForSegments(array $segmentIds): array {
return $this->entityManager->createQueryBuilder()
->select('no')
->from(NewsletterOptionEntity::class, 'no')
->join('no.newsletter', 'n')
->join('no.optionField', 'nof')
->where('n.deletedAt IS NULL')
->andWhere('n.type = :typeWelcome')
->andWhere('nof.name = :nameSegment')
->andWhere('no.value IN (:segmentIds)')
->setParameter('typeWelcome', NewsletterEntity::TYPE_WELCOME)
->setParameter('nameSegment', NewsletterOptionFieldEntity::NAME_SEGMENT)
->setParameter('segmentIds', $segmentIds)
->getQuery()->getResult();
}
}

169
lib/Subscribers/ImportExport/Import/Import.php
View File

@ -2,15 +2,17 @@
namespace MailPoet\Subscribers\ImportExport\Import;
use MailPoet\Models\CustomField;
use MailPoet\CustomFields\CustomFieldsRepository;
use MailPoet\Entities\CustomFieldEntity;
use MailPoet\Entities\SubscriberEntity;
use MailPoet\Models\ModelValidator;
use MailPoet\Models\Newsletter;
use MailPoet\Models\Subscriber;
use MailPoet\Models\SubscriberCustomField;
use MailPoet\Models\SubscriberSegment;
use MailPoet\Newsletter\Options\NewsletterOptionsRepository;
use MailPoet\Segments\WP;
use MailPoet\Subscribers\ImportExport\ImportExportFactory;
use MailPoet\Subscribers\Source;
use MailPoet\Subscribers\SubscriberCustomFieldRepository;
use MailPoet\Subscribers\SubscriberSegmentRepository;
use MailPoet\Subscribers\SubscribersRepository;
use MailPoet\Util\DateConverter;
use MailPoet\Util\Helpers;
use MailPoet\Util\Security;
@ -35,8 +37,36 @@ class Import {
/** @var WP */
private $wpSegment;
public function __construct(WP $wpSegment, $data) {
/** @var CustomFieldsRepository */
private $customFieldsRepository;
/** @var NewsletterOptionsRepository */
private $newsletterOptionsRepository;
/** @var SubscriberCustomFieldRepository */
private $subscriberCustomFieldRepository;
/** @var SubscribersRepository */
private $subscriberRepository;
/** @var SubscriberSegmentRepository */
private $subscriberSegmentRepository;
public function __construct(
WP $wpSegment,
CustomFieldsRepository $customFieldsRepository,
NewsletterOptionsRepository $newsletterOptionsRepository,
SubscriberCustomFieldRepository $subscriberCustomFieldRepository,
SubscribersRepository $subscriberRepository,
SubscriberSegmentRepository $subscriberSegmentRepository,
array $data
) {
$this->wpSegment = $wpSegment;
$this->customFieldsRepository = $customFieldsRepository;
$this->newsletterOptionsRepository = $newsletterOptionsRepository;
$this->subscriberCustomFieldRepository = $subscriberCustomFieldRepository;
$this->subscriberRepository = $subscriberRepository;
$this->subscriberSegmentRepository = $subscriberSegmentRepository;
$this->validateImportData($data);
$this->subscribersData = $this->transformSubscribersData(
$data['subscribers'],
@ -56,7 +86,7 @@ class Import {
$this->createdAt = Carbon::createFromTimestamp(WPFunctions::get()->currentTime('timestamp'));
$this->updatedAt = Carbon::createFromTimestamp(WPFunctions::get()->currentTime('timestamp') + 1);
$this->requiredSubscribersFields = [
'status' => Subscriber::STATUS_SUBSCRIBED,
'status' => SubscriberEntity::STATUS_SUBSCRIBED,
'first_name' => '',
'last_name' => '',
'created_at' => $this->createdAt,
@ -111,23 +141,22 @@ class Import {
$newSubscribers = $this->setLinkToken($newSubscribers);
$createdSubscribers =
$this->createOrUpdateSubscribers(
'create',
$newSubscribers,
$this->subscribersCustomFields
);
}
if ($existingSubscribers['data'] && $this->updateSubscribers) {
$allowedStatuses = [
Subscriber::STATUS_SUBSCRIBED,
Subscriber::STATUS_UNSUBSCRIBED,
Subscriber::STATUS_INACTIVE,
SubscriberEntity::STATUS_SUBSCRIBED,
SubscriberEntity::STATUS_UNSUBSCRIBED,
SubscriberEntity::STATUS_INACTIVE,
];
if (in_array($this->existingSubscribersStatus, $allowedStatuses, true)) {
$existingSubscribers = $this->addField($existingSubscribers, 'status', $this->existingSubscribersStatus);
}
$updatedSubscribers =
$this->createOrUpdateSubscribers(
'update',
$existingSubscribers,
$this->subscribersCustomFields
);
@ -144,7 +173,7 @@ class Import {
$segments = $importFactory->getSegments();
$welcomeNotificationsInSegments =
($createdSubscribers || $updatedSubscribers) ?
Newsletter::getWelcomeNotificationsForSegments($this->segmentsIds) :
$this->newsletterOptionsRepository->findWelcomeNotificationsForSegments($this->segmentsIds) :
false;
return [
@ -172,12 +201,12 @@ class Import {
}
// if this is a custom column
if (in_array($column, $this->subscribersCustomFields)) {
$customField = CustomField::findOne($column);
if (!$customField instanceof CustomField) {
$customField = $this->customFieldsRepository->findOneById($column);
if (!$customField instanceof CustomFieldEntity) {
continue;
}
// validate date type
if ($customField->type === 'date') {
if ($customField->getType() === CustomFieldEntity::TYPE_DATE) {
$validationRule = 'datetime';
$data = array_map(
function($index, $date) use($validationRule, &$invalidRecords) {
@ -219,11 +248,7 @@ class Import {
// with just wp_user_id and email fields: [[wp_user_id, email], [wp_user_id, email], ...]
$tempExistingSubscribers = array_merge(
$tempExistingSubscribers,
Subscriber::select('wp_user_id')
->selectExpr('LOWER(email)', 'email')
->whereIn('email', $subscribersEmails)
->whereNull('deleted_at')
->findArray()
$this->subscriberRepository->findWpUserIdAndEmailByEmails($subscribersEmails)
);
}
if (!$tempExistingSubscribers) {
@ -261,19 +286,15 @@ class Import {
public function deleteExistingTrashedSubscribers($subscribersData) {
$existingTrashedRecords = array_filter(
array_map(function($subscriberEmails) {
return Subscriber::selectMany(['id'])
->whereIn('email', $subscriberEmails)
->whereNotNull('deleted_at')
->findArray();
return $this->subscriberRepository->findIdsOfDeletedByEmails($subscriberEmails);
}, array_chunk($subscribersData['email'], self::DB_QUERY_CHUNK_SIZE))
);
if (!$existingTrashedRecords) return;
$existingTrashedRecords = Helpers::flattenArray($existingTrashedRecords);
if (!$existingTrashedRecords) {
return;
}
foreach (array_chunk($existingTrashedRecords, self::DB_QUERY_CHUNK_SIZE) as $subscriberIds) {
Subscriber::whereIn('id', $subscriberIds)
->deleteMany();
SubscriberSegment::whereIn('subscriber_id', $subscriberIds)
->deleteMany();
$this->subscriberRepository->bulkDelete($subscriberIds);
}
}
@ -305,7 +326,7 @@ class Import {
return $defaultStatus;
}, $subscribersData['data']['status']);
if ($defaultStatus === Subscriber::STATUS_SUBSCRIBED) {
if ($defaultStatus === SubscriberEntity::STATUS_SUBSCRIBED) {
if (!in_array('last_subscribed_at', $subscribersData['fields'])) {
$subscribersData['fields'][] = 'last_subscribed_at';
}
@ -332,7 +353,7 @@ class Import {
$subscribersData['fields'][] = 'link_token';
$subscribersData['data']['link_token'] = array_map(
function () {
return Security::generateRandomString(Subscriber::LINK_TOKEN_LENGTH);
return Security::generateRandomString(SubscriberEntity::LINK_TOKEN_LENGTH);
}, array_fill(0, $subscribersCount, null)
);
return $subscribersData;
@ -359,9 +380,8 @@ class Import {
}
public function createOrUpdateSubscribers(
$action,
$subscribersData,
$subscribersCustomFields = false
array $subscribersData,
array $subscribersCustomFields = []
) {
$subscribersCount = count($subscribersData['data'][key($subscribersData['data'])]);
$subscribers = array_map(function($index) use ($subscribersData) {
@ -370,23 +390,15 @@ class Import {
}, $subscribersData['fields']);
}, range(0, $subscribersCount - 1));
foreach (array_chunk($subscribers, self::DB_QUERY_CHUNK_SIZE) as $data) {
if ($action == 'create') {
Subscriber::createMultiple(
$subscribersData['fields'],
$data
);
}
if ($action == 'update') {
Subscriber::updateMultiple(
$subscribersData['fields'],
$data,
$this->updatedAt
);
}
$this->subscriberRepository->insertOrUpdateMultiple(
$subscribersData['fields'],
$data,
$this->updatedAt
);
}
$createdOrUpdatedSubscribers = [];
foreach (array_chunk($subscribersData['data']['email'], self::DB_QUERY_CHUNK_SIZE) as $data) {
foreach (Subscriber::selectMany(['id', 'email'])->whereIn('email', $data)->findArray() as $createdOrUpdatedSubscriber) {
foreach (array_chunk($subscribersData['data']['email'], self::DB_QUERY_CHUNK_SIZE) as $emails) {
foreach ($this->subscriberRepository->findIdAndEmailByEmails($emails) as $createdOrUpdatedSubscriber) {
// ensure emails loaded from the DB are lowercased (imported emails are lowercased as well)
$createdOrUpdatedSubscriber['email'] = mb_strtolower($createdOrUpdatedSubscriber['email']);
$createdOrUpdatedSubscribers[] = $createdOrUpdatedSubscriber;
@ -396,7 +408,6 @@ class Import {
$createdOrUpdatedSubscribersIds = array_column($createdOrUpdatedSubscribers, 'id');
if ($subscribersCustomFields) {
$this->createOrUpdateCustomFields(
$action,
$createdOrUpdatedSubscribers,
$subscribersData,
$subscribersCustomFields
@ -410,18 +421,17 @@ class Import {
}
public function createOrUpdateCustomFields(
$action,
$createdOrUpdatedSubscribers,
$subscribersData,
$subscribersCustomFieldsIds
array $createdOrUpdatedSubscribers,
array $subscribersData,
array $subscribersCustomFieldsIds
) {
// check if custom fields exist in the database
$subscribersCustomFieldsIds = Helpers::flattenArray(
CustomField::whereIn('id', $subscribersCustomFieldsIds)
->select('id')
->findArray()
);
if (!$subscribersCustomFieldsIds) return;
$subscribersCustomFieldsIds = array_map(function(CustomFieldEntity $customField): int {
return (int)$customField->getId();
}, $this->customFieldsRepository->findBy(['id' => $subscribersCustomFieldsIds]));
if (!$subscribersCustomFieldsIds) {
return;
}
// assemble a two-dimensional array: [[custom_field_id, subscriber_id, value], [custom_field_id, subscriber_id, value], ...]
$subscribersCustomFieldsData = [];
$subscribersEmails = array_flip($subscribersData['data']['email']);
@ -434,18 +444,22 @@ class Import {
(int)$field,
$createdOrUpdatedSubscriber['id'],
$values[$subscriberIndex],
$this->createdAt,
];
}
}
$columns = [
'custom_field_id',
'subscriber_id',
'value',
'created_at',
];
foreach (array_chunk($subscribersCustomFieldsData, self::DB_QUERY_CHUNK_SIZE) as $subscribersCustomFieldsDataChunk) {
SubscriberCustomField::createMultiple(
$subscribersCustomFieldsDataChunk
$this->subscriberCustomFieldRepository->insertOrUpdateMultiple(
$columns,
$subscribersCustomFieldsDataChunk,
$this->updatedAt
);
if ($action === 'update') {
SubscriberCustomField::updateMultiple(
$subscribersCustomFieldsDataChunk
);
}
}
}
@ -454,10 +468,23 @@ class Import {
}
public function addSubscribersToSegments($subscribersIds, $segmentsIds) {
$columns = [
'subscriber_id',
'segment_id',
'created_at',
];
foreach (array_chunk($subscribersIds, self::DB_QUERY_CHUNK_SIZE) as $subscriberIdsChunk) {
SubscriberSegment::subscribeManyToSegments(
$subscriberIdsChunk, $segmentsIds
);
$data = [];
foreach ($segmentsIds as $segmentId) {
$data = array_merge($data, array_map(function ($subscriberId) use ($segmentId): array {
return [
$subscriberId,
$segmentId,
$this->createdAt,
];
}, $subscriberIdsChunk));
}
$this->subscriberSegmentRepository->insertOrUpdateMultiple($columns, $data, $this->updatedAt);
}
}
}

39
lib/Subscribers/SubscribersRepository.php
View File

@ -15,6 +15,14 @@ use MailPoetVendor\Doctrine\ORM\Query\Expr\Join;
* @extends Repository<SubscriberEntity>
*/
class SubscribersRepository extends Repository {
protected $ignoreColumnsForUpdate = [
'wp_user_id',
'is_woocommerce_user',
'email',
'created_at',
'last_subscribed_at',
];
protected function getEntityClassName() {
return SubscriberEntity::class;
}
@ -244,4 +252,35 @@ class SubscribersRepository extends Repository {
return count($ids);
}
public function findWpUserIdAndEmailByEmails(array $emails): array {
return $this->entityManager->createQueryBuilder()
->select('s.wpUserId AS wp_user_id, LOWER(s.email) AS email')
->from(SubscriberEntity::class, 's')
->where('s.email IN (:emails)')
->setParameter('emails', $emails)
->getQuery()->getResult();
}
public function findIdAndEmailByEmails(array $emails): array {
return $this->entityManager->createQueryBuilder()
->select('s.id, s.email')
->from(SubscriberEntity::class, 's')
->where('s.email IN (:emails)')
->setParameter('emails', $emails)
->getQuery()->getResult();
}
/**
* @return int[]
*/
public function findIdsOfDeletedByEmails(array $emails): array {
return $this->entityManager->createQueryBuilder()
->select('s.id')
->from(SubscriberEntity::class, 's')
->where('s.email IN (:emails)')
->andWhere('s.deletedAt IS NOT NULL')
->setParameter('emails', $emails)
->getQuery()->getResult();
}
}