Cavemanon/piratepoet
7
Fork 0
Code Pull Requests Actions Releases 16 Activity

Create controller to count emails per subscriber

Browse Source 
This controller is built to be called by a cron SimpleWorker.

[MAILPOET-4177]
This commit is contained in:
Brezo Cordero
2022-03-15 17:28:18 -05:00
committed by Veljko V
parent 8fe323c9c7
commit 8b7b234446
4 changed files with 395 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
mailpoet/lib/Subscribers/SubscribersEmailCountsController.php Normal file
View File

@ -0,0 +1,142 @@
<?php declare(strict_types = 1);
namespace MailPoet\Subscribers;
use MailPoet\Entities\ScheduledTaskEntity;
use MailPoet\Entities\ScheduledTaskSubscriberEntity;
use MailPoet\Entities\SendingQueueEntity;
use MailPoet\Entities\SubscriberEntity;
use MailPoetVendor\Carbon\Carbon;
use MailPoetVendor\Doctrine\ORM\EntityManager;
class SubscribersEmailCountsController {
private $processedTaskIdsTableCreated = false;
/** @var EntityManager */
private $entityManager;
/** @var string */
private $subscribersTable;
public function __construct(
EntityManager $entityManager
) {
$this->entityManager = $entityManager;
$this->subscribersTable = $this->entityManager->getClassMetadata(SubscriberEntity::class)->getTableName();
}
public function updateSubscribersEmailCounts(?\DateTimeInterface $dateLastProcessed, int $batchSize, ?int $startId = null): array {
$sendingQueuesTable = $this->entityManager->getClassMetadata(SendingQueueEntity::class)->getTableName();
$scheduledTasksTable = $this->entityManager->getClassMetadata(ScheduledTaskEntity::class)->getTableName();
$scheduledTaskSubscribersTable = $this->entityManager->getClassMetadata(ScheduledTaskSubscriberEntity::class)->getTableName();
$connection = $this->entityManager->getConnection();
$dayAgo = new Carbon();
$dayAgoIso = $dayAgo->subDay()->toDateTimeString();
$startId = (int)$startId;
[$countSubscribersToUpdate, $endId] = $this->countAndMaxOfSubscribersInRange($startId, $batchSize);
if (!$countSubscribersToUpdate) {
return [0, 0];
}
// Temporary table with processed tasks from threshold date up to yesterday
$processedTaskIdsTable = 'processed_task_ids';
if (!$this->processedTaskIdsTableCreated) {
$queryParams = [];
$processedTaskIdsTableSql = "
CREATE TEMPORARY TABLE IF NOT EXISTS {$processedTaskIdsTable}
(INDEX task_id_ids (id))
SELECT DISTINCT task_id as id FROM {$sendingQueuesTable} as sq
JOIN {$scheduledTasksTable} as st ON sq.task_id = st.id
WHERE st.processed_at IS NOT NULL
AND st.processed_at < :dayAgo";
$queryParams['dayAgo'] = $dayAgoIso;
if ($dateLastProcessed) {
$processedTaskIdsTableSql .= " AND st.processed_at >= :dateFrom";
$carbonDateLastProcessed = Carbon::createFromTimestamp($dateLastProcessed->getTimestamp());
$dateFromIso = ($carbonDateLastProcessed->subDay())->toDateTimeString();
$queryParams['dateFrom'] = $dateFromIso;
}
$resultQuery = $connection->executeQuery($processedTaskIdsTableSql, $queryParams);
$this->processedTaskIdsTableCreated = true;
if ($resultQuery->rowCount() === 0) return [0,0];
}
// Temporary table needed for UPDATE query
// mySQL does not allow to modify the same table used in the select
$subscriberIdsEmailsCountTmpTable = 'subscribers_ids_email_counts';
$connection->executeQuery("
CREATE TEMPORARY TABLE IF NOT EXISTS {$subscriberIdsEmailsCountTmpTable}
(UNIQUE subscriber_id (id))
SELECT s.id, count(task_ids.id) as emails_count from {$this->subscribersTable} s
JOIN {$scheduledTaskSubscribersTable} sts ON s.id = sts.subscriber_id
JOIN {$processedTaskIdsTable} task_ids ON task_ids.id = sts.task_id
WHERE s.id >= :startId
AND s.id <= :endId
GROUP BY s.id
",
[
'startId' => $startId,
'endId' => $endId,
]
);
// If $dateLastProcessed provided, increment value, otherwise count all and reset value
$initUpdateValue = $dateLastProcessed ? 's.emails_count' : '';
$updateQuery = $connection->executeQuery("
UPDATE {$this->subscribersTable} as s
JOIN {$subscriberIdsEmailsCountTmpTable} as sc ON s.id = sc.id
SET s.emails_count = {$initUpdateValue} + IFNULL(sc.emails_count, 0)
WHERE s.id >= :startId
AND s.id <= :endId
",
[
'startId' => $startId,
'endId' => $endId,
]
);
$connection->executeQuery("DROP TABLE {$subscriberIdsEmailsCountTmpTable}");
return [$countSubscribersToUpdate, $endId];
}
private function countAndMaxOfSubscribersInRange(int $startId, int $batchSize): array {
$connection = $this->entityManager->getConnection();
$result = $connection->executeQuery("
SELECT s.id FROM {$this->subscribersTable} as s
WHERE s.id >= :startId
ORDER BY s.id
LIMIT :batchSize
",
[
'startId' => $startId,
'batchSize' => $batchSize,
],
[
'startId' => \PDO::PARAM_INT,
'batchSize' => \PDO::PARAM_INT,
]
);
$subscribersInRange = $result->fetchAllAssociative();
$countSubscribersInRange = count(array_map(
function ($id) {
return (int)$id['id'];
},
$subscribersInRange
));
if (!$countSubscribersInRange) {
return [0,0];
}
return [$countSubscribersInRange,$subscribersInRange[$countSubscribersInRange - 1]['id']];
}
}

5
mailpoet/tasks/phpstan/phpstan-7-baseline.neon
View File

@ -1106,6 +1106,11 @@ parameters:
count: 2
path: ../../lib/Subscribers/InactiveSubscribersController.php
-
message: "#^Cannot cast mixed to int\\.$#"
count: 1
path: ../../lib/Subscribers/SubscribersEmailCountsController.php
-
message: "#^Cannot cast mixed to int\\.$#"
count: 3

5
mailpoet/tasks/phpstan/phpstan-8-baseline.neon
View File

@ -1106,6 +1106,11 @@ parameters:
count: 2
path: ../../lib/Subscribers/InactiveSubscribersController.php
-
message: "#^Cannot cast mixed to int\\.$#"
count: 1
path: ../../lib/Subscribers/SubscribersEmailCountsController.php
-
message: "#^Cannot cast mixed to int\\.$#"
count: 3

243
mailpoet/tests/integration/Subscribers/SubscribersEmailCountsControllerTest.php Normal file
View File

@ -0,0 +1,243 @@
<?php declare(strict_types = 1);
namespace MailPoet\Subscribers;
use MailPoet\Entities\NewsletterEntity;
use MailPoet\Entities\ScheduledTaskEntity;
use MailPoet\Entities\ScheduledTaskSubscriberEntity;
use MailPoet\Entities\SendingQueueEntity;
use MailPoet\Entities\SubscriberEntity;
use MailPoet\Tasks\Sending;
use MailPoetVendor\Carbon\Carbon;
use MailPoetVendor\Doctrine\ORM\EntityManager;
class SubscribersEmailCountsControllerTest extends \MailPoetTest {
/** @var SubscribersEmailCountsController */
private $controller;
/** @var SubscribersRepository */
private $subscribersRepository;
/** @var NewsletterEntity */
private $newsletter;
public function _before() {
$this->controller = new SubscribersEmailCountsController(
$this->diContainer->get(EntityManager::class)
);
$this->subscribersRepository = $this->diContainer->get(SubscribersRepository::class);
$this->truncateEntity(SubscriberEntity::class);
$this->truncateEntity(ScheduledTaskEntity::class);
$this->truncateEntity(ScheduledTaskSubscriberEntity::class);
$this->truncateEntity(SendingQueueEntity::class);
$this->truncateEntity(NewsletterEntity::class);
$this->entityManager->getConnection()->executeQuery('DROP TABLE IF EXISTS processed_task_ids');
$this->newsletter = new NewsletterEntity();
$this->newsletter->setSubject('Subject');
$this->newsletter->setType(NewsletterEntity::TYPE_STANDARD);
$this->newsletter->setStatus(NewsletterEntity::STATUS_SENT);
$this->entityManager->persist($this->newsletter);
$this->entityManager->flush();
parent::_before();
}
public function testItCalculatesTotalSubscribersEmailCounts(): void {
$subscriber1 = $this->createSubscriber('s1@email.com', 100);
$subscriber2 = $this->createSubscriber('s2@email.com', 10);
$subscriber3 = $this->createSubscriber('s3@email.com', 10);
$this->createCompletedSendingTasksForSubscriber($subscriber1, 80, 90);
$this->createCompletedSendingTasksForSubscriber($subscriber2, 8, 3);
[$count, $maxSubscriberId] = $this->controller->updateSubscribersEmailCounts(null, 3);
expect($count)->equals(3);
$this->entityManager->clear();
$subscriber1 = $this->subscribersRepository->findOneById($subscriber1->getId());
$subscriber2 = $this->subscribersRepository->findOneById($subscriber2->getId());
$subscriber3 = $this->subscribersRepository->findOneById($subscriber3->getId());
assert($subscriber1 instanceof SubscriberEntity);
assert($subscriber2 instanceof SubscriberEntity);
assert($subscriber3 instanceof SubscriberEntity);
expect($subscriber1->getEmailsCount())->equals(80);
expect($subscriber2->getEmailsCount())->equals(8);
expect($subscriber3->getEmailsCount())->equals(0);
}
public function testItIncrementsSubscribersEmailCountsWhenDateProvided(): void {
$subscriber1 = $this->createSubscriber('s1@email.com', 100, SubscriberEntity::STATUS_SUBSCRIBED, 80);
$subscriber2 = $this->createSubscriber('s2@email.com', 20, SubscriberEntity::STATUS_SUBSCRIBED, 8);
$subscriber3 = $this->createSubscriber('s3@email.com', 10);
$this->createCompletedSendingTasksForSubscriber($subscriber1, 1, 5);
$this->createCompletedSendingTasksForSubscriber($subscriber2, 1, 5);
$this->createCompletedSendingTasksForSubscriber($subscriber3, 1, 5);
$dateFromCarbon = new Carbon();
$dateFrom = $dateFromCarbon->subDays(7)->toDateTime();
[$count, $maxSubscriberId] = $this->controller->updateSubscribersEmailCounts($dateFrom, 3);
expect($count)->equals(3);
$this->entityManager->clear();
$subscriber1 = $this->subscribersRepository->findOneById($subscriber1->getId());
$subscriber2 = $this->subscribersRepository->findOneById($subscriber2->getId());
$subscriber3 = $this->subscribersRepository->findOneById($subscriber3->getId());
assert($subscriber1 instanceof SubscriberEntity);
assert($subscriber2 instanceof SubscriberEntity);
assert($subscriber3 instanceof SubscriberEntity);
expect($subscriber1->getEmailsCount())->equals(81);
expect($subscriber2->getEmailsCount())->equals(9);
expect($subscriber3->getEmailsCount())->equals(1);
}
public function testItResetsValueToTotalIfNoDateProvided(): void {
$subscriber1 = $this->createSubscriber('s1@email.com', 100, SubscriberEntity::STATUS_SUBSCRIBED, 80);
$subscriber2 = $this->createSubscriber('s2@email.com', 20, SubscriberEntity::STATUS_SUBSCRIBED, 8);
$subscriber3 = $this->createSubscriber('s3@email.com', 10);
$this->createCompletedSendingTasksForSubscriber($subscriber1, 80, 90);
$this->createCompletedSendingTasksForSubscriber($subscriber2, 8, 3);
$this->createCompletedSendingTasksForSubscriber($subscriber3, 1, 4);
// Count
$this->controller->updateSubscribersEmailCounts(null, 3);
// Recount
$this->controller->updateSubscribersEmailCounts(null, 3);
$this->entityManager->clear();
$subscriber1 = $this->subscribersRepository->findOneById($subscriber1->getId());
$subscriber2 = $this->subscribersRepository->findOneById($subscriber2->getId());
$subscriber3 = $this->subscribersRepository->findOneById($subscriber3->getId());
assert($subscriber1 instanceof SubscriberEntity);
assert($subscriber2 instanceof SubscriberEntity);
assert($subscriber3 instanceof SubscriberEntity);
expect($subscriber1->getEmailsCount())->equals(80);
expect($subscriber2->getEmailsCount())->equals(8);
expect($subscriber3->getEmailsCount())->equals(1);
}
public function testItUpdatesOnlySubscribersInBatch() {
$subscriber1 = $this->createSubscriber('s1@email.com', 100, SubscriberEntity::STATUS_SUBSCRIBED, 80);
$subscriber2 = $this->createSubscriber('s2@email.com', 20, SubscriberEntity::STATUS_SUBSCRIBED, 8);
$subscriber3 = $this->createSubscriber('s3@email.com', 10);
$this->createCompletedSendingTasksForSubscriber($subscriber1, 80, 90);
$this->createCompletedSendingTasksForSubscriber($subscriber2, 8, 3);
$this->createCompletedSendingTasksForSubscriber($subscriber3, 1, 4);
// First batch of 1
[$count, $maxSubscriberId] = $this->controller->updateSubscribersEmailCounts(null, 1);
expect($count)->equals(1);
// Second batch of 1
$this->controller->updateSubscribersEmailCounts(null, 1, $maxSubscriberId + 1);
$this->entityManager->clear();
$subscriber1 = $this->subscribersRepository->findOneById($subscriber1->getId());
$subscriber2 = $this->subscribersRepository->findOneById($subscriber2->getId());
$subscriber3 = $this->subscribersRepository->findOneById($subscriber3->getId());
assert($subscriber1 instanceof SubscriberEntity);
assert($subscriber2 instanceof SubscriberEntity);
assert($subscriber3 instanceof SubscriberEntity);
expect($subscriber1->getEmailsCount())->equals(80);
expect($subscriber2->getEmailsCount())->equals(8);
// Subscriber not in batch should not be updated
expect($subscriber3->getEmailsCount())->equals(0);
}
public function testItDoesNotCountIfThereAreNoSubscribersOrTasksToUpdate() {
// Subscribers empty table
[$count, $maxSubscriberId] = $this->controller->updateSubscribersEmailCounts(null, 1);
expect($count)->equals(0);
$subscriber1 = $this->createSubscriber('s1@email.com', 100);
$subscriber2 = $this->createSubscriber('s2@email.com', 20);
$subscriber3 = $this->createSubscriber('s3@email.com', 10);
// Tasks empty table
[$count, $maxSubscriberId] = $this->controller->updateSubscribersEmailCounts(null, 1);
expect($count)->equals(0);
$this->createCompletedSendingTasksForSubscriber($subscriber1, 80, 90);
$this->createCompletedSendingTasksForSubscriber($subscriber2, 8, 3);
$this->createCompletedSendingTasksForSubscriber($subscriber3, 1, 4);
// No subscribers to update from startId
[$count, $maxSubscriberId] = $this->controller->updateSubscribersEmailCounts(null, 1, 4);
expect($count)->equals(0);
$this->entityManager->clear();
$subscriber1 = $this->subscribersRepository->findOneById($subscriber1->getId());
$subscriber2 = $this->subscribersRepository->findOneById($subscriber2->getId());
$subscriber3 = $this->subscribersRepository->findOneById($subscriber3->getId());
assert($subscriber1 instanceof SubscriberEntity);
assert($subscriber2 instanceof SubscriberEntity);
assert($subscriber3 instanceof SubscriberEntity);
expect($subscriber1->getEmailsCount())->equals(0);
expect($subscriber2->getEmailsCount())->equals(0);
expect($subscriber3->getEmailsCount())->equals(0);
}
private function createCompletedSendingTasksForSubscriber(SubscriberEntity $subscriber, int $numTasks = 1, int $processedDaysAgo = 0): void {
for ($i = 0; $i < $numTasks; $i++) {
[$task] = $this->createCompletedSendingTask($processedDaysAgo);
$this->addSubscriberToTask($subscriber, $task);
}
}
private function createCompletedSendingTask(int $processedDaysAgo = 0): array {
$processedAt = (new Carbon())->subDays($processedDaysAgo);
$task = new ScheduledTaskEntity();
$task->setType(Sending::TASK_TYPE);
$task->setStatus(ScheduledTaskEntity::STATUS_COMPLETED);
$task->setCreatedAt($processedAt);
$task->setProcessedAt($processedAt);
$this->entityManager->persist($task);
$this->entityManager->flush();
$queue = new SendingQueueEntity();
$queue->setTask($task);
$queue->setNewsletter($this->newsletter);
$this->entityManager->persist($queue);
$this->entityManager->flush();
return [$task, $queue];
}
private function addSubscriberToTask(
SubscriberEntity $subscriber,
ScheduledTaskEntity $task,
int $daysAgo = 0
): ScheduledTaskSubscriberEntity {
$createdAt = (new Carbon())->subDays($daysAgo);
$taskSubscriber = new ScheduledTaskSubscriberEntity($task, $subscriber);
$taskSubscriber->setCreatedAt($createdAt);
$this->entityManager->persist($taskSubscriber);
$this->entityManager->flush();
return $taskSubscriber;
}
private function createSubscriber(
string $email,
int $createdDaysAgo = 0,
string $status = SubscriberEntity::STATUS_SUBSCRIBED,
int $emailCounts = 0
): SubscriberEntity {
$createdAt = (new Carbon())->subDays($createdDaysAgo);
$subscriber = new SubscriberEntity();
$subscriber->setEmail($email);
$subscriber->setStatus($status);
$subscriber->setCreatedAt($createdAt);
$subscriber->setEmailsCount($emailCounts);
$this->entityManager->persist($subscriber);
// we need to set lastSubscribeAt after persist due to LastSubscribedAtListener
$subscriber->setLastSubscribedAt($createdAt);
$this->entityManager->flush();
return $subscriber;
}
}