Cavemanon/piratepoet
8
Fork 0
Code Pull Requests Actions Releases 16 Activity

Memory usage optimization

Browse Source 
[MAILPOET-2140]
This commit is contained in:
Rostislav Wolny
2019-07-08 07:35:02 +02:00
committed by M. Shull
parent 5e85e6d2c4
commit 2f09aad2bd
2 changed files with 71 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
tests/DataGenerator/Generators/GeneratorHelper.php
View File

@@ -58,10 +58,20 @@ class GeneratorHelper {
$task->save(); $task->save();
// Add subscribers to task // Add subscribers to task
$batch_data = [];
foreach($subscribers_ids as $subscriber_id) { foreach($subscribers_ids as $subscriber_id) {
$task_subscriber = ScheduledTaskSubscriber::createOrUpdate(['task_id' => $task->id, 'subscriber_id' => $subscriber_id, 'processed' => true]); $batch_data[] = "({$task->id}, $subscriber_id, 1, '$sent_at')";
$task_subscriber->created_at = $sent_at; if (count($batch_data) % 1000 === 0) {
$task_subscriber->save(); \ORM::rawExecute(
"INSERT INTO " . ScheduledTaskSubscriber::$_table . " (`task_id`, `subscriber_id`, `processed`, `created_at`) VALUES " . implode(', ', $batch_data)
);
$batch_data = [];
}
}
if ($batch_data) {
\ORM::rawExecute(
"INSERT INTO " . ScheduledTaskSubscriber::$_table . " (`task_id`, `subscriber_id`, `processed`, `created_at`) VALUES " . implode(', ', $batch_data)
);
} }
// Sending queue // Sending queue
@@ -89,45 +99,39 @@ class GeneratorHelper {
} }
return [ return [
'newsletter' => $newsletter, 'newsletter_id' => $newsletter->id,
'task' => $task, 'task_id' => $task->id,
'queue' => $queue, 'queue_id' => $queue->id,
'sent_at' => $sent_at, 'sent_at' => strtotime($sent_at),
'link' => $link 'link_id' => $link->id,
]; ];
} }
/** /**
* @return StatisticsOpens * @return StatisticsOpens
*/ */
public function openSentNewsletter(array $sent_newsletter_data, $subscriber_id, $hours_after_send = 1) { public function openSentNewsletter(array $sent_newsletter_data, $subscriber_id, $created_at) {
$created_at = (new Carbon($sent_newsletter_data['sent_at']))->addHours($hours_after_send)->toDateTimeString(); StatisticsOpens::createOrUpdate([
$opened = StatisticsOpens::createOrUpdate([
'subscriber_id' => $subscriber_id, 'subscriber_id' => $subscriber_id,
'newsletter_id' => $sent_newsletter_data['newsletter']->id, 'newsletter_id' => $sent_newsletter_data['newsletter_id'],
'queue_id' => $sent_newsletter_data['queue']->id, 'queue_id' => $sent_newsletter_data['queue_id'],
'created_at' => $created_at, 'created_at' => $created_at,
]); ])->save();
$opened->save();
return $opened;
} }
/** /**
* @return StatisticsClicks * @return array
*/ */
public function clickSentNewsletter(array $sent_newsletter_data, $subscriber_id, $hours_after_send = 1) { public function clickSentNewsletter(array $sent_newsletter_data, $subscriber_id, $created_at) {
$created_at = (new Carbon($sent_newsletter_data['sent_at']))->addHours($hours_after_send)->toDateTimeString(); StatisticsClicks::createOrUpdate([
$click = StatisticsClicks::createOrUpdate([
'subscriber_id' => $subscriber_id, 'subscriber_id' => $subscriber_id,
'newsletter_id' => $sent_newsletter_data['newsletter']->id, 'newsletter_id' => $sent_newsletter_data['newsletter_id'],
'queue_id' => $sent_newsletter_data['queue']->id, 'queue_id' => $sent_newsletter_data['queue_id'],
'link_id' => $sent_newsletter_data['link']->id, 'link_id' => $sent_newsletter_data['link_id'],
'count' => 1, 'count' => 1,
'created_at' => $created_at, 'created_at' => $created_at,
'updated_at' => $created_at, 'updated_at' => $created_at,
]); ])->save();
$click->save();
return $click;
} }
/** /**

76
tests/DataGenerator/Generators/WooCommercePastRevenues.php
View File

@@ -120,6 +120,10 @@ class WooCommercePastRevenues {
$sent_welcome_emails = []; $sent_welcome_emails = [];
foreach ($subscribers_ids as $subscriber_id) { foreach ($subscribers_ids as $subscriber_id) {
$sent_welcome_emails[$subscriber_id] = $this->helper->createSentEmailData($welcome_email, $minimal_created_at_date, [$subscriber_id], $subscribers_list->id); $sent_welcome_emails[$subscriber_id] = $this->helper->createSentEmailData($welcome_email, $minimal_created_at_date, [$subscriber_id], $subscribers_list->id);
$batch_log = $this->getBatchLog('Welcome emails sent', count($sent_welcome_emails));
if ($batch_log) {
yield $batch_log;
}
} }
yield "Welcome emails done"; yield "Welcome emails done";
@@ -177,60 +181,60 @@ class WooCommercePastRevenues {
foreach ($emails_to_send as $email) { foreach ($emails_to_send as $email) {
$sent_automatic_emails[$subscriber_id][] = $this->helper->createSentEmailData($email, $this->getRandomDateInPast(), [$subscriber_id], $subscribers_list->id); $sent_automatic_emails[$subscriber_id][] = $this->helper->createSentEmailData($email, $this->getRandomDateInPast(), [$subscriber_id], $subscribers_list->id);
} }
$batch_log = $this->getBatchLog('Automatic emails sent', count($sent_automatic_emails));
if ($batch_log) {
yield $batch_log;
}
} }
yield "Automatic emails done"; yield "Automatic emails done";
// Clicks // Clicks and orders
$subscribers_clicks = []; // Pick random subscribers which will have an order
$subscribers_with_orders = array_flip(array_intersect_key(
$subscribers_ids,
array_flip(array_rand($subscribers_ids, self::SUBSCRIBERS_WITH_ORDERS_COUNT))
));
$i = 0;
foreach ($subscribers_ids as $subscriber_id) { foreach ($subscribers_ids as $subscriber_id) {
$subscribers_clicks[$subscriber_id] = []; $i++;
$subscriber_click_times = [];
$subscriber_received_emails = array_merge( $subscriber_received_emails = array_merge(
$sent_automatic_emails[$subscriber_id], $sent_automatic_emails[$subscriber_id],
[$sent_welcome_emails[$subscriber_id]], [$sent_welcome_emails[$subscriber_id]],
$sent_post_notifications, $sent_post_notifications,
$sent_standard_newsletters $sent_standard_newsletters
); );
// Pick random half of received emails and generate opens and clicks // Pick amount of received emails and generate opens and clicks
$opened_count = floor(count($subscriber_received_emails)/rand(2,5));
$emails_to_click = array_intersect_key( $emails_to_click = array_intersect_key(
$subscriber_received_emails, $subscriber_received_emails,
array_flip(array_rand($subscriber_received_emails, ceil(count($subscriber_received_emails)/2))) array_flip(array_rand($subscriber_received_emails, $opened_count))
); );
// Click and open selected emails
foreach ($emails_to_click as $email) { foreach ($emails_to_click as $email) {
$this->helper->openSentNewsletter($email, $subscriber_id); $click_created_at = (new Carbon())->setTimestamp($email['sent_at'])->addHours(1)->toDateTimeString();
$subscribers_clicks[$subscriber_id][] = $this->helper->clickSentNewsletter($email, $subscriber_id); $this->helper->openSentNewsletter($email, $subscriber_id, $click_created_at);
$this->helper->clickSentNewsletter($email, $subscriber_id, $click_created_at);
$subscriber_click_times[] = $click_created_at;
} }
$batch_log = $this->getBatchLog('Subscriber clicks', count($subscribers_clicks)); // Create order
if (isset($subscribers_with_orders[$subscriber_id])) {
// Pick a random logged click time and generate an order day after the click
$click_time = $subscriber_click_times[array_rand($subscriber_click_times)];
$order_completed_at = (new Carbon($click_time))->addDay();
$this->helper->createCompletedWooCommerceOrder(
$subscriber_id,
$subscriber_emails[$subscriber_id],
[$products[array_rand($products)]],
$order_completed_at
);
}
$batch_log = $this->getBatchLog('Subscriber clicks and orders', $i);
if ($batch_log) { if ($batch_log) {
yield $batch_log; yield $batch_log;
} }
} }
yield "Clicks done"; yield "Clicks and Orders done";
// ORDERS
// Select random subscribers to generate orders for them
$subscribers_with_purchase_clicks = array_intersect_key(
$subscribers_clicks,
array_flip(array_rand($subscribers_clicks, self::SUBSCRIBERS_WITH_ORDERS_COUNT))
);
$i = 0;
foreach ($subscribers_with_purchase_clicks as $subscriber_id => $clicks) {
$i++;
// Pick a random logged click and generate an order day after the click
$click = $clicks[array_rand($clicks)];
$order_completed_at = (new Carbon($click->created_at))->addDay();
$this->helper->createCompletedWooCommerceOrder(
$subscriber_id,
$subscriber_emails[$subscriber_id],
[$products[array_rand($products)]],
$order_completed_at
);
$batch_log = $this->getBatchLog('Orders', $i);
if ($batch_log) {
yield $batch_log;
}
}
yield "Orders done";
$this->restoreDatabaseTables(); $this->restoreDatabaseTables();
} }
@@ -266,6 +270,7 @@ class WooCommercePastRevenues {
\ORM::rawExecute("ALTER TABLE `" . SendingQueue::$_table . "` DISABLE KEYS"); \ORM::rawExecute("ALTER TABLE `" . SendingQueue::$_table . "` DISABLE KEYS");
\ORM::rawExecute("ALTER TABLE `" . StatisticsOpens::$_table . "` DISABLE KEYS"); \ORM::rawExecute("ALTER TABLE `" . StatisticsOpens::$_table . "` DISABLE KEYS");
\ORM::rawExecute("ALTER TABLE `" . StatisticsClicks::$_table . "` DISABLE KEYS"); \ORM::rawExecute("ALTER TABLE `" . StatisticsClicks::$_table . "` DISABLE KEYS");
\ORM::rawExecute("SET UNIQUE_CHECKS = 0;");
} }
private function restoreDatabaseTables() { private function restoreDatabaseTables() {
@@ -287,5 +292,6 @@ class WooCommercePastRevenues {
\ORM::rawExecute("ALTER TABLE `" . SendingQueue::$_table . "` ENABLE KEYS"); \ORM::rawExecute("ALTER TABLE `" . SendingQueue::$_table . "` ENABLE KEYS");
\ORM::rawExecute("ALTER TABLE `" . StatisticsOpens::$_table . "` ENABLE KEYS"); \ORM::rawExecute("ALTER TABLE `" . StatisticsOpens::$_table . "` ENABLE KEYS");
\ORM::rawExecute("ALTER TABLE `" . StatisticsClicks::$_table . "` ENABLE KEYS"); \ORM::rawExecute("ALTER TABLE `" . StatisticsClicks::$_table . "` ENABLE KEYS");
\ORM::rawExecute("SET UNIQUE_CHECKS = 1;");
} }
} }