Cavemanon/piratepoet
8
Fork 0
Code Pull Requests Actions Releases 16 Activity

Add parallel execution for data generator

Browse Source 
[MAILPOET-3226]
This commit is contained in:
Rostislav Wolny
2021-05-06 15:32:43 +02:00
committed by Veljko V
parent ab00ff7869
commit 81fc441d4f
4 changed files with 74 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -104,6 +104,8 @@ $ ./do release:changelog-get [--version-name=...] # Prints out changelog an
$ ./do release:changelog-update [--version-name=...] [--quiet] # Updates changelog in readme.txt for given version or for newest version. $ ./do release:changelog-update [--version-name=...] [--quiet] # Updates changelog in readme.txt for given version or for newest version.
$ ./do container:dump # Generates DI container cache. $ ./do container:dump # Generates DI container cache.
$ ./do generate:data [<generatorName>] [<threads>] # Generates random usage data (Note: requires WooCommerce active) e.g. ./do generate:data past_revenues 4
``` ```
# Storybook # Storybook

22
RoboFile.php
View File

@@ -908,7 +908,27 @@ class RoboFile extends \Robo\Tasks {
->downloadReleaseZip('woocommerce.zip', __DIR__ . '/tests/plugins/', $tag); ->downloadReleaseZip('woocommerce.zip', __DIR__ . '/tests/plugins/', $tag);
} }
public function generateData($generatorName = null) { public function generateData($generatorName = null, $threads = 1) {
require_once __DIR__ . '/tests/DataGenerator/_bootstrap.php';
$generator = new \MailPoet\Test\DataGenerator\DataGenerator(new \Codeception\Lib\Console\Output([]));
$generator->runBefore($generatorName);
if ((int)$threads === 1) {
$this->generateUnitOfData($generatorName);
} else {
$parallelTask = $this->taskParallelExec();
for ($i = 1; $i <= $threads; $i++) {
$parallelTask = $parallelTask->process("./do generate:unit-of-data $generatorName");
}
$parallelTask->run();
}
$generator->runAfter($generatorName);
}
/**
* This is intended only for usage as a child process in parallel execution
* @param string|null $generatorName
*/
public function generateUnitOfData($generatorName = null) {
require_once __DIR__ . '/tests/DataGenerator/_bootstrap.php'; require_once __DIR__ . '/tests/DataGenerator/_bootstrap.php';
$generator = new \MailPoet\Test\DataGenerator\DataGenerator(new \Codeception\Lib\Console\Output([])); $generator = new \MailPoet\Test\DataGenerator\DataGenerator(new \Codeception\Lib\Console\Output([]));
$generator->run($generatorName); $generator->run($generatorName);

10
tests/DataGenerator/DataGenerator.php
View File

@@ -31,6 +31,16 @@ class DataGenerator {
$this->log($timer, 'DONE!'); $this->log($timer, 'DONE!');
} }
public function runBefore($generatorName = null) {
if (!$generatorName) $generatorName = self::PAST_REVENUES_GENERATOR;
$this->createGenerator($generatorName)->runBefore();
}
public function runAfter($generatorName = null) {
if (!$generatorName) $generatorName = self::PAST_REVENUES_GENERATOR;
$this->createGenerator($generatorName)->runAfter();
}
private function createGenerator($generatorName) { private function createGenerator($generatorName) {
switch ($generatorName) { switch ($generatorName) {
case self::PAST_REVENUES_GENERATOR: case self::PAST_REVENUES_GENERATOR:

69
tests/DataGenerator/Generators/WooCommercePastRevenues.php
View File

@@ -31,7 +31,6 @@ class WooCommercePastRevenues {
const STANDARD_NEWSLETTER = 30; const STANDARD_NEWSLETTER = 30;
public function generate() { public function generate() {
$this->prepareDatabaseTables();
// Reset hooks to prevent revenues calculation during generating // Reset hooks to prevent revenues calculation during generating
remove_all_actions('woocommerce_order_status_completed'); remove_all_actions('woocommerce_order_status_completed');
remove_all_actions('woocommerce_order_status_processing'); remove_all_actions('woocommerce_order_status_processing');
@@ -40,15 +39,15 @@ class WooCommercePastRevenues {
// Create list // Create list
$segmentFactory = new Segment(); $segmentFactory = new Segment();
$subscribersListEntity = $segmentFactory->withName('WC revenues load test')->create(); $subscribersListEntity = $segmentFactory->withName('WC revenues load test ' . $this->getRandomString())->create();
$subscribersList = \MailPoet\Models\Segment::findOne($subscribersListEntity->getId()); $subscribersList = \MailPoet\Models\Segment::findOne($subscribersListEntity->getId());
// Create subscribers // Create subscribers
$subscribersIds = []; $subscribersIds = [];
$subscriberEmails = []; $subscriberEmails = [];
for ($i = 1; $i <= self::SUBSCRIBERS_COUNT; $i++) { for ($i = 1; $i <= self::SUBSCRIBERS_COUNT; $i++) {
$email = "address$i@email.com"; $email = $this->getRandomString() . "address$i@email.com";
$subscriber = $this->createSubscriber("address$i@email.com", "last_name_$i", $minimalCreatedAtDate, $subscribersList); $subscriber = $this->createSubscriber($email, "last_name_$i", $minimalCreatedAtDate, $subscribersList);
$subscribersIds[] = $subscriber->id; $subscribersIds[] = $subscriber->id;
$subscriberEmails[$subscriber->id] = $email; $subscriberEmails[$subscriber->id] = $email;
$batchLog = $this->getBatchLog('Subscribers', count($subscribersIds)); $batchLog = $this->getBatchLog('Subscribers', count($subscribersIds));
@@ -59,10 +58,10 @@ class WooCommercePastRevenues {
yield "Subscribers done"; yield "Subscribers done";
// Products // Products
$productCategory = $this->createProductCategory('WC Revenues Test Category', 'revenues-test-cat'); $productCategory = $this->createProductCategory('WC Revenues Test Category ' . $this->getRandomString(), 'revenues-test-cat-' . $this->getRandomString());
$products = []; $products = [];
for ($i = 1; $i <= self::PRODUCTS_COUNT; $i++) { for ($i = 1; $i <= self::PRODUCTS_COUNT; $i++) {
$products[] = $this->createProduct("Product $i", 100, [$productCategory->term_id]); // phpcs:ignore Squiz.NamingConventions.ValidVariableName.NotCamelCaps $products[] = $this->createProduct("Product $i " . $this->getRandomString(), 100, [$productCategory->term_id]); // phpcs:ignore Squiz.NamingConventions.ValidVariableName.NotCamelCaps
} }
yield "Products done"; yield "Products done";
@@ -73,7 +72,7 @@ class WooCommercePastRevenues {
for ($i = 1; $i <= self::STANDARD_NEWSLETTER; $i++) { for ($i = 1; $i <= self::STANDARD_NEWSLETTER; $i++) {
$sentAt = $this->getRandomDateInPast(); $sentAt = $this->getRandomDateInPast();
$newsletter = $emailFactory $newsletter = $emailFactory
->withSubject("Standard $i") ->withSubject("Standard $i " . $this->getRandomString())
->withSegments([$subscribersListEntity]) ->withSegments([$subscribersListEntity])
->withCreatedAt($sentAt) ->withCreatedAt($sentAt)
->create(); ->create();
@@ -108,7 +107,7 @@ class WooCommercePastRevenues {
// Welcome emails // Welcome emails
$emailFactory = new Newsletter(); $emailFactory = new Newsletter();
$welcomeEmail = $emailFactory $welcomeEmail = $emailFactory
->withSubject("Welcome email") ->withSubject("Welcome email" . $this->getRandomString())
->withActiveStatus() ->withActiveStatus()
->withWelcomeTypeForSegment($subscribersList->id) ->withWelcomeTypeForSegment($subscribersList->id)
->withSegments([$subscribersListEntity]) ->withSegments([$subscribersListEntity])
@@ -130,7 +129,7 @@ class WooCommercePastRevenues {
$emailFactory = new Newsletter(); $emailFactory = new Newsletter();
// First purchase // First purchase
$automaticEmails[] = $emailFactory $automaticEmails[] = $emailFactory
->withSubject("First Purchase") ->withSubject("First Purchase" . $this->getRandomString())
->withActiveStatus() ->withActiveStatus()
->withAutomaticTypeWooCommerceFirstPurchase() ->withAutomaticTypeWooCommerceFirstPurchase()
->withSegments([]) ->withSegments([])
@@ -143,7 +142,7 @@ class WooCommercePastRevenues {
'name' => $products[$i]->get_id(), 'name' => $products[$i]->get_id(),
]; ];
$automaticEmails[] = $emailFactory $automaticEmails[] = $emailFactory
->withSubject("Purchased Product $i") ->withSubject("Purchased Product $i " . $this->getRandomString())
->withActiveStatus() ->withActiveStatus()
->withAutomaticTypeWooCommerceProductPurchased([$product]) ->withAutomaticTypeWooCommerceProductPurchased([$product])
->withSegments([]) ->withSegments([])
@@ -161,7 +160,7 @@ class WooCommercePastRevenues {
]], ]],
]; ];
$automaticEmails[] = $emailFactory $automaticEmails[] = $emailFactory
->withSubject("Purchased Product in Category $i") ->withSubject("Purchased Product in Category $i " . $this->getRandomString())
->withActiveStatus() ->withActiveStatus()
->withAutomaticTypeWooCommerceProductInCategoryPurchased([$product]) ->withAutomaticTypeWooCommerceProductInCategoryPurchased([$product])
->withSegments([]) ->withSegments([])
@@ -219,15 +218,18 @@ class WooCommercePastRevenues {
} }
// Create order // Create order
if (isset($subscribersWithOrders[$subscriberId])) { if (isset($subscribersWithOrders[$subscriberId])) {
// Pick a random logged click time and generate an order day after the click $orderCount = rand(1, 5);
$clickTime = $subscriberClickTimes[array_rand($subscriberClickTimes)]; for ($i = 1; $i <= $orderCount; $i++) {
$orderCompletedAt = (new Carbon($clickTime))->addDay(); // Pick a random logged click time and generate an order day after the click
$this->createCompletedWooCommerceOrder( $clickTime = $subscriberClickTimes[array_rand($subscriberClickTimes)];
$subscriberId, $orderCompletedAt = (new Carbon($clickTime))->addDay();
$subscriberEmails[$subscriberId], $this->createCompletedWooCommerceOrder(
[$products[array_rand($products)]], $subscriberId,
$orderCompletedAt $subscriberEmails[$subscriberId],
); [$products[array_rand($products)]],
$orderCompletedAt
);
}
} }
$batchLog = $this->getBatchLog('Subscriber clicks and orders', $i); $batchLog = $this->getBatchLog('Subscriber clicks and orders', $i);
if ($batchLog) { if ($batchLog) {
@@ -235,7 +237,6 @@ class WooCommercePastRevenues {
} }
} }
yield "Clicks and Orders done"; yield "Clicks and Orders done";
$this->restoreDatabaseTables();
} }
private function getRandomDateInPast() { private function getRandomDateInPast() {
@@ -250,7 +251,7 @@ class WooCommercePastRevenues {
return "$dataType: $generatedCount"; return "$dataType: $generatedCount";
} }
private function prepareDatabaseTables() { public function runBefore() {
// Turn off CURRENT_TIMESTAMP to be able to save generated value // Turn off CURRENT_TIMESTAMP to be able to save generated value
ORM::rawExecute( ORM::rawExecute(
"ALTER TABLE `" . StatisticsClicks::$_table . "` "ALTER TABLE `" . StatisticsClicks::$_table . "`
@@ -273,7 +274,7 @@ class WooCommercePastRevenues {
ORM::rawExecute("SET UNIQUE_CHECKS = 0;"); ORM::rawExecute("SET UNIQUE_CHECKS = 0;");
} }
private function restoreDatabaseTables() { public function runAfter() {
ORM::rawExecute( ORM::rawExecute(
"ALTER TABLE `" . StatisticsClicks::$_table . "` "ALTER TABLE `" . StatisticsClicks::$_table . "`
CHANGE `updated_at` `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP;" CHANGE `updated_at` `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP;"
@@ -429,15 +430,17 @@ class WooCommercePastRevenues {
* @return \WC_Order|\WP_Error * @return \WC_Order|\WP_Error
*/ */
private function createCompletedWooCommerceOrder($subscriberId, $email, $products = [], Carbon $completedAt = null) { private function createCompletedWooCommerceOrder($subscriberId, $email, $products = [], Carbon $completedAt = null) {
$random = $this->getRandomString();
$countries = ['FR', 'GB', 'US', 'IE', 'IT'];
$address = [ $address = [
'first_name' => "name_$subscriberId", 'first_name' => "{$random}_name_{$subscriberId}",
'last_name' => "lastname_$subscriberId", 'last_name' => "{$random}_lastname_{$subscriberId}",
'email' => $email, 'email' => $email,
'phone' => '123-456-789', 'phone' => '123-456-789',
'address_1' => "$subscriberId Main st.", 'address_1' => "{$random} {$subscriberId} Main st.",
'city' => "City of $subscriberId", 'city' => "City of {$random} {$subscriberId}",
'postcode' => '92121', 'postcode' => '92121',
'country' => 'France', 'country' => $countries[array_rand($countries)],
]; ];
$order = wc_create_order(); $order = wc_create_order();
@@ -462,4 +465,14 @@ class WooCommercePastRevenues {
} }
return $order; return $order;
} }
private function getRandomString($length = 5) {
$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$charactersLength = strlen($characters);
$randomString = '';
for ($i = 0; $i < $length; $i++) {
$randomString .= $characters[rand(0, $charactersLength - 1)];
}
return $randomString;
}
} }