Add parallel execution for data generator

[MAILPOET-3226]
2021-05-06 15:32:43 +02:00
parent ab00ff7869
commit 81fc441d4f
4 changed files with 74 additions and 29 deletions
--- a/README.md
+++ b/README.md
@@ -104,6 +104,8 @@ $ ./do release:changelog-get  [--version-name=...]     # Prints out changelog an
 $ ./do release:changelog-update  [--version-name=...] [--quiet] # Updates changelog in readme.txt for given version or for newest version.

 $ ./do container:dump      # Generates DI container cache.
+
+$ ./do generate:data [<generatorName>] [<threads>] # Generates random usage data (Note: requires WooCommerce active) e.g. ./do generate:data past_revenues 4
 ```

 # Storybook
--- a/RoboFile.php
+++ b/RoboFile.php
@@ -908,7 +908,27 @@ class RoboFile extends \Robo\Tasks {
      ->downloadReleaseZip('woocommerce.zip', __DIR__ . '/tests/plugins/', $tag);
  }

-  public function generateData($generatorName = null) {
+  public function generateData($generatorName = null, $threads = 1) {
+    require_once __DIR__ . '/tests/DataGenerator/_bootstrap.php';
+    $generator = new \MailPoet\Test\DataGenerator\DataGenerator(new \Codeception\Lib\Console\Output([]));
+    $generator->runBefore($generatorName);
+    if ((int)$threads === 1) {
+      $this->generateUnitOfData($generatorName);
+    } else {
+      $parallelTask = $this->taskParallelExec();
+      for ($i = 1; $i <= $threads; $i++) {
+        $parallelTask = $parallelTask->process("./do generate:unit-of-data $generatorName");
+      }
+      $parallelTask->run();
+    }
+    $generator->runAfter($generatorName);
+  }
+
+  /**
+   * This is intended only for usage as a child process in parallel execution
+   * @param string|null $generatorName
+   */
+  public function generateUnitOfData($generatorName = null) {
    require_once __DIR__ . '/tests/DataGenerator/_bootstrap.php';
    $generator = new \MailPoet\Test\DataGenerator\DataGenerator(new \Codeception\Lib\Console\Output([]));
    $generator->run($generatorName);
--- a/tests/DataGenerator/DataGenerator.php
+++ b/tests/DataGenerator/DataGenerator.php
@@ -31,6 +31,16 @@ class DataGenerator {
    $this->log($timer, 'DONE!');
  }

+  public function runBefore($generatorName = null) {
+    if (!$generatorName) $generatorName = self::PAST_REVENUES_GENERATOR;
+    $this->createGenerator($generatorName)->runBefore();
+  }
+
+  public function runAfter($generatorName = null) {
+    if (!$generatorName) $generatorName = self::PAST_REVENUES_GENERATOR;
+    $this->createGenerator($generatorName)->runAfter();
+  }
+
  private function createGenerator($generatorName) {
    switch ($generatorName) {
      case self::PAST_REVENUES_GENERATOR:
--- a/tests/DataGenerator/Generators/WooCommercePastRevenues.php
+++ b/tests/DataGenerator/Generators/WooCommercePastRevenues.php
@@ -31,7 +31,6 @@ class WooCommercePastRevenues {
  const STANDARD_NEWSLETTER = 30;

  public function generate() {
-    $this->prepareDatabaseTables();
    // Reset hooks to prevent revenues calculation during generating
    remove_all_actions('woocommerce_order_status_completed');
    remove_all_actions('woocommerce_order_status_processing');
@@ -40,15 +39,15 @@ class WooCommercePastRevenues {

    // Create list
    $segmentFactory = new Segment();
-    $subscribersListEntity = $segmentFactory->withName('WC revenues load test')->create();
+    $subscribersListEntity = $segmentFactory->withName('WC revenues load test ' . $this->getRandomString())->create();
    $subscribersList = \MailPoet\Models\Segment::findOne($subscribersListEntity->getId());

    // Create subscribers
    $subscribersIds = [];
    $subscriberEmails = [];
    for ($i = 1; $i <= self::SUBSCRIBERS_COUNT; $i++) {
-      $email = "address$i@email.com";
-      $subscriber = $this->createSubscriber("address$i@email.com", "last_name_$i", $minimalCreatedAtDate, $subscribersList);
+      $email = $this->getRandomString() . "address$i@email.com";
+      $subscriber = $this->createSubscriber($email, "last_name_$i", $minimalCreatedAtDate, $subscribersList);
      $subscribersIds[] = $subscriber->id;
      $subscriberEmails[$subscriber->id] = $email;
      $batchLog = $this->getBatchLog('Subscribers', count($subscribersIds));
@@ -59,10 +58,10 @@ class WooCommercePastRevenues {
    yield "Subscribers done";

    // Products
-    $productCategory = $this->createProductCategory('WC Revenues Test Category', 'revenues-test-cat');
+    $productCategory = $this->createProductCategory('WC Revenues Test Category ' . $this->getRandomString(), 'revenues-test-cat-' . $this->getRandomString());
    $products = [];
    for ($i = 1; $i <= self::PRODUCTS_COUNT; $i++) {
-      $products[] = $this->createProduct("Product $i", 100, [$productCategory->term_id]); // phpcs:ignore Squiz.NamingConventions.ValidVariableName.NotCamelCaps
+      $products[] = $this->createProduct("Product $i " . $this->getRandomString(), 100, [$productCategory->term_id]); // phpcs:ignore Squiz.NamingConventions.ValidVariableName.NotCamelCaps
    }
    yield "Products done";

@@ -73,7 +72,7 @@ class WooCommercePastRevenues {
    for ($i = 1; $i <= self::STANDARD_NEWSLETTER; $i++) {
      $sentAt = $this->getRandomDateInPast();
      $newsletter = $emailFactory
-        ->withSubject("Standard $i")
+        ->withSubject("Standard $i " . $this->getRandomString())
        ->withSegments([$subscribersListEntity])
        ->withCreatedAt($sentAt)
        ->create();
@@ -108,7 +107,7 @@ class WooCommercePastRevenues {
    // Welcome emails
    $emailFactory = new Newsletter();
    $welcomeEmail = $emailFactory
-      ->withSubject("Welcome email")
+      ->withSubject("Welcome email" . $this->getRandomString())
      ->withActiveStatus()
      ->withWelcomeTypeForSegment($subscribersList->id)
      ->withSegments([$subscribersListEntity])
@@ -130,7 +129,7 @@ class WooCommercePastRevenues {
    $emailFactory = new Newsletter();
    // First purchase
    $automaticEmails[] = $emailFactory
-      ->withSubject("First Purchase")
+      ->withSubject("First Purchase" . $this->getRandomString())
      ->withActiveStatus()
      ->withAutomaticTypeWooCommerceFirstPurchase()
      ->withSegments([])
@@ -143,7 +142,7 @@ class WooCommercePastRevenues {
        'name' => $products[$i]->get_id(),
      ];
      $automaticEmails[] = $emailFactory
-        ->withSubject("Purchased Product $i")
+        ->withSubject("Purchased Product $i " . $this->getRandomString())
        ->withActiveStatus()
        ->withAutomaticTypeWooCommerceProductPurchased([$product])
        ->withSegments([])
@@ -161,7 +160,7 @@ class WooCommercePastRevenues {
        ]],
      ];
      $automaticEmails[] = $emailFactory
-        ->withSubject("Purchased Product in Category $i")
+        ->withSubject("Purchased Product in Category $i " . $this->getRandomString())
        ->withActiveStatus()
        ->withAutomaticTypeWooCommerceProductInCategoryPurchased([$product])
        ->withSegments([])
@@ -219,15 +218,18 @@ class WooCommercePastRevenues {
      }
      // Create order
      if (isset($subscribersWithOrders[$subscriberId])) {
-        // Pick a random logged click time and generate an order day after the click
-        $clickTime = $subscriberClickTimes[array_rand($subscriberClickTimes)];
-        $orderCompletedAt = (new Carbon($clickTime))->addDay();
-        $this->createCompletedWooCommerceOrder(
-          $subscriberId,
-          $subscriberEmails[$subscriberId],
-          [$products[array_rand($products)]],
-          $orderCompletedAt
-        );
+        $orderCount = rand(1, 5);
+        for ($i = 1; $i <= $orderCount; $i++) {
+          // Pick a random logged click time and generate an order day after the click
+          $clickTime = $subscriberClickTimes[array_rand($subscriberClickTimes)];
+          $orderCompletedAt = (new Carbon($clickTime))->addDay();
+          $this->createCompletedWooCommerceOrder(
+            $subscriberId,
+            $subscriberEmails[$subscriberId],
+            [$products[array_rand($products)]],
+            $orderCompletedAt
+          );
+        }
      }
      $batchLog = $this->getBatchLog('Subscriber clicks and orders', $i);
      if ($batchLog) {
@@ -235,7 +237,6 @@ class WooCommercePastRevenues {
      }
    }
    yield "Clicks and Orders done";
-    $this->restoreDatabaseTables();
  }

  private function getRandomDateInPast() {
@@ -250,7 +251,7 @@ class WooCommercePastRevenues {
    return "$dataType: $generatedCount";
  }

-  private function prepareDatabaseTables() {
+  public function runBefore() {
    // Turn off CURRENT_TIMESTAMP to be able to save generated value
    ORM::rawExecute(
      "ALTER TABLE `" . StatisticsClicks::$_table . "`
@@ -273,7 +274,7 @@ class WooCommercePastRevenues {
    ORM::rawExecute("SET UNIQUE_CHECKS = 0;");
  }

-  private function restoreDatabaseTables() {
+  public function runAfter() {
    ORM::rawExecute(
      "ALTER TABLE `" . StatisticsClicks::$_table . "`
      CHANGE `updated_at` `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP;"
@@ -429,15 +430,17 @@ class WooCommercePastRevenues {
   * @return \WC_Order|\WP_Error
   */
  private function createCompletedWooCommerceOrder($subscriberId, $email, $products = [], Carbon $completedAt = null) {
+    $random = $this->getRandomString();
+    $countries = ['FR', 'GB', 'US', 'IE', 'IT'];
    $address = [
-      'first_name' => "name_$subscriberId",
-      'last_name' => "lastname_$subscriberId",
+      'first_name' => "{$random}_name_{$subscriberId}",
+      'last_name' => "{$random}_lastname_{$subscriberId}",
      'email' => $email,
      'phone' => '123-456-789',
-      'address_1' => "$subscriberId Main st.",
-      'city' => "City of $subscriberId",
+      'address_1' => "{$random} {$subscriberId} Main st.",
+      'city' => "City of {$random} {$subscriberId}",
      'postcode' => '92121',
-      'country' => 'France',
+      'country' => $countries[array_rand($countries)],
    ];

    $order = wc_create_order();
@@ -462,4 +465,14 @@ class WooCommercePastRevenues {
    }
    return $order;
  }
+
+  private function getRandomString($length = 5) {
+    $characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+    $charactersLength = strlen($characters);
+    $randomString = '';
+    for ($i = 0; $i < $length; $i++) {
+      $randomString .= $characters[rand(0, $charactersLength - 1)];
+    }
+    return $randomString;
+  }
 }