MQE-778: robo generate:tests generates all tests and suites by default

- add new generate all method to SuiteGenerator - modify TestManifest to only clear once - introduce argument to exclude tests during generation

Author: imeron2433

dev/tests/verification/Tests/SuiteGenerationTest.php

@@ -17,13 +17,6 @@ class SuiteGenerationTest extends MftfTestCase
  const RESOURCES_DIR = TESTS_BP . DIRECTORY_SEPARATOR . 'verification' . DIRECTORY_SEPARATOR . 'Resources';
  const CONFIG_YML_FILE = FW_BP . DIRECTORY_SEPARATOR . SuiteGenerator::YAML_CODECEPTION_CONFIG_FILENAME;
 
- const MANIFEST_RESULTS = [
- 'IncludeTest2Cest.php',
- 'additionalTestCest.php',
- 'IncludeTestCest.php',
- 'additionalIncludeTest2Cest.php'
- ];
-
  /**
  * Flag to track existence of config.yml file
  *
@@ -48,6 +41,11 @@ public static function setUpBeforeClass()
  return;
  }
 
+ // destroy manifest file if it exists
+ if (file_exists(self::getManifestFilePath())) {
+ unlink(self::getManifestFilePath());
+ }
+
  $configYml = fopen(self::CONFIG_YML_FILE, "w");
  fclose($configYml);
  }
@@ -59,6 +57,13 @@ public function testSuiteGeneration1()
  {
  $groupName = 'functionalSuite1';
 
+ $expectedContents = [
+ 'additionalTestCest.php',
+ 'additionalIncludeTest2Cest.php',
+ 'IncludeTest2Cest.php',
+ 'IncludeTestCest.php'
+ ];
+
  // Generate the Suite
  SuiteGenerator::getInstance()->generateSuite($groupName);
 
@@ -79,23 +84,10 @@ public function testSuiteGeneration1()
  $groupName .
  DIRECTORY_SEPARATOR;
 
- // Validate test manifest contents
- $actualManifest = dirname($suiteResultBaseDir). DIRECTORY_SEPARATOR . 'testManifest.txt';
- $actualTestReferences = explode(PHP_EOL, file_get_contents($actualManifest));
-
- for ($i = 0; $i < count($actualTestReferences); $i++) {
- if (empty($actualTestReferences[$i])) {
- continue;
- }
-
- $this->assertStringEndsWith(self::MANIFEST_RESULTS[$i], $actualTestReferences[$i]);
- $this->assertNotFalse(strpos($actualTestReferences[$i], $groupName));
- }
-
- // Validate expected php files exist
- foreach (self::MANIFEST_RESULTS as $expectedTestReference) {
- $cestName = explode(":", $expectedTestReference, 2);
- $this->assertFileExists($suiteResultBaseDir . $cestName[0]);
+ // Validate tests have been generated
+ $dirContents = array_diff(scandir($suiteResultBaseDir), ['..', '.']);
+ foreach ($expectedContents as $expectedFile) {
+ $this->assertTrue(in_array($expectedFile, $dirContents));
  }
  }
 
@@ -117,4 +109,20 @@ public static function tearDownAfterClass()
 
  unlink(self::CONFIG_YML_FILE);
  }
+
+ /**
+ * Getter for manifest file path
+ *
+ * @return string
+ */
+ private static function getManifestFilePath()
+ {
+ return TESTS_BP .
+ DIRECTORY_SEPARATOR .
+ "verification" .
+ DIRECTORY_SEPARATOR .
+ "_generated" .
+ DIRECTORY_SEPARATOR .
+ 'testManifest.txt';
+ }
 }

src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

@@ -32,6 +32,13 @@ class SuiteGenerator
  */
  private static $SUITE_GENERATOR_INSTANCE;
 
+ /**
+ * Boolean to track whether we have already cleared the yaml file.
+ *
+ * @var bool
+ */
+ private $ymlFileCleared = false;
+
  /**
  * Group Class Generator initialized in constructor.
  *
@@ -61,14 +68,35 @@ public static function getInstance()
  return self::$SUITE_GENERATOR_INSTANCE;
  }
 
+ /**
+ * Function which takes all suite configurations and generates to appropriate directory, updating yml configuration
+ * as needed. Returns an array of all tests generated keyed by test name.
+ *
+ * @param string $config
+ * @return array
+ */
+ public function generateAllSuites($config)
+ {
+ $testsReferencedInSuites = [];
+ $suites = SuiteObjectHandler::getInstance()->getAllObjects();
+ foreach ($suites as $suite) {
+ /** @var SuiteObject $suite */
+ $testsReferencedInSuites = array_merge($testsReferencedInSuites, $suite->getTests());
+ $this->generateSuite($suite->getName(), $config);
+ }
+
+ return $testsReferencedInSuites;
+ }
+
  /**
  * Function which takes a suite name and generates corresponding dir, test files, group class, and updates
  * yml configuration for group run.
  *
  * @param string $suiteName
+ * @param string $config
  * @return void
  */
- public function generateSuite($suiteName)
+ public function generateSuite($suiteName, $config = null)
  {
  /**@var SuiteObject $suite **/
  $suite = SuiteObjectHandler::getInstance()->getObject($suiteName);
@@ -77,7 +105,7 @@ public function generateSuite($suiteName)
  $groupNamespace = null;
 
  DirSetupUtil::createGroupDir($fullPath);
- $this->generateRelevantGroupTests($suiteName, $suite->getTests());
+ $this->generateRelevantGroupTests($suiteName, $suite->getTests(), $config);
 
  if ($suite->requiresGroupFile()) {
  // if the suite requires a group file, generate it and set the namespace
@@ -117,29 +145,69 @@ private function appendEntriesToConfig($suiteName, $suitePath, $groupNamespace)
  $ymlArray[self::YAML_GROUPS_TAG]= [];
  }
 
- $ymlArray[self::YAML_GROUPS_TAG][$suiteName] = [$relativeSuitePath];
+ $ymlArray = $this->clearPreviousSessionConfigEntries($ymlArray);
 
- if ($groupNamespace &&
- !in_array($groupNamespace, $ymlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG])) {
+ if ($groupNamespace) {
  $ymlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG][] = $groupNamespace;
  }
+ $ymlArray[self::YAML_GROUPS_TAG][$suiteName] = [$relativeSuitePath];
 
  $ymlText = self::YAML_COPYRIGHT_TEXT . Yaml::dump($ymlArray, 10);
  file_put_contents($configYmlFile, $ymlText);
  }
 
+ /**
+ * Function which takes the current config.yml array and clears any previous configuration for suite group object
+ * files.
+ *
+ * @param array $ymlArray
+ * @return array
+ */
+ private function clearPreviousSessionConfigEntries($ymlArray)
+ {
+ if ($this->ymlFileCleared) {
+ return $ymlArray;
+ }
+
+ $newYmlArray = $ymlArray;
+ // if the yaml entries haven't already been cleared
+
+ if (array_key_exists(self::YAML_EXTENSIONS_TAG, $ymlArray)) {
+ foreach ($ymlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG] as $key => $entry) {
+ if (preg_match('/(Group\\\\.*)/', $entry)) {
+ unset($newYmlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG][$key]);
+ }
+
+ }
+
+ // needed for proper yml file generation based on indices
+ $newYmlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG] =
+ array_values($newYmlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG]);
+
+ }
+
+ if (array_key_exists(self::YAML_GROUPS_TAG, $newYmlArray)) {
+ unset($newYmlArray[self::YAML_GROUPS_TAG]);
+ }
+
+ $this->ymlFileCleared = true;
+
+ return $newYmlArray;
+ }
+
  /**
  * Function which takes a string which is the desired output directory (under _generated) and an array of tests
  * relevant to the suite to be generated. The function takes this information and creates a new instance of the test
  * generator which is then called to create all the test files for the suite.
  *
  * @param string $path
  * @param array $tests
+ * @param string $config
  * @return void
  */
- private function generateRelevantGroupTests($path, $tests)
+ private function generateRelevantGroupTests($path, $tests, $config)
  {
  $testGenerator = TestGenerator::getInstance($path, $tests);
- $testGenerator->createAllTestFiles();
+ $testGenerator->createAllTestFiles($config);
  }
 }

src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php

@@ -44,6 +44,12 @@ public function parseSuiteDataIntoObjects($parsedSuiteData)
  {
  $suiteObjects = [];
  $testHookObjectExtractor = new TestHookObjectExtractor();
+
+ // make sure there are suites defined before trying to parse as objects.
+ if (!array_key_exists(self::SUITE_ROOT_TAG, $parsedSuiteData)) {
+ return $suiteObjects;
+ }
+
  foreach ($parsedSuiteData[self::SUITE_ROOT_TAG] as $parsedSuite) {
  if (!is_array($parsedSuite)) {
  // skip non array items parsed from suite (suite objects will always be arrays)

src/Magento/FunctionalTestingFramework/Util/Filesystem/DirSetupUtil.php

@@ -11,6 +11,13 @@
 
 class DirSetupUtil
 {
+ /**
+ * Array which will track any previously cleared directories, to prevent any unintended removal.
+ *
+ * @var array
+ */
+ private static $DIR_CONTEXT = [];
+
  /**
  * Method used to clean export dir if needed and create new empty export dir.
  *
@@ -19,11 +26,17 @@ class DirSetupUtil
  */
  public static function createGroupDir($fullPath)
  {
+ // make sure we haven't already cleaned up this directory at any point before deletion
+ if (in_array($fullPath, self::$DIR_CONTEXT)) {
+ return;
+ }
+
  if (file_exists($fullPath)) {
  self::rmDirRecursive($fullPath);
  }
 
  mkdir($fullPath, 0777, true);
+ self::$DIR_CONTEXT[] = $fullPath;
  }
 
  /**

src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php

@@ -7,6 +7,7 @@
 namespace Magento\FunctionalTestingFramework\Util\Manifest;
 
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
+use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
 
 class DefaultTestManifest extends BaseTestManifest
 {
@@ -19,6 +20,13 @@ class DefaultTestManifest extends BaseTestManifest
  */
  protected $manifestPath;
 
+ /**
+ * A static array to track which test manifests have been cleared to prevent overwriting during generation.
+ *
+ * @var array
+ */
+ private static $CLEARED_MANIFESTS = [];
+
  /**
  * An array containing all test names for output.
  *
@@ -34,8 +42,9 @@ class DefaultTestManifest extends BaseTestManifest
  public function __construct($manifestPath, $testPath)
  {
  $this->manifestPath = $manifestPath . DIRECTORY_SEPARATOR . 'testManifest.txt';
+ $this->cleanManifest($this->manifestPath);
  parent::__construct($testPath, self::DEFAULT_CONFIG);
- $fileResource = fopen($this->manifestPath, 'w');
+ $fileResource = fopen($this->manifestPath, 'a');
  fclose($fileResource);
  }
 
@@ -67,4 +76,26 @@ public function generate($nodes = null)
 
  fclose($fileResource);
  }
+
+ /**
+ * Function which checks the path for an existing test manifest and clears if the file has not already been cleared
+ * during current runtime.
+ *
+ * @param string $path
+ * @return void
+ */
+ private function cleanManifest($path)
+ {
+ // if we have already cleared the file then simply return
+ if (in_array($path, self::$CLEARED_MANIFESTS)) {
+ return;
+ }
+
+ // if the file exists remove
+ if (file_exists($path)) {
+ unlink($path);
+ }
+
+ self::$CLEARED_MANIFESTS[] = $path;
+ }
 }

src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php

@@ -19,7 +19,7 @@ public function __construct($manifestPath, $testPath)
  {
  parent::__construct($manifestPath, $testPath);
  $this->runTypeConfig = self::SINGLE_RUN_CONFIG;
- $fileResource = fopen($this->manifestPath, 'w');
+ $fileResource = fopen($this->manifestPath, 'a');
  fclose($fileResource);
  }
 

src/Magento/FunctionalTestingFramework/Util/TestGenerator.php

@@ -135,11 +135,12 @@ private function createCestFile($testPhp, $filename)
  *
  * @param string $runConfig
  * @param int $nodes
+ * @param TestObject[] $testsToIgnore
  * @return void
  * @throws TestReferenceException
  * @throws \Exception
  */
- public function createAllTestFiles($runConfig = null, $nodes = null)
+ public function createAllTestFiles($runConfig = null, $nodes = null, $testsToIgnore = [])
  {
  DirSetupUtil::createGroupDir($this->exportDirectory);
 
@@ -149,8 +150,8 @@ public function createAllTestFiles($runConfig = null, $nodes = null)
  $this->exportDirectory,
  $runConfig
  );
- $testPhpArray = $this->assembleAllTestPhp($testManifest, $nodes);
 
+ $testPhpArray = $this->assembleAllTestPhp($testManifest, $nodes, $testsToIgnore);
  foreach ($testPhpArray as $testPhpFile) {
  $this->createCestFile($testPhpFile[1], $testPhpFile[0]);
  }
@@ -196,14 +197,16 @@ private function assembleTestPhp($testObject)
  *
  * @param BaseTestManifest $testManifest
  * @param int $nodes
+ * @param TestObject[] $testsToIgnore
  * @return array
  * @throws TestReferenceException
  * @throws \Exception
  */
- private function assembleAllTestPhp($testManifest, $nodes)
+ private function assembleAllTestPhp($testManifest, $nodes, $testsToIgnore)
  {
  /** @var TestObject[] $testObjects */
  $testObjects = $this->loadAllTestObjects();
+ $testObjects = array_diff_key($testObjects, $testsToIgnore);
  $cestPhpArray = [];
 
  foreach ($testObjects as $test) {