diff options
author | diosmosis <benaka@piwik.pro> | 2014-12-29 03:37:05 +0300 |
---|---|---|
committer | diosmosis <benaka@piwik.pro> | 2014-12-29 03:37:16 +0300 |
commit | 2847e40b5e7e5a00e64eb05c68a193b758566405 (patch) | |
tree | aa89686990b7bb1e64f25fdd827c32eff75f1521 /plugins/TestRunner/TravisYml | |
parent | 41a0e0df6c642b8ddf90f2ea0972418dd89e7e16 (diff) |
Refs #6076, refactor .travis.yml generation code and allow more than 2 contexts for generation. Added context for generating the .travis.yml file for piwik-tests-plugins repo.
Diffstat (limited to 'plugins/TestRunner/TravisYml')
6 files changed, 708 insertions, 0 deletions
diff --git a/plugins/TestRunner/TravisYml/Generator.php b/plugins/TestRunner/TravisYml/Generator.php new file mode 100644 index 0000000000..33d57bc2fe --- /dev/null +++ b/plugins/TestRunner/TravisYml/Generator.php @@ -0,0 +1,201 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ +namespace Piwik\Plugins\TestRunner\TravisYml; + +use Exception; +use Piwik\Container\StaticContainer; +use Piwik\Plugins\TestRunner\Commands\GenerateTravisYmlFile; +use Psr\Log\LoggerInterface; + +/** + * Base class for .travis.yml file generators. + */ +abstract class Generator +{ + /** + * @var string[] + */ + protected $options; + + /** + * @var LoggerInterface + */ + protected $logger; + + /** + * @var TravisYmlView + */ + protected $view; + + /** + * Constructor. + * + * @param string[] $options The string options applied to the generate:travis-yml command. + */ + public function __construct($options) + { + $this->options = $options; + $this->logger = StaticContainer::getContainer()->get('Psr\Log\LoggerInterface'); + + $this->view = new TravisYmlView(); + } + + /** + * Generates the contents of a .travis.yml file and returns them. + * + * @return string + */ + public function generate() + { + $this->configureView(); + + return $this->view->render(); + } + + /** + * Writes the contents of a .travis.yml file to the correct destination. If the --dump option + * is specified, the file is saved here instead of the .travis.yml file it should be saved to. + * + * @param string $travisYmlContents + * @return string Returns the path of the file that was written to. + * @throws Exception if the path being written is not writable. + */ + public function dumpTravisYmlContents($travisYmlContents) + { + $writePath = @$this->options['dump']; + if (empty($writePath)) { + $writePath = $this->getTravisYmlOutputPath(); + } + + if (!is_writable($writePath)) { + throw new Exception("Cannot write to '$writePath'!"); + } + + file_put_contents($writePath, $travisYmlContents); + + return $writePath; + } + + /** + * Returns the path of the .travis.yml file we are generating. The --dump option has no effect on + * this path. + */ + public abstract function getTravisYmlOutputPath(); + + protected function configureView() + { + $thisConsoleCommand = $this->getExecutedConsoleCommandForTravis(); + $this->view->setGenerateYmlCommand($thisConsoleCommand); + + $phpVersions = @$this->options['php-versions']; + if (!empty($phpVersions)) { + $this->view->setPhpVersions(explode(',', $phpVersions)); + } + + $outputYmlPath = $this->getTravisYmlOutputPath(); + if (file_exists($outputYmlPath)) { + $this->logger->info("Found existing YAML file at {path}.", array('path' => $outputYmlPath)); + + $parser = new Parser(); + $existingSections = $parser->processExistingTravisYml($outputYmlPath); + $this->view->setExistingSections($existingSections); + } else { + $this->logger->info("Could not find existing YAML file at {path}, generating a new one.", array('path' => $outputYmlPath)); + } + + $this->setExtraEnvironmentVariables(); + } + + protected function travisEncrypt($data) + { + $this->logger->info("Encrypting \"{data}\"...", $data); + + $command = "travis encrypt \"$data\""; + + exec($command, $commandOutput, $returnCode); + if ($returnCode !== 0) { + throw new Exception("Cannot encrypt \"$data\" for travis! Please make sure you have the travis command line " + . "utility installed (see http://blog.travis-ci.com/2013-01-14-new-client/).\n\n" + . "return code: $returnCode\n\n" + . "travis output:\n\n" . implode("\n", $commandOutput)); + } + + if (empty($commandOutput)) { + throw new Exception("Cannot parse travis encrypt output:\n\n" . implode("\n", $commandOutput)); + } + + // when not executed from a command line travis encrypt will return only the encrypted data + $encryptedData = $commandOutput[0]; + + if (substr($encryptedData, 0, 1) != '"' + || substr($encryptedData, -1) != '"' + ) { + $encryptedData = '"' . addslashes($encryptedData) . '"'; + } + + return "secure: " . $encryptedData; + } + + protected function getExecutedConsoleCommandForTravis() + { + $command = "php ./console " . GenerateTravisYmlFile::COMMAND_NAME; + + $options = $this->getOptionsForSelfReferentialCommand(); + + foreach ($options as $name => $value) { + if ($value === false + || $value === null + ) { + continue; + } + + if ($value === true) { + $command .= " --$name"; + } else if (is_array($value)) { + foreach ($value as $arrayValue) { + $command .= " --$name=\"" . addslashes($arrayValue) . "\""; + } + } else { + $command .= " --$name=\"" . addslashes($value) . "\""; + } + } + + return $command; + } + + private function setExtraEnvironmentVariables() + { + if (!empty($this->view->existingEnv)) { + $this->logger->info("Existing .yml file found, ignoring global variables specified on command line."); + return; + } + + $extraVars = array(); + + $artifactsPass = @$this->options['artifacts-pass']; + if (!empty($artifactsPass)) { + $extraVars[] = $this->travisEncrypt("ARTIFACTS_PASS=" . $artifactsPass); + } + + $githubToken = @$this->options['github-token']; + if (!empty($githubToken)) { + $extraVars[] = $this->travisEncrypt("GITHUB_USER_TOKEN=" . $githubToken); + } + + $this->view->setExtraGlobalEnvVars($extraVars); + } + + protected function getOptionsForSelfReferentialCommand() + { + $options = $this->options; + unset($options['github-token']); + unset($options['artifacts-pass']); + unset($options['dump']); + return $options; + } +}
\ No newline at end of file diff --git a/plugins/TestRunner/TravisYml/Generator/CoreTravisYmlGenerator.php b/plugins/TestRunner/TravisYml/Generator/CoreTravisYmlGenerator.php new file mode 100644 index 0000000000..9fb549daac --- /dev/null +++ b/plugins/TestRunner/TravisYml/Generator/CoreTravisYmlGenerator.php @@ -0,0 +1,25 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ +namespace Piwik\Plugins\TestRunner\TravisYml\Generator; + +use Piwik\Plugins\TestRunner\TravisYml\Generator; + +class CoreTravisYmlGenerator extends Generator +{ + protected function configureView() + { + parent::configureView(); + + $this->view->setGenerationMode('core'); + } + + public function getTravisYmlOutputPath() + { + return PIWIK_INCLUDE_PATH . '/.travis.yml'; + } +}
\ No newline at end of file diff --git a/plugins/TestRunner/TravisYml/Generator/PiwikTestsPluginsTravisYmlGenerator.php b/plugins/TestRunner/TravisYml/Generator/PiwikTestsPluginsTravisYmlGenerator.php new file mode 100644 index 0000000000..3079af4883 --- /dev/null +++ b/plugins/TestRunner/TravisYml/Generator/PiwikTestsPluginsTravisYmlGenerator.php @@ -0,0 +1,47 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ +namespace Piwik\Plugins\TestRunner\TravisYml\Generator; + +use Exception; +use Piwik\Plugins\TestRunner\TravisYml\Generator; + +class PiwikTestsPluginsTravisYmlGenerator extends Generator +{ + protected function configureView() + { + parent::configureView(); + + $this->view->setGenerationMode('piwik-tests-plugins'); + $this->view->setTravisShScriptLocation("./travis.sh"); + $this->view->setPathToCustomTravisStepsFiles($this->getTestsRepoPath() . "/travis"); + $this->view->setTravisShCwd("\$TRAVIS_BUILD_DIR"); + } + + public function getTravisYmlOutputPath() + { + $dumpPath = @$this->options['dump']; + if (empty($dumpPath)) { + throw new Exception("--dump option must be used when generating a .travis.yml for the piwik-tests-plugins repo." + . " Set it to the path to the repo's .travis.yml."); + } + + return $dumpPath; + } + + private function getTestsRepoPath() + { + return dirname($this->getTravisYmlOutputPath()); + } + + protected function getOptionsForSelfReferentialCommand() + { + $options = parent::getOptionsForSelfReferentialCommand(); + $options['dump'] = '../.travis.yml'; // make sure --dump is used correctly when executed in travis-ci + return $options; + } +}
\ No newline at end of file diff --git a/plugins/TestRunner/TravisYml/Generator/PluginTravisYmlGenerator.php b/plugins/TestRunner/TravisYml/Generator/PluginTravisYmlGenerator.php new file mode 100644 index 0000000000..72dddd101c --- /dev/null +++ b/plugins/TestRunner/TravisYml/Generator/PluginTravisYmlGenerator.php @@ -0,0 +1,128 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ +namespace Piwik\Plugins\TestRunner\TravisYml\Generator; + +use Exception; +use Piwik\Filesystem; +use Piwik\Plugins\TestRunner\TravisYml\Generator; + +class PluginTravisYmlGenerator extends Generator +{ + /** + * @var string + */ + private $targetPlugin; + + public function __construct($targetPlugin, $options) + { + parent::__construct($options); + + $this->targetPlugin = $targetPlugin; + } + + protected function travisEncrypt($data) + { + $cwd = getcwd(); + + // change dir to target plugin since plugin will be in its own git repo + chdir($this->getPluginRootFolder()); + + try { + $result = parent::travisEncrypt($data); + + chdir($cwd); + + return $result; + } catch (Exception $ex) { + chdir($cwd); + + throw $ex; + } + } + + public function getTravisYmlOutputPath() + { + return $this->getPluginRootFolder() . "/.travis.yml"; + } + + public function getPluginRootFolder() + { + return PIWIK_INCLUDE_PATH . "/plugins/{$this->targetPlugin}"; + } + + protected function configureView() + { + parent::configureView(); + + $this->view->setGenerationMode('plugin'); + $this->view->setPlugin($this->targetPlugin); + $this->view->setPathToCustomTravisStepsFiles($this->getPluginRootFolder() . "/tests/travis"); + + $testsToRun = array(); + $testsToExclude = array(); + + if ($this->isTargetPluginContainsPluginTests()) { + $testsToRun[] = array('name' => 'PluginTests', + 'vars' => "MYSQL_ADAPTER=PDO_MYSQL"); + $testsToRun[] = array('name' => 'PluginTests', + 'vars' => "MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_CORE=latest_stable"); + + $testsToExclude[] = array('description' => 'execute latest stable tests only w/ PHP 5.5', + 'php' => '5.3.3', + 'env' => 'TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_CORE=latest_stable'); + $testsToExclude[] = array('php' => '5.4', + 'env' => 'TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_CORE=latest_stable'); + } + + if ($this->isTargetPluginContainsUITests()) { + $testsToRun[] = array('name' => 'UITests', + 'vars' => "MYSQL_ADAPTER=PDO_MYSQL"); + + $testsToExclude[] = array('description' => 'execute UI tests only w/ PHP 5.6', + 'php' => '5.3.3', + 'env' => 'TEST_SUITE=UITests MYSQL_ADAPTER=PDO_MYSQL'); + $testsToExclude[] = array('php' => '5.4', + 'env' => 'TEST_SUITE=UITests MYSQL_ADAPTER=PDO_MYSQL'); + $testsToExclude[] = array('php' => '5.5', + 'env' => 'TEST_SUITE=UITests MYSQL_ADAPTER=PDO_MYSQL'); + } + + if (empty($testsToRun)) { + throw new Exception("No tests to run for this plugin, aborting .travis.yml generation."); + } + + $this->view->setTestsToRun($testsToRun); + $this->view->setTestsToExclude($testsToExclude); + } + + private function isTargetPluginContainsPluginTests() + { + $pluginPath = $this->getPluginRootFolder(); + return $this->doesFolderContainPluginTests($pluginPath . "/tests") + || $this->doesFolderContainPluginTests($pluginPath . "/Test"); + } + + private function doesFolderContainPluginTests($folderPath) + { + $testFiles = Filesystem::globr($folderPath, "*Test.php"); + return !empty($testFiles); + } + + private function isTargetPluginContainsUITests() + { + $pluginPath = $this->getPluginRootFolder(); + return $this->doesFolderContainUITests($pluginPath . "/tests") + || $this->doesFolderContainUITests($pluginPath . "/Test"); + } + + private function doesFolderContainUITests($folderPath) + { + $testFiles = Filesystem::globr($folderPath, "*_spec.js"); + return !empty($testFiles); + } +}
\ No newline at end of file diff --git a/plugins/TestRunner/TravisYml/Parser.php b/plugins/TestRunner/TravisYml/Parser.php new file mode 100644 index 0000000000..7b0fd0e505 --- /dev/null +++ b/plugins/TestRunner/TravisYml/Parser.php @@ -0,0 +1,78 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ +namespace Piwik\Plugins\TestRunner\TravisYml; + +/** + * Utility class that will parse a .travis.yml file and return the contents of the + * file's root YAML sections. + */ +class Parser +{ + /** + * Parse existing data in a .travis.yml file that should be preserved in the output .travis.yml. + * Includes comments. + * + * @var string $existingYmlPath The path to the existing .travis.yml file. + * @return string[] + */ + public function processExistingTravisYml($existingYmlPath) + { + $result = array(); + + $existingYamlText = file_get_contents($existingYmlPath); + foreach ($this->getRootSectionsFromYaml($existingYamlText) as $sectionName => $offset) { + $section = $this->getRootSectionText($existingYamlText, $offset); + $result[$sectionName] = $section; + } + + return $result; + } + + /** + * Extracts the name and offset of all root elements of a YAML document. This method does this by + * checking for text that starts at the beginning of a line and ends with a ':'. + * + * @param string $yamlText The YAML text to search through. + * @return array Array mapping string section names with the starting offset of the text in the YAML. + */ + private function getRootSectionsFromYaml($yamlText) + { + preg_match_all("/^[a-zA-Z_]+:/m", $yamlText, $allMatches, PREG_OFFSET_CAPTURE); + + $result = array(); + + foreach ($allMatches[0] as $match) { + $matchLength = strlen($match[0]); + $sectionName = substr($match[0], 0, $matchLength - 1); + + $result[$sectionName] = $match[1] + $matchLength; + } + + return $result; + } + + /** + * Gets the text of a root YAML element in a YAML doc using the name of the element and the starting + * offset of the element's text. This is accomplished by searching for the first line that doesn't + * start with whitespace after the given offset and using the text between the given offset and the + * line w/o starting whitespace. + * + * @param string $yamlText The YAML text to search through. + * @param int $offset The offset start of the YAML text (does not include the element name and colon, ie + * the offset is after `'element:'`). + * @return string + */ + private function getRootSectionText($yamlText, $offset) + { + preg_match("/^[^\s]/m", $yamlText, $endMatches, PREG_OFFSET_CAPTURE, $offset); + + $endPos = isset($endMatches[0][1]) ? $endMatches[0][1] : strlen($yamlText); + + return substr($yamlText, $offset, $endPos - $offset); + } +}
\ No newline at end of file diff --git a/plugins/TestRunner/TravisYml/TravisYmlView.php b/plugins/TestRunner/TravisYml/TravisYmlView.php new file mode 100644 index 0000000000..71faf4408b --- /dev/null +++ b/plugins/TestRunner/TravisYml/TravisYmlView.php @@ -0,0 +1,229 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ +namespace Piwik\Plugins\TestRunner\TravisYml; + +use Piwik\View; + +/** + * View class for the travis.yml.twig template file. Generates the contents for a .travis.yml file. + */ +class TravisYmlView extends View +{ + /** + * The .travis.yml section names that are overwritten by this command. + * + * @var string[] + */ + private static $travisYmlSectionNames = array( + 'php', + 'services', + 'language', + 'script', + 'before_install', + 'install', + 'before_script', + 'after_script', + 'after_success' + ); + + /** + * The names of .travis.yml sections that can be extended w/ custom steps by plugins. Twig templates + * in the plugins/PluginName/tests/travis directory can be used to insert travis commands at the + * beginning or end of a section. For example, before_install.before.yml will add steps + * at the beginning of the before_install: section. + * + * @var string[] + */ + private static $travisYmlExtendableSectionNames = array( + 'before_install', + 'install', + 'before_script', + 'after_script', + 'after_success' + ); + + /** + * Constructor. + */ + public function __construct() + { + parent::__construct("@TestRunner/travis.yml"); + + $this->setTestsToRun(array()); + $this->setTestsToExclude(array()); + $this->setTravisShScriptLocation("\$PIWIK_ROOT_DIR/tests/travis/travis.sh"); + $this->setTravisShCwd("tests/PHPUnit"); + } + + /** + * Sets the generation mode. Can be 'core' for generating the core .travis.yml file, + * 'plugin' for generating a plugin's .travis.yml file or 'piwik-tests-plugins' + * for generating the .travis.yml file for the piwik-tests-plugins repo. + * + * @param string $mode + */ + public function setGenerationMode($mode) + { + $this->generationMode = $mode; + } + + /** + * Sets the name of plugin the generated .travis.yml file is for. + * + * @param string $pluginName ie, ExamplePlugin, UserSettings, etc. + */ + public function setPlugin($pluginName) + { + $this->pluginName = $pluginName; + } + + /** + * Sets the path where custom travis.yml files should be searched for. The view will load + * files in this directory that look like "XXX.before.yml" or "XXX.after.yml" where XXX + * is the name of a .travis.yml section (eg, install.before.yml). The view will insert + * the contents of these files in the correct positions in the generated output. + * + * @param string $path A path to a directory. + */ + public function setPathToCustomTravisStepsFiles($path) + { + $customTravisBuildSteps = array(); + + foreach (self::$travisYmlExtendableSectionNames as $name) { + $customTravisBuildSteps[$name] = array(); + + $beforeStepsTemplate = $this->getPathToCustomTravisStepsFile($path, $name, 'before'); + if (file_exists($beforeStepsTemplate)) { + $customTravisBuildSteps[$name]['before'] = $this->changeIndent(file_get_contents($beforeStepsTemplate), ' '); + } + + $afterStepsTemplate = $this->getPathToCustomTravisStepsFile($path, $name, 'after'); + if (file_exists($afterStepsTemplate)) { + $customTravisBuildSteps[$name]['after'] = $this->changeIndent(file_get_contents($afterStepsTemplate), ' '); + } + } + + $this->customTravisBuildSteps = $customTravisBuildSteps; + } + + /** + * Set extra global environment variables that should be set in the generated .travis.yml file. The entries + * should be whole statements like `"MY_VAR=myvalue"` or `"secure: mysecurevalue"`. + * + * @param string[] $extraVars + */ + public function setExtraGlobalEnvVars($extraVars) + { + $this->extraGlobalEnvVars = $extraVars; + } + + /** + * Sets the self-referential command that will generate the .travis.yml file on travis. + * + * @param string $consoleCommand ie, `"./console generate:travis-yml ..."` + */ + public function setGenerateYmlCommand($consoleCommand) + { + $this->consoleCommand = addslashes($consoleCommand); + } + + /** + * Sets the PHP versions to run tests against in travis. + * + * @param string[] $phpVersions ie, `array("5.3.3", "5.4", "5.5")`. + */ + public function setPhpVersions($phpVersions) + { + $this->phpVersions = $phpVersions; + } + + /** + * Sets the YAML sections that were found in an existing .travis.yml file and + * should be preserved. See {@link $travisYmlSectionNames} for list of sections + * that will NOT be preserved. + * + * @param $existingSections + */ + public function setExistingSections($existingSections) + { + foreach ($existingSections as $sectionName => $section) { + if ($sectionName == 'env') { + $this->existingEnv = $section; + } else if ($sectionName == 'matrix') { + $this->existingMatrix = $section; + } else if (!in_array($sectionName, self::$travisYmlSectionNames)) { + $this->extraSections .= "\n\n$sectionName:" . $section; + } + } + } + + /** + * Sets the test jobs to run. + * + * @param array $testsToRun Each element must be an array w/ two elements: + * + * **name**: The test suite name (ie, PluginTests, UITests, etc.) + * **vars**: The environment variables (ie, TEST_AGAINST_CORE=latest_stable) + */ + public function setTestsToRun($testsToRun) + { + $this->testsToRun = $testsToRun; + } + + /** + * Sets the tests to exclude. + * + * @param array $testsToExclude Each element must be an array w/ the following elements: + * + * **php**: The PHP version of the job to exclude. + * **env**: The environment variables of the job to exclude. + * **description**: (optional) If supplied, this will be + * output as a comment above the excluding + * YAML. + */ + public function setTestsToExclude($testsToExclude) + { + $this->testsToExclude = $testsToExclude; + } + + /** + * Sets the location of the travis.sh script to use in the .travis.yml file. This + * will be the value of the `script:` section. + * + * @param string $path + */ + public function setTravisShScriptLocation($path) + { + $this->travisShScriptLocation = $path; + } + + /** + * Sets the current working directory that the travis.sh script should be using. This + * will generate a .travis.yml file that will cd into this directory right before + * travis executes the build script. + * + * @param string $path + */ + public function setTravisShCwd($path) + { + $this->travisShCwd = $path; + } + + private function changeIndent($text, $newIndent) + { + $text = trim($text); + + return preg_replace("/^\\s*/", $newIndent, $text); + } + + private function getPathToCustomTravisStepsFile($rootPath, $sectionName, $type) + { + return "$rootPath/$sectionName.$type.yml"; + } +}
\ No newline at end of file |