Created unified system for running tests in travis for Piwik Core and any Piwik plugin. System includes a console command (generate:travis-yml) that generates a .travis.yml file for a plugin or core. The travis file will check in travis if it needs to be updated, and if so it will fail the build and tell the user. If a specific command line option is specified supplying a github user token, the .travis.yml file will be updated within travis automatically.

Notes:

  - comments and non-travis sections will be preserved
  - includes logic for plugins to test themselves against latest stable version instead of just on master
  - the system is primarily controlled by environment variables. developers can change/add variables in .travis.yml and the auto-updating will preserve the changes.

6 files changed, 108 insertions, 8 deletions

diff --git a/tests/PHPUnit/Core/ReleaseCheckListTest.php b/tests/PHPUnit/Core/ReleaseCheckListTest.php
index 0d8acb94e2..8177a63e36 100644
--- a/tests/PHPUnit/Core/ReleaseCheckListTest.php
+++ b/tests/PHPUnit/Core/ReleaseCheckListTest.php
@@ -1,14 +1,17 @@
 <?php
-use Piwik\Filesystem;
-use Piwik\Plugin\Manager;
-use Piwik\SettingsServer;
-
 /**
  * Piwik - free/libre analytics platform
  *
  * @link http://piwik.org
  * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
  */
+use Piwik\Filesystem;
+use Piwik\Plugin\Manager;
+use Piwik\SettingsServer;
+
+/**
+ * @group ReleaseCheckListTest
+ */
 class ReleaseCheckListTest extends PHPUnit_Framework_TestCase
 {
     public function setUp()
@@ -113,6 +116,10 @@ class ReleaseCheckListTest extends PHPUnit_Framework_TestCase
         $patternFailIfFound = 'dump(';
         $files = Filesystem::globr(PIWIK_INCLUDE_PATH . '/plugins', '*.twig');
         foreach ($files as $file) {
+            if ($file == PIWIK_INCLUDE_PATH . '/plugins/CoreConsole/templates/travis.yml.twig') {
+                continue;
+            }
+
             $content = file_get_contents($file);
             $this->assertFalse(strpos($content, $patternFailIfFound), 'found in ' . $file);
         }
diff --git a/tests/travis/autoupdate_travis_yml.sh b/tests/travis/autoupdate_travis_yml.sh
new file mode 100755
index 0000000000..ae5b504eeb
--- /dev/null
+++ b/tests/travis/autoupdate_travis_yml.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# if we're on master, check if .travis.yml is out of date. if github token is supplied we will try to auto-update,
+# otherwise we just print a message and exit.
+GENERATE_TRAVIS_YML_COMMAND="$GENERATE_TRAVIS_YML_COMMAND --dump=./generated.travis.yml"
+$GENERATE_TRAVIS_YML_COMMAND
+
+diff .travis.yml generated.travis.yml > /dev/null
+DIFF_RESULT=$?
+
+if [ "$DIFF_RESULT" -eq "1" ]; then
+    if [ "$GITHUB_USER_TOKEN" != "" ]; then
+        cp generated.travis.yml .travis.yml
+
+        LAST_COMMIT_MESSAGE=$(git log -1 HEAD --pretty=format:%s)
+
+        grep ".travis.yml file is out of date" <<< "$LAST_COMMIT_MESSAGE" > /dev/null
+        LAST_COMMIT_IS_NOT_UPDATE=$?
+
+        if [ "$LAST_COMMIT_MESSAGE" == "" ] || [ "$LAST_COMMIT_IS_NOT_UPDATE" -eq "0" ]; then
+            echo "Last commit message was '$LAST_COMMIT_MESSAGE', possible recursion or error in auto-update, aborting."
+        else
+            # only run auto-update for first travis job and if not a pull request
+            if [ "$TRAVIS_PULL_REQUEST" == "false" ] && [[ "$TRAVIS_JOB_NUMBER" == *.1 ]]; then
+                SCRIPT_DIR=$( dirname "$0" )
+                $SCRIPT_DIR/configure_git.sh # re-configure in case git hasn't been configured yet
+
+                git add .travis.yml
+                git commit -m ".travis.yml file is out of date, auto-updating .travis.yml file."
+
+                git remote set-url origin "https://$GITHUB_USER_TOKEN:@github.com/$TRAVIS_REPO_SLUG.git"
+
+                if ! git push origin $TRAVIS_BRANCH 2> /dev/null; then
+                    echo "Failed to push!"
+                    exit 1
+                fi
+            else
+                echo "Building for pull request or not first job, skipping .travis.yml out of date check."
+                echo ""
+                echo "TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST"
+                echo "TRAVIS_JOB_NUMBER=$TRAVIS_JOB_NUMBER"
+            fi
+        fi
+
+        echo ""
+        echo "Generated .travis.yml:"
+        echo ""
+        cat generated.travis.yml
+    else
+        echo "${RED}Your .travis.yml file is out of date! Please update it using the generate:travis-yml command.${RESET}"
+    fi
+
+    exit 1
+else
+    echo ".travis.yml file is up-to-date."
+fi
\ No newline at end of file
diff --git a/tests/travis/configure_git.sh b/tests/travis/configure_git.sh
new file mode 100755
index 0000000000..d9faa80d2d
--- /dev/null
+++ b/tests/travis/configure_git.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+if [ "$TRAVIS_COMMITTER_EMAIL" == "" ]; then
+    TRAVIS_COMMITTER_EMAIL="hello@piwik.org"
+fi
+
+if [ "$TRAVIS_COMMITTER_NAME" == "" ]; then
+    TRAVIS_COMMITTER_NAME="Piwik Automation"
+fi
+
+echo "Configuring git [email = $TRAVIS_COMMITTER_EMAIL, user = $TRAVIS_COMMITTER_NAME]..."
+
+git config --global user.email "$TRAVIS_COMMITTER_EMAIL"
+git config --global user.name "$TRAVIS_COMMITTER_NAME"
\ No newline at end of file
diff --git a/tests/travis/initiate_ui_tests.sh b/tests/travis/initiate_ui_tests.sh
index a65a5f6ac5..e44395938b 100755
--- a/tests/travis/initiate_ui_tests.sh
+++ b/tests/travis/initiate_ui_tests.sh
@@ -19,9 +19,6 @@ COMMIT_MESSAGE=$(git log "$TRAVIS_COMMIT" -1 --pretty=%B)
 
 cd tests/PHPUnit/UI
 
-git config --global user.email "hello@piwik.org"
-git config --global user.name "Piwik Automation"
-
 UI_BRANCH="master"
 git checkout $UI_BRANCH
 git pull --rebase origin $UI_BRANCH
diff --git a/tests/PHPUnit/travis-helper.sh b/tests/travis/travis-helper.sh
index 23e3943481..58ebc4cd55 100644
--- a/tests/PHPUnit/travis-helper.sh
+++ b/tests/travis/travis-helper.sh
@@ -54,4 +54,25 @@ count=$(($count + 1))
 
 echo -e "\n${RED}Timeout (${timeout} minutes) reached. Terminating \"$@\"${RESET}\n"
   kill -9 $cmd_pid
+}
+
+travis_retry() {
+  local result=0
+  local count=1
+  while [ $count -le 3 ]; do
+    [ $result -ne 0 ] && {
+      echo -e "\n${RED}The command \"$@\" failed. Retrying, $count of 3.${RESET}\n" >&2
+    }
+    "$@"
+    result=$?
+    [ $result -eq 0 ] && break
+count=$(($count + 1))
+    sleep 1
+  done
+
+  [ $count -gt 3 ] && {
+    echo "\n${RED}The command \"$@\" failed 3 times.${RESET}\n" >&2
+  }
+
+  return $result
 }
\ No newline at end of file
diff --git a/tests/PHPUnit/travis.sh b/tests/travis/travis.sh
index 0c6631b550..a9aa252b18 100755
--- a/tests/PHPUnit/travis.sh
+++ b/tests/travis/travis.sh
@@ -1,7 +1,12 @@
 #!/bin/bash
 
+SCRIPT_DIR=$( dirname "$0" )
+
 # for travis_wait function
-source travis-helper.sh
+source $SCRIPT_DIR/travis-helper.sh
+
+# go to tests directory
+cd ../PHPUnit
 
 if [ "$TEST_SUITE" != "UITests" ] && [ "$TEST_SUITE" != "AngularJSTests" ]
 then