refs #6429 added a command to trigger tests on aws

18 files changed, 1115 insertions, 3 deletions

diff --git a/composer.json b/composer.json
index 1e79ad7255..95f688e3f7 100644
--- a/composer.json
+++ b/composer.json
@@ -46,8 +46,10 @@
         "piwik/decompress": "~0.1.0"
     },
     "require-dev": {
+        "aws/aws-sdk-php": "2.7.1",
         "phpunit/phpunit": "~4.1",
-        "facebook/xhprof": "dev-master"
+        "facebook/xhprof": "dev-master",
+        "phpseclib/phpseclib": "~0.3.8"
     },
     "repositories": [
         {
diff --git a/composer.lock b/composer.lock
index 079fbe0a57..3c46167dc7 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "hash": "2b25fd70ade7d65dd13a492195c96dc1",
+    "hash": "22cbe8affc3293d56e008f7b08a140b6",
     "packages": [
         {
             "name": "leafo/lessphp",
@@ -109,7 +109,7 @@
                 "shasum": ""
             },
             "require": {
-                "php": ">=5.3.2"
+                "php": ">=5.3.3"
             },
             "require-dev": {
                 "phpunit/phpunit": "~4.0"
@@ -332,6 +332,73 @@
     ],
     "packages-dev": [
         {
+            "name": "aws/aws-sdk-php",
+            "version": "2.7.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/aws/aws-sdk-php.git",
+                "reference": "937a39ca3cee98d31a7410a17db24e0496c41494"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/937a39ca3cee98d31a7410a17db24e0496c41494",
+                "reference": "937a39ca3cee98d31a7410a17db24e0496c41494",
+                "shasum": ""
+            },
+            "require": {
+                "guzzle/guzzle": "~3.7",
+                "php": ">=5.3.3"
+            },
+            "require-dev": {
+                "doctrine/cache": "~1.0",
+                "ext-openssl": "*",
+                "monolog/monolog": "~1.4",
+                "phpunit/phpunit": "~4.0",
+                "symfony/yaml": "~2.1"
+            },
+            "suggest": {
+                "doctrine/cache": "Adds support for caching of credentials and responses",
+                "ext-apc": "Allows service description opcode caching, request and response caching, and credentials caching",
+                "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages",
+                "monolog/monolog": "Adds support for logging HTTP requests and responses",
+                "symfony/yaml": "Eases the ability to write manifests for creating jobs in AWS Import/Export"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "2.7-dev"
+                }
+            },
+            "autoload": {
+                "psr-0": {
+                    "Aws": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "Apache-2.0"
+            ],
+            "authors": [
+                {
+                    "name": "Amazon Web Services",
+                    "homepage": "http://aws.amazon.com"
+                }
+            ],
+            "description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project",
+            "homepage": "http://aws.amazon.com/sdkforphp",
+            "keywords": [
+                "amazon",
+                "aws",
+                "cloud",
+                "dynamodb",
+                "ec2",
+                "glacier",
+                "s3",
+                "sdk"
+            ],
+            "time": "2014-10-16 21:37:55"
+        },
+        {
             "name": "doctrine/instantiator",
             "version": "1.0.4",
             "source": {
@@ -415,6 +482,196 @@
             "time": "2014-08-28 17:34:52"
         },
         {
+            "name": "guzzle/guzzle",
+            "version": "v3.9.2",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/guzzle/guzzle3.git",
+                "reference": "54991459675c1a2924122afbb0e5609ade581155"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/guzzle/guzzle3/zipball/54991459675c1a2924122afbb0e5609ade581155",
+                "reference": "54991459675c1a2924122afbb0e5609ade581155",
+                "shasum": ""
+            },
+            "require": {
+                "ext-curl": "*",
+                "php": ">=5.3.3",
+                "symfony/event-dispatcher": "~2.1"
+            },
+            "replace": {
+                "guzzle/batch": "self.version",
+                "guzzle/cache": "self.version",
+                "guzzle/common": "self.version",
+                "guzzle/http": "self.version",
+                "guzzle/inflection": "self.version",
+                "guzzle/iterator": "self.version",
+                "guzzle/log": "self.version",
+                "guzzle/parser": "self.version",
+                "guzzle/plugin": "self.version",
+                "guzzle/plugin-async": "self.version",
+                "guzzle/plugin-backoff": "self.version",
+                "guzzle/plugin-cache": "self.version",
+                "guzzle/plugin-cookie": "self.version",
+                "guzzle/plugin-curlauth": "self.version",
+                "guzzle/plugin-error-response": "self.version",
+                "guzzle/plugin-history": "self.version",
+                "guzzle/plugin-log": "self.version",
+                "guzzle/plugin-md5": "self.version",
+                "guzzle/plugin-mock": "self.version",
+                "guzzle/plugin-oauth": "self.version",
+                "guzzle/service": "self.version",
+                "guzzle/stream": "self.version"
+            },
+            "require-dev": {
+                "doctrine/cache": "~1.3",
+                "monolog/monolog": "~1.0",
+                "phpunit/phpunit": "3.7.*",
+                "psr/log": "~1.0",
+                "symfony/class-loader": "~2.1",
+                "zendframework/zend-cache": "2.*,<2.3",
+                "zendframework/zend-log": "2.*,<2.3"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "3.9-dev"
+                }
+            },
+            "autoload": {
+                "psr-0": {
+                    "Guzzle": "src/",
+                    "Guzzle\\Tests": "tests/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Michael Dowling",
+                    "email": "mtdowling@gmail.com",
+                    "homepage": "https://github.com/mtdowling"
+                },
+                {
+                    "name": "Guzzle Community",
+                    "homepage": "https://github.com/guzzle/guzzle/contributors"
+                }
+            ],
+            "description": "Guzzle is a PHP HTTP client library and framework for building RESTful web service clients",
+            "homepage": "http://guzzlephp.org/",
+            "keywords": [
+                "client",
+                "curl",
+                "framework",
+                "http",
+                "http client",
+                "rest",
+                "web service"
+            ],
+            "time": "2014-08-11 04:32:36"
+        },
+        {
+            "name": "phpseclib/phpseclib",
+            "version": "0.3.8",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/phpseclib/phpseclib.git",
+                "reference": "5085202f1f37769aae59f9711c423f28159c9b29"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/5085202f1f37769aae59f9711c423f28159c9b29",
+                "reference": "5085202f1f37769aae59f9711c423f28159c9b29",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.0.0"
+            },
+            "require-dev": {
+                "phing/phing": "2.7.*",
+                "phpunit/phpunit": "4.0.*",
+                "sami/sami": "1.*",
+                "squizlabs/php_codesniffer": "1.*"
+            },
+            "suggest": {
+                "ext-gmp": "Install the GMP (GNU Multiple Precision) extension in order to speed up arbitrary precision integer arithmetic operations.",
+                "ext-mcrypt": "Install the Mcrypt extension in order to speed up a wide variety of cryptographic operations.",
+                "pear-pear/PHP_Compat": "Install PHP_Compat to get phpseclib working on PHP < 4.3.3."
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "0.3-dev"
+                }
+            },
+            "autoload": {
+                "psr-0": {
+                    "Crypt": "phpseclib/",
+                    "File": "phpseclib/",
+                    "Math": "phpseclib/",
+                    "Net": "phpseclib/",
+                    "System": "phpseclib/"
+                },
+                "files": [
+                    "phpseclib/Crypt/Random.php"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "include-path": [
+                "phpseclib/"
+            ],
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Jim Wigginton",
+                    "email": "terrafrost@php.net",
+                    "role": "Lead Developer"
+                },
+                {
+                    "name": "Patrick Monnerat",
+                    "email": "pm@datasphere.ch",
+                    "role": "Developer"
+                },
+                {
+                    "name": "Andreas Fischer",
+                    "email": "bantu@phpbb.com",
+                    "role": "Developer"
+                },
+                {
+                    "name": "Hans-Jürgen Petrich",
+                    "email": "petrich@tronic-media.com",
+                    "role": "Developer"
+                }
+            ],
+            "description": "PHP Secure Communications Library - Pure-PHP implementations of RSA, AES, SSH2, SFTP, X.509 etc.",
+            "homepage": "http://phpseclib.sourceforge.net",
+            "keywords": [
+                "BigInteger",
+                "aes",
+                "asn.1",
+                "asn1",
+                "blowfish",
+                "crypto",
+                "cryptography",
+                "encryption",
+                "rsa",
+                "security",
+                "sftp",
+                "signature",
+                "signing",
+                "ssh",
+                "twofish",
+                "x.509",
+                "x509"
+            ],
+            "time": "2014-09-13 02:42:45"
+        },
+        {
             "name": "phpunit/php-code-coverage",
             "version": "2.0.11",
             "source": {
@@ -1057,6 +1314,63 @@
             "time": "2014-03-07 15:35:33"
         },
         {
+            "name": "symfony/event-dispatcher",
+            "version": "v2.5.5",
+            "target-dir": "Symfony/Component/EventDispatcher",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/EventDispatcher.git",
+                "reference": "f6281337bf5f985f585d1db6a83adb05ce531f46"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/f6281337bf5f985f585d1db6a83adb05ce531f46",
+                "reference": "f6281337bf5f985f585d1db6a83adb05ce531f46",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.3.3"
+            },
+            "require-dev": {
+                "psr/log": "~1.0",
+                "symfony/config": "~2.0",
+                "symfony/dependency-injection": "~2.0,<2.6.0",
+                "symfony/stopwatch": "~2.2"
+            },
+            "suggest": {
+                "symfony/dependency-injection": "",
+                "symfony/http-kernel": ""
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "2.5-dev"
+                }
+            },
+            "autoload": {
+                "psr-0": {
+                    "Symfony\\Component\\EventDispatcher\\": ""
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Symfony Community",
+                    "homepage": "http://symfony.com/contributors"
+                },
+                {
+                    "name": "Fabien Potencier",
+                    "email": "fabien@symfony.com"
+                }
+            ],
+            "description": "Symfony EventDispatcher Component",
+            "homepage": "http://symfony.com",
+            "time": "2014-09-28 15:56:11"
+        },
+        {
             "name": "symfony/yaml",
             "version": "v2.5.5",
             "target-dir": "Symfony/Component/Yaml",
diff --git a/config/global.ini.php b/config/global.ini.php
index 3210c1efda..d5918cef82 100644
--- a/config/global.ini.php
+++ b/config/global.ini.php
@@ -38,6 +38,19 @@ adapter = PDO\MYSQL
 type = InnoDB
 schema = Mysql
 
+[tests]
+; access key and secret as listed in AWS -> IAM -> Users
+aws_accesskey = ""
+aws_secret = ""
+; key pair name as listed in AWS -> EC2 -> Key Pairs. Key name should be different per user.
+aws_keyname = ""
+; PEM file can be downloaded after creating a new key pair in AWS -> EC2 -> Key Pairs
+aws_pem_file = "<path to pem file>"
+aws_securitygroups[] = "default"
+aws_region = "us-east-1"
+aws_ami = "ami-b69c1ade"
+aws_instance_type = "c3.large"
+
 [log]
 ; possible values for log: screen, database, file
 log_writers[] = screen
@@ -683,6 +696,7 @@ Plugins[] = ZenMode
 Plugins[] = LeftMenu
 Plugins[] = Morpheus
 Plugins[] = Contents
+Plugins[] = TestRunner
 
 [PluginsInstalled]
 PluginsInstalled[] = Login
diff --git a/plugins/TestRunner/Aws/CloudWatch.php b/plugins/TestRunner/Aws/CloudWatch.php
new file mode 100644
index 0000000000..0748ac129f
--- /dev/null
+++ b/plugins/TestRunner/Aws/CloudWatch.php
@@ -0,0 +1,111 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\TestRunner\Aws;
+
+use Aws\CloudWatch\CloudWatchClient;
+use Aws\CloudWatch\Enum\ComparisonOperator;
+use Aws\CloudWatch\Enum\Statistic;
+use Aws\CloudWatch\Enum\Unit;
+
+class CloudWatch
+{
+    /**
+     * @var Config
+     */
+    private $config;
+
+    public function __construct(Config $awsConfig)
+    {
+        $this->config = $awsConfig;
+    }
+
+    public function terminateInstanceIfIdleForTooLong($instanceIds)
+    {
+        $client = $this->getCloudWatchClient();
+
+        $client->putMetricAlarm(array(
+            'AlarmName' => 'TerminateInstanceBecauseIdle',
+            'AlarmDescription' => 'Terminate instances if CPU is on average < 10% for 5 minutes in a row 8 times consecutively',
+            'ActionsEnabled' => true,
+            'OKActions' => array(),
+            'AlarmActions' => $this->getAlarmActions(),
+            'InsufficientDataActions' => array(),
+            'MetricName' => 'CPUUtilization',
+            'Namespace' => $this->getNamespace(),
+            'Statistic' => Statistic::AVERAGE,
+            'Dimensions' => $this->getDimensions($instanceIds),
+            'Period' => 300,
+            'Unit' => Unit::PERCENT,
+            'EvaluationPeriods' => 8,
+            'Threshold' => 10,
+            'ComparisonOperator' => ComparisonOperator::LESS_THAN_THRESHOLD,
+        ));
+
+        $client->putMetricAlarm(array(
+            'AlarmName' => 'TerminateInstanceIfStatusCheckFails',
+            'AlarmDescription' => 'Terminate instances in case two status check fail within one minute',
+            'ActionsEnabled' => true,
+            'OKActions' => array(),
+            'AlarmActions' => $this->getAlarmActions(),
+            'InsufficientDataActions' => array(),
+            'MetricName' => 'StatusCheckFailed',
+            'Namespace' => $this->getNamespace(),
+            'Statistic' => Statistic::AVERAGE,
+            'Dimensions' => $this->getDimensions($instanceIds),
+            'Period' => 60,
+            'Unit' => Unit::PERCENT,
+            'EvaluationPeriods' => 2,
+            'Threshold' => 1,
+            'ComparisonOperator' => ComparisonOperator::GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+        ));
+    }
+
+    private function getCloudWatchClient()
+    {
+        return CloudWatchClient::factory($this->getConnectionOptions());
+    }
+
+    private function getConnectionOptions()
+    {
+        return array(
+            'key'    => $this->config->getAccessKey(),
+            'secret' => $this->config->getSecretKey(),
+            'region' => $this->config->getRegion()
+        );
+    }
+
+    private function getDimensions($instanceIds)
+    {
+        $dimensions = array();
+
+        foreach ($instanceIds as $instanceId) {
+            $dimensions[] = array(
+                'Name'  => 'InstanceId',
+                'Value' => $instanceId,
+            );
+        }
+
+        return $dimensions;
+    }
+
+    private function getNamespace()
+    {
+        return 'AWS/EC2';
+    }
+
+    private function getAlarmActions()
+    {
+        return array(
+            'arn:aws:automate:' . $this->config->getRegion() . ':ec2:terminate',
+            'arn:aws:sns:' . $this->config->getRegion() . ':682510200394:TerminateInstanceBecauseIdle'
+        );
+    }
+
+}
diff --git a/plugins/TestRunner/Aws/Config.php b/plugins/TestRunner/Aws/Config.php
new file mode 100644
index 0000000000..3649cf10f4
--- /dev/null
+++ b/plugins/TestRunner/Aws/Config.php
@@ -0,0 +1,99 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\TestRunner\Aws;
+
+use \Piwik\Config as PiwikConfig;
+
+class Config
+{
+    public function getRegion()
+    {
+        return trim($this->getConfigValue('aws_region'));
+    }
+
+    public function getAmi()
+    {
+        return trim($this->getConfigValue('aws_ami'));
+    }
+
+    public function getInstanceType()
+    {
+        return trim($this->getConfigValue('aws_instance_type'));
+    }
+
+    public function getKeyName()
+    {
+        return $this->getConfigValue('aws_keyname');
+    }
+
+    public function getPemFile()
+    {
+        return trim($this->getConfigValue('aws_pem_file'));
+    }
+
+    public function getAccessKey()
+    {
+        return trim($this->getConfigValue('aws_accesskey'));
+    }
+
+    public function getSecretKey()
+    {
+        return trim($this->getConfigValue('aws_secret'));
+    }
+
+    public function getSecurityGroups()
+    {
+        $groups = $this->getConfigValue('aws_securitygroups');
+
+        if (empty($groups)) {
+            $groups = array();
+        }
+
+        return (array) $groups;
+    }
+
+    public function validate()
+    {
+        $configKeysToValidate = array(
+            'aws_accesskey',
+            'aws_secret',
+            'aws_region',
+            'aws_ami',
+            'aws_instance_type',
+            'aws_pem_file',
+            'aws_keyname',
+            'aws_securitygroups',
+        );
+
+        foreach ($configKeysToValidate as $key) {
+            if (!$this->getConfigValue($key)) {
+                throw new \RuntimeException("[tests]$key is not configured");
+            }
+        }
+
+        $pemFile = $this->getPemFile();
+
+        if (!file_exists($pemFile)) {
+            throw new \RuntimeException('[tests]aws_pem_file the file does not exist or is not readable');
+        }
+    }
+
+    private function getConfig()
+    {
+        return PiwikConfig::getInstance()->tests;
+    }
+
+    private function getConfigValue($key)
+    {
+        $config = $this->getConfig();
+
+        return $config[$key];
+    }
+}
diff --git a/plugins/TestRunner/Aws/Instance.php b/plugins/TestRunner/Aws/Instance.php
new file mode 100644
index 0000000000..e7bcb7a6ad
--- /dev/null
+++ b/plugins/TestRunner/Aws/Instance.php
@@ -0,0 +1,162 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\TestRunner\Aws;
+
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Aws\Ec2\Ec2Client;
+
+class Instance
+{
+
+    /**
+     * @var Config
+     */
+    private $config;
+
+    /**
+     * @var Ec2Client
+     */
+    private $client;
+
+    private $testSuite;
+
+    private $useOneInstancePerTestSuite = false;
+
+    public function __construct(Config $config, $testSuite)
+    {
+        $this->config    = $config;
+        $this->testSuite = $testSuite;
+        $this->client    = $this->createEc2Client();
+    }
+
+    public function enableUseOneInstancePerTestSuite()
+    {
+        $this->useOneInstancePerTestSuite = true;
+    }
+
+    public function findExisting()
+    {
+        $filters = array(
+            array('Name' => 'image-id', 'Values' => array($this->config->getAmi())),
+            array('Name' => 'key-name', 'Values' => array($this->config->getKeyName())),
+            array('Name' => 'instance-state-name', 'Values' => array('running')),
+        );
+
+        if (!empty($this->testSuite) && $this->useOneInstancePerTestSuite) {
+            $filters[] = array('Name' => 'tag:TestSuite', 'Values' => array($this->testSuite));
+        }
+
+        $instances = $this->client->describeInstances(array('Filters' => $filters));
+
+        $reservations = $instances->getPath('Reservations');
+
+        if (!empty($reservations)) {
+            $host = $this->getHostFromDescribedInstances($instances);
+
+            return $host;
+        }
+    }
+
+    public function terminate($instanceIds)
+    {
+        $this->client->terminateInstances(array(
+            'InstanceIds' => $instanceIds
+        ));
+
+        $this->client->waitUntilInstanceTerminated(array(
+            'InstanceIds' => $instanceIds
+        ));
+    }
+
+    public function launch()
+    {
+        $result = $this->client->runInstances(array(
+            'ImageId' => $this->config->getAmi(),
+            'MinCount' => 1,
+            'MaxCount' => 1,
+            'InstanceType' => $this->config->getInstanceType(),
+            'KeyName' => $this->config->getKeyName(),
+            'SecurityGroups' => $this->config->getSecurityGroups(),
+            'InstanceInitiatedShutdownBehavior' => 'terminate'
+        ));
+
+        $instanceIds = $result->getPath('Instances/*/InstanceId');
+
+        return $instanceIds;
+    }
+
+    public function setup($instanceIds)
+    {
+        $this->client->waitUntilInstanceRunning(array(
+            'InstanceIds' => $instanceIds,
+        ));
+
+        $awsCloudWatch = new CloudWatch($this->config);
+        $awsCloudWatch->terminateInstanceIfIdleForTooLong($instanceIds);
+
+        $awsTags = new Tags($this->client);
+        $awsTags->assignTagsToInstances($instanceIds, $this->testSuite);
+
+        $instances = $this->client->describeInstances(array(
+            'InstanceIds' => $instanceIds,
+        ));
+
+        $host = $this->getHostFromDescribedInstances($instances);
+
+        return $host;
+    }
+
+    /**
+     * @param \Guzzle\Service\Resource\Model $resources
+     * @return mixed
+     */
+    private function getHostFromDescribedInstances($resources)
+    {
+        $instances = $resources->getPath('Reservations/*/Instances');
+
+        $instanceToUse = null;
+
+        foreach ($instances as $index => $instance) {
+            foreach ($instance['Tags'] as $tag) {
+                if (!empty($this->testSuite)
+                    && $tag['Key'] === 'TestSuite'
+                    && $tag['Value'] === $this->testSuite) {
+
+                    $instanceToUse = $instance;
+                }
+            }
+        }
+
+        if (empty($instanceToUse)) {
+            $instanceToUse = array_shift($instances);
+        }
+
+        $host = $instanceToUse['PublicDnsName'];
+
+        return $host;
+    }
+
+    private function createEc2Client()
+    {
+        return Ec2Client::factory($this->getConnectionOptions());
+    }
+
+    private function getConnectionOptions()
+    {
+        return array(
+            'key'    => $this->config->getAccessKey(),
+            'secret' => $this->config->getSecretKey(),
+            'region' => $this->config->getRegion()
+        );
+    }
+}
\ No newline at end of file
diff --git a/plugins/TestRunner/Aws/Ssh.php b/plugins/TestRunner/Aws/Ssh.php
new file mode 100644
index 0000000000..6e41eb0d37
--- /dev/null
+++ b/plugins/TestRunner/Aws/Ssh.php
@@ -0,0 +1,54 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\TestRunner\Aws;
+use Symfony\Component\Console\Output\OutputInterface;
+use Crypt_RSA;
+use Net_SSH2;
+
+class Ssh extends Net_SSH2
+{
+    /**
+     * @var OutputInterface
+     */
+    private $output;
+
+    public static function connectToAws($host, $pemFile)
+    {
+        $key = new Crypt_RSA();
+        $key->loadKey(file_get_contents($pemFile));
+
+        $ssh = new Ssh($host);
+
+        if (!$ssh->login('ubuntu', $key)) {
+            throw new \RuntimeException("Login to $host using $pemFile failed");
+        }
+
+        return $ssh;
+    }
+
+    public function setOutput(OutputInterface $output)
+    {
+        $this->output = $output;
+    }
+
+    public function exec($command)
+    {
+        $command = 'cd www/piwik && ' . $command;
+        $output  = $this->output;
+
+        $output->writeln("Executing <comment>$command</comment>");
+
+        return parent::exec($command, function($tempOutput) use ($output) {
+            if ($output) {
+                $output->write($tempOutput);
+            }
+        });
+    }
+}
diff --git a/plugins/TestRunner/Aws/Tags.php b/plugins/TestRunner/Aws/Tags.php
new file mode 100644
index 0000000000..aa6c156318
--- /dev/null
+++ b/plugins/TestRunner/Aws/Tags.php
@@ -0,0 +1,43 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\TestRunner\Aws;
+use Aws\Ec2\Ec2Client;
+
+class Tags
+{
+    /**
+     * @var Ec2Client
+     */
+    private $ec2Client;
+
+    public function __construct(Ec2Client $client)
+    {
+        $this->ec2Client = $client;
+    }
+
+    public function assignTagsToInstances($instanceIds, $testSuite)
+    {
+        $tags = array($this->buildTag('Name', 'PiwikTesting'));
+
+        if (!empty($testSuite)) {
+            $tags[] = $this->buildTag('TestSuite', $testSuite);
+        }
+
+        $this->ec2Client->createTags(array('Resources' => $instanceIds, 'Tags' => $tags));
+    }
+
+    private function buildTag($name, $value)
+    {
+        return array(
+            'Key'   => $name,
+            'Value' => $value,
+        );
+    }
+}
\ No newline at end of file
diff --git a/plugins/TestRunner/Commands/TestRunOnAws.php b/plugins/TestRunner/Commands/TestRunOnAws.php
new file mode 100644
index 0000000000..34a00cdae8
--- /dev/null
+++ b/plugins/TestRunner/Commands/TestRunOnAws.php
@@ -0,0 +1,141 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\TestRunner\Commands;
+
+use Piwik\Development;
+use Piwik\Plugin\ConsoleCommand;
+use Piwik\Plugins\TestRunner\Aws\Config;
+use Piwik\Plugins\TestRunner\Aws\Instance;
+use Piwik\Plugins\TestRunner\Aws\Ssh;
+use Piwik\Plugins\TestRunner\Runner\InstanceLauncher;
+use Piwik\Plugins\TestRunner\Runner\Remote;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class TestRunOnAws extends ConsoleCommand
+{
+    private $allowedTestSuites = array('integration', 'system', 'all', 'ui');
+
+    public function isEnabled()
+    {
+        return Development::isEnabled();
+    }
+
+    protected function configure()
+    {
+        $this->setName('tests:run-aws');
+        $this->addArgument('testsuite', InputArgument::OPTIONAL, 'Allowed values: ' . implode(', ', $this->allowedTestSuites));
+        $this->addOption('launch-only', null, InputOption::VALUE_NONE, 'Only launches an instance and outputs the connection parameters. Useful if you want to connect via SSH.');
+        $this->addOption('update-only', null, InputOption::VALUE_NONE, 'Launches an instance, outputs the connection parameters and prepares the instance for a test run but does not actually run the tests. It will also checkout the specified version.');
+        $this->addOption('one-instance-per-testsuite', null, InputOption::VALUE_NONE, 'Launches an instance, outputs the connection parameters and prepares the instance for a test run but does not actually run the tests. It will also checkout the specified version.');
+        $this->addOption('checkout', null, InputOption::VALUE_REQUIRED, 'Git hash, tag or branch to checkout. Defaults to current hash', $this->getCurrentGitHash());
+        $this->setDescription('Run a specific testsuite on AWS');
+        $this->setHelp('To use this command you have to configure the [tests]aws_* section in config/config.ini.php. See config/global.ini.php for all available options.
+
+To run a test simply specify the testsuite you want to run: <comment>./console tests:run-aws system</comment>. This will launch a new instance on AWS or reuse an already running one. We start one instance per keyname. This makes sure two different developers do not use the same instance at the same time.
+
+By default it will execute the tests of the git hash you are currently on. If this hash is not pushed yet or if you want to run tests of a specific git hash / branch / tag use the <comment>--checkout</comment> option: <comment>./console tests:run-aws --checkout="master" system</comment>.
+
+If you want to debug a problem and access the AWS instance using SSH you can specify the <comment>--launch-only</comment> or <comment>--update-only</comment> option.
+
+By default we will launch only one instance per keyname meaning you should not execute this command while another test is running. It would start the tests twice on the same instance and lead to errors. If you want to run two different testsuites at the same time (for instance <comment>system</comment> and <comment>ui</comment>) specify the <comment>one-instance-per-testsuite</comment> option. This will launch one instance for system tests and one for ui tests:
+<comment>./console tests:run-aws system</comment>
+<comment>./console tests:run-aws --one-instance-per-testsuite ui // will launch a new instance for ui testsuites</comment>
+');
+    }
+
+    /**
+     * Execute command like: ./console core:clear-caches
+     */
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $testSuite  = $this->getTestSuite($input);
+        $launchOnly = $input->getOption('launch-only');
+        $updateOnly = $input->getOption('update-only');
+        $gitHash    = $input->getOption('checkout');
+        $perTestsuite = $input->getOption('one-instance-per-testsuite');
+
+        if (empty($testSuite) && empty($launchOnly) && empty($updateOnly)) {
+            throw new \InvalidArgumentException('Either provide a testsuite argument or define <comment>--launch-only</comment> or <comment>--update-only</comment>');
+        }
+
+        $awsConfig = new Config();
+        $awsConfig->validate();
+
+        $host = $this->launchInstance($output, $perTestsuite, $awsConfig, $testSuite);
+
+        if ($launchOnly) {
+            return 0;
+        }
+
+        $ssh = Ssh::connectToAws($host, $awsConfig->getPemFile());
+        $ssh->setOutput($output);
+
+        $testRunner = new Remote($ssh);
+        $testRunner->updatePiwik($gitHash);
+
+        if ($updateOnly) {
+            $ssh->disconnect();
+
+            return 0;
+        }
+
+        $testRunner->runTests($host, $testSuite);
+
+        if (in_array($testSuite, array('system', 'all'))) {
+            $output->writeln("<info>Tests finished. You can browse processed files at </info><comment>http://$host/tests/PHPUnit/System/processed/</comment>");
+        } elseif ('ui' === $testSuite) {
+            $output->writeln("<info>Tests finished. You can browse processed screenshots at </info><comment>http://$host/tests/PHPUnit/UI/processed-ui-screenshots/</comment>");
+        } else {
+            $output->writeln("<info>Tests finished</info>");
+        }
+
+        $ssh->disconnect();
+    }
+
+    private function launchInstance(OutputInterface $output, $useOneInstancePerTestSuite, Config $awsConfig, $testSuite)
+    {
+        $awsInstance = new Instance($awsConfig, $testSuite);
+
+        if ($useOneInstancePerTestSuite) {
+            $awsInstance->enableUseOneInstancePerTestSuite();
+        }
+
+        $launcher = new InstanceLauncher($awsInstance);
+        $host     = $launcher->launchOrResumeInstance();
+
+        $output->writeln(sprintf("Access instance using <comment>ssh -i %s ubuntu@%s</comment>", $awsConfig->getPemFile(), $host));
+        $output->writeln("You can log in to Piwik via root:secure at <comment>http://$host</comment>");
+        $output->writeln("You can access database via root:secure (<comment>mysql -uroot -psecure</comment>)");
+        $output->writeln("Files are located in <comment>~/www/piwik</comment>");
+        $output->writeln(' ');
+
+        return $host;
+    }
+
+    private function getTestSuite(InputInterface $input)
+    {
+        $testsuite = $input->getArgument('testsuite');
+
+        if (!empty($testsuite) && !in_array($testsuite, $this->allowedTestSuites)) {
+            throw new \InvalidArgumentException('Test suite argument is wrong, use one of following: ' . implode(', ', $this->allowedTestSuites));
+        }
+
+        return $testsuite;
+    }
+
+    private function getCurrentGitHash()
+    {
+        return trim(`git rev-parse HEAD`);
+    }
+
+}
diff --git a/plugins/CoreConsole/Commands/TestsRun.php b/plugins/TestRunner/Commands/TestsRun.php
index e8b5e3ffec..e8b5e3ffec 100644
--- a/plugins/CoreConsole/Commands/TestsRun.php
+++ b/plugins/TestRunner/Commands/TestsRun.php
diff --git a/plugins/CoreConsole/Commands/TestsRunUI.php b/plugins/TestRunner/Commands/TestsRunUI.php
index c3859579ed..c3859579ed 100644
--- a/plugins/CoreConsole/Commands/TestsRunUI.php
+++ b/plugins/TestRunner/Commands/TestsRunUI.php
diff --git a/plugins/CoreConsole/Commands/TestsSetupFixture.php b/plugins/TestRunner/Commands/TestsSetupFixture.php
index 8002eaaa7e..8002eaaa7e 100644
--- a/plugins/CoreConsole/Commands/TestsSetupFixture.php
+++ b/plugins/TestRunner/Commands/TestsSetupFixture.php
diff --git a/plugins/TestRunner/README.md b/plugins/TestRunner/README.md
new file mode 100644
index 0000000000..4005bb93b7
--- /dev/null
+++ b/plugins/TestRunner/README.md
@@ -0,0 +1,18 @@
+# Piwik TestRunner Plugin
+
+## Description
+
+Add your plugin description here.
+
+## FAQ
+
+__My question?__
+My answer
+
+## Changelog
+
+Here goes the changelog text.
+
+## Support
+
+Please direct any feedback to ...
\ No newline at end of file
diff --git a/plugins/TestRunner/Runner/InstanceLauncher.php b/plugins/TestRunner/Runner/InstanceLauncher.php
new file mode 100644
index 0000000000..4f9faf97c7
--- /dev/null
+++ b/plugins/TestRunner/Runner/InstanceLauncher.php
@@ -0,0 +1,52 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\TestRunner\Runner;
+
+use Piwik\Plugins\TestRunner\Aws\Instance;
+
+class InstanceLauncher {
+
+    /**
+     * @var Instance
+     */
+    private $instance;
+
+    public function __construct(Instance $instance)
+    {
+        $this->instance = $instance;
+    }
+
+    public function launchOrResumeInstance()
+    {
+        $host = $this->instance->findExisting();
+
+        if (empty($host)) {
+            $host = $this->launchInstance();
+        }
+
+        return $host;
+    }
+
+    private function launchInstance()
+    {
+        $instanceIds = $this->instance->launch();
+
+        try {
+            $host = $this->instance->setup($instanceIds);
+        } catch (\Exception $e) {
+            $this->instance->terminate($instanceIds);
+
+            throw new \RuntimeException('We failed to launch a new instance so we terminated it directly. Try again! Error Message: ' . $e->getMessage());
+        }
+
+        return $host;
+    }
+
+}
\ No newline at end of file
diff --git a/plugins/TestRunner/Runner/Remote.php b/plugins/TestRunner/Runner/Remote.php
new file mode 100644
index 0000000000..d068aa2f93
--- /dev/null
+++ b/plugins/TestRunner/Runner/Remote.php
@@ -0,0 +1,68 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\TestRunner\Runner;
+
+use \Net_SSH2;
+
+class Remote
+{
+    /**
+     * @var \Net_SSH2
+     */
+    private $ssh;
+
+    public function __construct(Net_SSH2 $ssh)
+    {
+        $this->ssh = $ssh;
+    }
+
+    public function updatePiwik($gitHash)
+    {
+        $this->ssh->exec('git reset --hard');
+        $this->ssh->exec('git clean -d -f');
+        $this->ssh->exec('git fetch --all');
+        $this->ssh->exec('git checkout ' . trim($gitHash));
+        $this->ssh->exec('sudo composer.phar self-update');
+        $this->ssh->exec('composer.phar install');
+    }
+
+    public function runTests($host, $testSuite)
+    {
+        $this->prepareTestRun($host);
+        $this->printVersionInfo();
+        $this->doRunTests($testSuite);
+    }
+
+    private function prepareTestRun($host)
+    {
+        $this->ssh->exec('cp ./tests/PHPUnit/phpunit.xml.dist ./tests/PHPUnit/phpunit.xml');
+        $this->ssh->exec("sed -i 's/@REQUEST_URI@/\\//g' ./tests/PHPUnit/phpunit.xml");
+        $this->ssh->exec("sed -i 's/amazonAwsUrl/$host/g' ./config/config.ini.php");
+    }
+
+    private function printVersionInfo()
+    {
+        $this->ssh->exec('php --version');
+        $this->ssh->exec('mysql --version');
+        $this->ssh->exec('phantomjs --version');
+    }
+
+    private function doRunTests($testSuite)
+    {
+        if ('all' === $testSuite) {
+            $this->ssh->exec('php console tests:run --options="--colors"');
+        } elseif ('ui' === $testSuite) {
+            $this->ssh->exec('php console tests:run-ui');
+        } else {
+            $this->ssh->exec('php console tests:run --options="--colors" --testsuite="unit"');
+            $this->ssh->exec('php console tests:run --options="--colors" --testsuite="' . $testSuite . '"');
+        }
+    }
+}
diff --git a/plugins/TestRunner/TestRunner.php b/plugins/TestRunner/TestRunner.php
new file mode 100644
index 0000000000..fc5462e536
--- /dev/null
+++ b/plugins/TestRunner/TestRunner.php
@@ -0,0 +1,15 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\TestRunner;
+
+/**
+ */
+class TestRunner extends \Piwik\Plugin
+{
+}
diff --git a/plugins/TestRunner/plugin.json b/plugins/TestRunner/plugin.json
new file mode 100644
index 0000000000..b56ff70a21
--- /dev/null
+++ b/plugins/TestRunner/plugin.json
@@ -0,0 +1,19 @@
+{
+  "name": "TestRunner",
+  "version": "0.1.0",
+  "description": "Run Tests",
+  "theme": false,
+  "require": {
+     "piwik": ">=2.8.1-rc1"
+  },
+  "authors": [
+      {
+          "name": "Piwik",
+          "email": "hello@piwik.org",
+          "homepage": "http://piwik.org"
+      }
+  ],
+  "license": "GPL v3+",
+  "keywords": ["test", "runner"],
+  "homepage": "http://piwik.org"
+}
\ No newline at end of file
diff --git a/plugins/TestRunner/screenshots/.gitkeep b/plugins/TestRunner/screenshots/.gitkeep
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/plugins/TestRunner/screenshots/.gitkeep