Update Marketplace to work with new API (#10799)

* starting to port marketplace to piwik 3

* updating tests

* fix translation key

* fix various issues

* use material select

* fix plugin upload

* deprecate license_homepage plugin metadata and link to a LICENSE[.md|.txt] file if found (#10756)

* deprecate license_homepage plugin metadata, and link to a LICENSE[.md|.txt] file if found

* Make license view HTML only without menu

* fix tests and update

* fix some links did not work

* we need to show warnings even when plugin is installed, not only when activated. otherwise it is not clear why something is not downloadable

* fix install was not working

* improved responsiveness of marketplace

* fix more tests

* fix search was shown when only a few plugins are there

* fix ui tests

* fix some translations

* fix tests and remove duplicated test

4 files changed, 520 insertions, 0 deletions

diff --git a/plugins/Marketplace/Api/Client.php b/plugins/Marketplace/Api/Client.php
new file mode 100644
index 0000000000..0b4589f33a
--- /dev/null
+++ b/plugins/Marketplace/Api/Client.php
@@ -0,0 +1,326 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Marketplace\Api;
+
+use Piwik\Cache;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Filesystem;
+use Piwik\Plugin;
+use Piwik\Plugins\Marketplace\Environment;
+use Piwik\Plugins\Marketplace\Api\Service;
+use Piwik\SettingsServer;
+use Exception as PhpException;
+use Psr\Log\LoggerInterface;
+
+/**
+ *
+ */
+class Client
+{
+    const CACHE_TIMEOUT_IN_SECONDS = 3600;
+    const HTTP_REQUEST_TIMEOUT = 60;
+
+    /**
+     * @var Service
+     */
+    private $service;
+
+    /**
+     * @var Cache\Lazy
+     */
+    private $cache;
+
+    /**
+     * @var Plugin\Manager
+     */
+    private $pluginManager;
+
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    /**
+     * @var Environment
+     */
+    private $environment;
+
+    public function __construct(Service $service, Cache\Lazy $cache, LoggerInterface $logger, Environment $environment)
+    {
+        $this->service = $service;
+        $this->cache = $cache;
+        $this->logger = $logger;
+        $this->pluginManager = Plugin\Manager::getInstance();
+        $this->environment = $environment;
+    }
+
+    public function setEnvironment($environment)
+    {
+        $this->environment = $environment;
+    }
+
+    public function getEnvironment()
+    {
+        return $this->environment;
+    }
+
+    public function getPluginInfo($name)
+    {
+        $action = sprintf('plugins/%s/info', $name);
+
+        $plugin = $this->fetch($action, array());
+
+        if (!empty($plugin) && $this->shouldIgnorePlugin($plugin)) {
+            return;
+        }
+
+        return $plugin;
+    }
+
+    public function getInfo()
+    {
+        try {
+            $info = $this->fetch('info', array());
+        } catch (Exception $e) {
+            $info = null;
+        }
+
+        return $info;
+    }
+
+    public function getConsumer()
+    {
+        try {
+            $licenses = $this->fetch('consumer', array());
+        } catch (Exception $e) {
+            $licenses = null;
+        }
+
+        return $licenses;
+    }
+
+    public function isValidConsumer()
+    {
+        try {
+            $consumer = $this->fetch('consumer/validate', array());
+        } catch (Exception $e) {
+            $consumer = null;
+        }
+
+        return !empty($consumer['isValid']);
+    }
+
+    private function getRandomTmpPluginDownloadFilename()
+    {
+        $tmpPluginPath = StaticContainer::get('path.tmp') . '/latest/plugins/';
+
+        // we generate a random unique id as filename to prevent any user could possibly download zip directly by
+        // opening $piwikDomain/tmp/latest/plugins/$pluginName.zip in the browser. Instead we make it harder here
+        // and try to make sure to delete file in case of any error.
+        $tmpPluginFolder = Common::generateUniqId();
+
+        return $tmpPluginPath . $tmpPluginFolder . '.zip';
+    }
+
+    public function download($pluginOrThemeName)
+    {
+        @ignore_user_abort(true);
+        SettingsServer::setMaxExecutionTime(0);
+
+        $downloadUrl = $this->getDownloadUrl($pluginOrThemeName);
+
+        if (empty($downloadUrl)) {
+            return false;
+        }
+
+        // in the beginning we allowed to specify a download path but this way we make sure security is always taken
+        // care of and we always generate a random download filename.
+        $target = $this->getRandomTmpPluginDownloadFilename();
+
+        Filesystem::deleteFileIfExists($target);
+
+        $success = $this->service->download($downloadUrl, $target, static::HTTP_REQUEST_TIMEOUT);
+
+        if ($success) {
+            return $target;
+        }
+
+        return false;
+    }
+
+    /**
+     * @param \Piwik\Plugin[] $plugins
+     * @return array|mixed
+     */
+    public function checkUpdates($plugins)
+    {
+        $params = array();
+
+        foreach ($plugins as $plugin) {
+            $pluginName = $plugin->getPluginName();
+            if (!$this->pluginManager->isPluginBundledWithCore($pluginName)) {
+                $params[] = array('name' => $plugin->getPluginName(), 'version' => $plugin->getVersion());
+            }
+        }
+
+        if (empty($params)) {
+            return array();
+        }
+
+        $params = array('plugins' => $params);
+
+        $hasUpdates = $this->fetch('plugins/checkUpdates', array('plugins' => json_encode($params)));
+
+        if (empty($hasUpdates)) {
+            return array();
+        }
+
+        return $hasUpdates;
+    }
+
+    /**
+     * @param  \Piwik\Plugin[] $plugins
+     * @return array
+     */
+    public function getInfoOfPluginsHavingUpdate($plugins)
+    {
+        $hasUpdates = $this->checkUpdates($plugins);
+
+        $pluginDetails = array();
+
+        foreach ($hasUpdates as $pluginHavingUpdate) {
+            if (empty($pluginHavingUpdate)) {
+                continue;
+            }
+
+            try {
+                $plugin = $this->getPluginInfo($pluginHavingUpdate['name']);
+            } catch (PhpException $e) {
+                $this->logger->error($e->getMessage());
+                $plugin = null;
+            }
+
+            if (!empty($plugin)) {
+                $plugin['repositoryChangelogUrl'] = $pluginHavingUpdate['repositoryChangelogUrl'];
+                $pluginDetails[] = $plugin;
+            }
+
+        }
+
+        return $pluginDetails;
+    }
+
+    public function searchForPlugins($keywords, $query, $sort, $purchaseType)
+    {
+        $response = $this->fetch('plugins', array('keywords' => $keywords, 'query' => $query, 'sort' => $sort, 'purchase_type' => $purchaseType));
+
+        if (!empty($response['plugins'])) {
+            return $this->removeNotNeededPluginsFromResponse($response);
+        }
+
+        return array();
+    }
+
+    private function removeNotNeededPluginsFromResponse($response)
+    {
+        foreach ($response['plugins'] as $index => $plugin) {
+            if ($this->shouldIgnorePlugin($plugin)) {
+                unset($response['plugins'][$index]);
+                continue;
+            }
+        }
+        return array_values($response['plugins']);
+    }
+
+    private function shouldIgnorePlugin($plugin)
+    {
+        return !empty($plugin['isCustomPlugin']);
+    }
+
+    public function searchForThemes($keywords, $query, $sort, $purchaseType)
+    {
+        $response = $this->fetch('themes', array('keywords' => $keywords, 'query' => $query, 'sort' => $sort, 'purchase_type' => $purchaseType));
+
+        if (!empty($response['plugins'])) {
+            return $this->removeNotNeededPluginsFromResponse($response);
+        }
+
+        return array();
+    }
+
+    private function fetch($action, $params)
+    {
+        ksort($params); // sort params so cache is reused more often even if param order is different
+
+        $releaseChannel = $this->environment->getReleaseChannel();
+
+        if (!empty($releaseChannel)) {
+            $params['release_channel'] = $releaseChannel;
+        }
+
+        $params['prefer_stable'] = (int) $this->environment->doesPreferStable();
+        $params['piwik'] = $this->environment->getPiwikVersion();
+        $params['php'] = $this->environment->getPhpVersion();
+        $params['mysql'] = $this->environment->getMySQLVersion();
+        $params['num_users'] = $this->environment->getNumUsers();
+        $params['num_websites'] = $this->environment->getNumWebsites();
+
+        $query = http_build_query($params);
+        $cacheId = $this->getCacheKey($action, $query);
+
+        $result = $this->cache->fetch($cacheId);
+
+        if ($result !== false) {
+            return $result;
+        }
+
+        try {
+            $result = $this->service->fetch($action, $params);
+        } catch (Service\Exception $e) {
+            throw new Exception($e->getMessage(), $e->getCode());
+        }
+
+        $this->cache->save($cacheId, $result, self::CACHE_TIMEOUT_IN_SECONDS);
+
+        return $result;
+    }
+
+    public function clearAllCacheEntries()
+    {
+        $this->cache->flushAll();
+    }
+
+    private function getCacheKey($action, $query)
+    {
+        $version = $this->service->getVersion();
+
+        return sprintf('marketplace.api.%s.%s.%s', $version, str_replace('/', '.', $action), md5($query));
+    }
+
+    /**
+     * @param  $pluginOrThemeName
+     * @throws Exception
+     * @return string
+     */
+    public function getDownloadUrl($pluginOrThemeName)
+    {
+        $plugin = $this->getPluginInfo($pluginOrThemeName);
+
+        if (empty($plugin['versions'])) {
+            throw new Exception('Plugin has no versions.');
+        }
+
+        $latestVersion = array_pop($plugin['versions']);
+        $downloadUrl = $latestVersion['download'];
+
+        return $this->service->getDomain() . $downloadUrl . '?coreVersion=' . $this->environment->getPiwikVersion();
+    }
+
+}
diff --git a/plugins/Marketplace/Api/Exception.php b/plugins/Marketplace/Api/Exception.php
new file mode 100644
index 0000000000..7dce508bf9
--- /dev/null
+++ b/plugins/Marketplace/Api/Exception.php
@@ -0,0 +1,17 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\Marketplace\Api;
+
+/**
+ */
+class Exception extends \Exception
+{
+
+}
diff --git a/plugins/Marketplace/Api/Service.php b/plugins/Marketplace/Api/Service.php
new file mode 100644
index 0000000000..e493b69941
--- /dev/null
+++ b/plugins/Marketplace/Api/Service.php
@@ -0,0 +1,158 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Marketplace\Api;
+
+use Piwik\Http;
+
+/**
+ *
+ */
+class Service
+{
+    const CACHE_TIMEOUT_IN_SECONDS = 1200;
+    const HTTP_REQUEST_TIMEOUT = 60;
+
+    /**
+     * @var string
+     */
+    private $domain;
+
+    /**
+     * @var null|string
+     */
+    private $accessToken;
+
+    /**
+     * API version to use on the Marketplace
+     * @var string
+     */
+    private $version = '2.0';
+
+    public function __construct($domain)
+    {
+        $this->domain = $domain;
+    }
+
+    public function authenticate($accessToken)
+    {
+        if (empty($accessToken)) {
+            $this->accessToken = null;
+        } elseif (ctype_alnum($accessToken)) {
+            $this->accessToken = $accessToken;
+        }
+    }
+
+    /**
+     * The API version that will be used on the Marketplace.
+     * @return string eg 2.0
+     */
+    public function getVersion()
+    {
+        return $this->version;
+    }
+
+    /**
+     * Returns the currently set access token
+     * @return null|string
+     */
+    public function getAccessToken()
+    {
+        return $this->accessToken;
+    }
+
+    public function hasAccessToken()
+    {
+        return !empty($this->accessToken);
+    }
+
+    /**
+     * Downloads data from the given URL via a POST request. If a destination path is given, the downloaded data
+     * will be stored in the given path and returned otherwise.
+     *
+     * Make sure to call {@link authenticate()} to download paid plugins.
+     *
+     * @param string $url An absolute URL to the marketplace including domain.
+     * @param null|string $destinationPath
+     * @param null|int $timeout  Defaults to 60 seconds see {@link self::HTTP_REQUEST_METHOD}
+     * @return bool|string  Returns the downloaded data or true if a destination path was given.
+     * @throws \Exception
+     */
+    public function download($url, $destinationPath = null, $timeout = null)
+    {
+        $method = Http::getTransportMethod();
+
+        if (!isset($timeout)) {
+            $timeout = static::HTTP_REQUEST_TIMEOUT;
+        }
+
+        $post = null;
+        if ($this->accessToken) {
+            $post = array('access_token' => $this->accessToken);
+        }
+
+        $file = Http::ensureDestinationDirectoryExists($destinationPath);
+
+        $response = Http::sendHttpRequestBy($method,
+                                            $url,
+                                            $timeout,
+                                            $userAgent = null,
+                                            $destinationPath,
+                                            $file,
+                                            $followDepth = 0,
+                                            $acceptLanguage = false,
+                                            $acceptInvalidSslCertificate = false,
+                                            $byteRange = false, $getExtendedInfo = false, $httpMethod = 'POST',
+                                            $httpUsername = null, $httpPassword = null, $post);
+
+        return $response;
+    }
+
+    /**
+     * Executes the given API action on the Marketplace using the given params and returns the result.
+     *
+     * Make sure to call {@link authenticate()} to download paid plugins.
+     *
+     * @param string $action  eg 'plugins', 'plugins/$pluginName/info', ...
+     * @param array $params   eg array('sort' => 'alpha')
+     * @return mixed
+     * @throws Service\Exception
+     */
+    public function fetch($action, $params)
+    {
+        $endpoint = sprintf('%s/api/%s/', $this->domain, $this->version);
+
+        $query = http_build_query($params);
+        $url   = sprintf('%s%s?%s', $endpoint, $action, $query);
+
+        $response = $this->download($url);
+
+        $result = json_decode($response, true);
+
+        if (is_null($result)) {
+            $message = sprintf('There was an error reading the response from the Marketplace: Please try again later.');
+            throw new Service\Exception($message, Service\Exception::HTTP_ERROR);
+        }
+
+        if (!empty($result['error'])) {
+            throw new Service\Exception($result['error'], Service\Exception::API_ERROR);
+        }
+
+        return $result;
+    }
+
+    /**
+     * Get the domain that is used in order to access the Marketplace. Eg http://plugins.piwik.org
+     * @return string
+     */
+    public function getDomain()
+    {
+        return $this->domain;
+    }
+
+}
diff --git a/plugins/Marketplace/Api/Service/Exception.php b/plugins/Marketplace/Api/Service/Exception.php
new file mode 100644
index 0000000000..181c0be568
--- /dev/null
+++ b/plugins/Marketplace/Api/Service/Exception.php
@@ -0,0 +1,19 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\Marketplace\Api\Service;
+
+/**
+ */
+class Exception extends \Exception
+{
+    const HTTP_ERROR = 100;
+    const API_ERROR  = 101;
+
+}