Prepares Weblate migration (#17927)

* Add translation stats to readme

* improve generate-intl command

* update readme

* remove translation workflow

* allow empty translation files

* Adds internal parameter to ignore config in LanguageManager API

* ignore missing translators

* extend commands to ignore language info

* Update lang/README.md

Co-authored-by: Lukas Winkler <git@lw1.at>

* Update commands to use Weblate instead of Transifex API

* Replace remaining Transifex occurences

* submodule updates

* fix/update tests

Co-authored-by: Lukas Winkler <git@lw1.at>

25 files changed, 95 insertions, 330 deletions

diff --git a/plugins/Bandwidth b/plugins/Bandwidth
-Subproject 9f591fdb82980ddf3a4e36f1b1c71e135bd5992
+Subproject a04b8c3de354b8547e2d54ee3e748c1d102175b
diff --git a/plugins/CustomAlerts b/plugins/CustomAlerts
-Subproject 28c937b253b5b94c6e9d1d39451c6228bf69647
+Subproject 2af4a1d48dc6d67101c75f0bd7cccd8ba69c2b1
diff --git a/plugins/CustomVariables b/plugins/CustomVariables
-Subproject b84e62768e37a8cd4b93bffbd273336c8ddd315
+Subproject b998e663991bc3000cce0af5f918d2a53bcca5e
diff --git a/plugins/DeviceDetectorCache b/plugins/DeviceDetectorCache
-Subproject dd3f240bf8b850ab5d5ab0811f3b7e772eefbca
+Subproject ec65d1736713a8b984f3ad5ad48cac293d07561
diff --git a/plugins/Intl/Commands/GenerateIntl.php b/plugins/Intl/Commands/GenerateIntl.php
index 207029f9ca..af7ad7903d 100644
--- a/plugins/Intl/Commands/GenerateIntl.php
+++ b/plugins/Intl/Commands/GenerateIntl.php
@@ -314,6 +314,10 @@ class GenerateIntl extends ConsoleCommand
             $territoryData = json_decode($territoryData, true);
             $territoryData = $territoryData['main'][$requestLangCode]['localeDisplayNames']['territories'];
 
+            if (empty($territoryData)) {
+                throw new \Exception();
+            }
+
             foreach ($countryCodes AS $code) {
                 if (!empty($territoryData[$code]) && $territoryData[$code] != $code) {
                     $translations['Intl']['Country_' . $code] = $this->transform($territoryData[$code]);
@@ -341,6 +345,10 @@ class GenerateIntl extends ConsoleCommand
             $calendarData = json_decode($calendarData, true);
             $calendarData = $calendarData['main'][$requestLangCode]['dates']['calendars']['gregorian'];
 
+            if (empty($calendarData)) {
+                throw new \Exception();
+            }
+
             for ($i = 1; $i <= 12; $i++) {
                 $translations['Intl']['Month_Short_' . $i] = $calendarData['months']['format']['abbreviated'][$i];
                 $translations['Intl']['Month_Long_' . $i] = $calendarData['months']['format']['wide'][$i];
@@ -413,6 +421,10 @@ class GenerateIntl extends ConsoleCommand
             $dateFieldData = json_decode($dateFieldData, true);
             $dateFieldData = $dateFieldData['main'][$requestLangCode]['dates']['fields'];
 
+            if (empty($dateFieldData)) {
+                throw new \Exception();
+            }
+
             $translations['Intl']['PeriodWeek'] = $dateFieldData['week']['displayName'];
             $translations['Intl']['PeriodYear'] = $dateFieldData['year']['displayName'];
             $translations['Intl']['PeriodDay'] = $dateFieldData['day']['displayName'];
@@ -436,6 +448,10 @@ class GenerateIntl extends ConsoleCommand
             $timeZoneData = json_decode($timeZoneData, true);
             $timeZoneData = $timeZoneData['main'][$requestLangCode]['dates']['timeZoneNames'];
 
+            if (empty($timeZoneData)) {
+                throw new \Exception();
+            }
+
             $cities = array();
             foreach ($timeZoneData['zone'] as $key1 => $level1) {
                 foreach ($level1 as $key2 => $level2) {
@@ -495,6 +511,10 @@ class GenerateIntl extends ConsoleCommand
             $unitsData = json_decode($unitsData, true);
             $unitsData = $unitsData['main'][$requestLangCode]['numbers'];
 
+            if (empty($unitsData)) {
+                throw new \Exception();
+            }
+
             $numberingSystem = $unitsData['defaultNumberingSystem'];
 
             $translations['Intl']['NumberSymbolDecimal']  = $unitsData['symbols-numberSystem-' . $numberingSystem]['decimal'];
@@ -521,6 +541,10 @@ class GenerateIntl extends ConsoleCommand
             $unitsData = json_decode($unitsData, true);
             $unitsData = $unitsData['main'][$requestLangCode]['units'];
 
+            if (empty($unitsData)) {
+                throw new \Exception();
+            }
+
             $translations['Intl']['NSeconds']       = $this->replacePlaceHolder($unitsData['long']['duration-second']['unitPattern-count-other']);
             $translations['Intl']['NSecondsShort']  = $this->replacePlaceHolder($unitsData['narrow']['duration-second']['unitPattern-count-other']);
             $translations['Intl']['Seconds']        = $unitsData['long']['duration-second']['displayName'];
@@ -575,6 +599,10 @@ class GenerateIntl extends ConsoleCommand
             $currencyData = json_decode($currencyData, true);
             $currencyData = $currencyData['main'][$requestLangCode]['numbers']['currencies'];
 
+            if (empty($currencyData)) {
+                throw new \Exception();
+            }
+
             $dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\CurrencyDataProvider');
             foreach ($dataProvider->getCurrencyList() as $code => $currency) {
                 if (isset($currencyData[$code]['displayName'])) {
diff --git a/plugins/LanguagesManager/API.php b/plugins/LanguagesManager/API.php
index fa80210065..8b071ae5da 100644
--- a/plugins/LanguagesManager/API.php
+++ b/plugins/LanguagesManager/API.php
@@ -32,31 +32,33 @@ use Piwik\Translation\Loader\DevelopmentLoader;
  */
 class API extends \Piwik\Plugin\API
 {
-    protected $availableLanguageNames = null;
-    protected $languageNames = null;
+    protected $availableLanguageNames = [];
+    protected $languageNames = [];
 
     /**
      * Returns true if specified language is available
      *
      * @param string $languageCode
+     * @param bool $_ignoreConfig
      * @return bool true if language available; false otherwise
      */
-    public function isLanguageAvailable($languageCode)
+    public function isLanguageAvailable($languageCode, $_ignoreConfig = false)
     {
         return $languageCode !== false
         && Filesystem::isValidFilename($languageCode)
-        && in_array($languageCode, $this->getAvailableLanguages());
+        && in_array($languageCode, $this->getAvailableLanguages($_ignoreConfig));
     }
 
     /**
      * Return array of available languages
      *
+     * @param bool $_ignoreConfig
      * @return array Array of strings, each containing its ISO language code
      */
-    public function getAvailableLanguages()
+    public function getAvailableLanguages($_ignoreConfig = false)
     {
-        if (!is_null($this->languageNames)) {
-            return $this->languageNames;
+        if (!empty($this->languageNames[$_ignoreConfig])) {
+            return $this->languageNames[$_ignoreConfig];
         }
         $path = PIWIK_INCLUDE_PATH . "/lang/";
         $languagesPath = _glob($path . "*.json");
@@ -71,7 +73,12 @@ class API extends \Piwik\Plugin\API
 
         $configLanguages = Config::getInstance()->Languages["Languages"];
 
-        $languages = array_intersect($filesystemLanguages, $configLanguages);
+        if ($_ignoreConfig) {
+            $languages = $filesystemLanguages;
+        } else {
+            $languages = array_intersect($filesystemLanguages, $configLanguages);
+        }
+
         $this->enableDevelopmentLanguageInDevEnvironment($languages);
 
         /**
@@ -83,18 +90,19 @@ class API extends \Piwik\Plugin\API
          */
         Piwik::postEvent('LanguagesManager.getAvailableLanguages', array(&$languages));
 
-        $this->languageNames = $languages;
+        $this->languageNames[$_ignoreConfig] = $languages;
         return $languages;
     }
 
     /**
      * Return information on translations (code, language, % translated, etc)
      *
-     * @param boolean $excludeNonCorePlugins excludes non core plugin from percentage calculation
+     * @param bool $excludeNonCorePlugins excludes non core plugin from percentage calculation
+     * @param bool $_ignoreConfig
      *
      * @return array Array of arrays
      */
-    public function getAvailableLanguagesInfo($excludeNonCorePlugins=true)
+    public function getAvailableLanguagesInfo($excludeNonCorePlugins=true, $_ignoreConfig = false)
     {
         $data = file_get_contents(PIWIK_INCLUDE_PATH . '/lang/en.json');
         $englishTranslation = json_decode($data, true);
@@ -120,7 +128,7 @@ class API extends \Piwik\Plugin\API
             }
         }
 
-        $filenames = $this->getAvailableLanguages();
+        $filenames = $this->getAvailableLanguages($_ignoreConfig);
         $languagesInfo = array();
         foreach ($filenames as $filename) {
             $data = file_get_contents(sprintf('%s/lang/%s.json', PIWIK_INCLUDE_PATH, $filename));
@@ -168,7 +176,7 @@ class API extends \Piwik\Plugin\API
             $languageInfo = array('code'                => $filename,
                                   'name'                => $translations['Intl']['OriginalLanguageName'],
                                   'english_name'        => $translations['Intl']['EnglishLanguageName'],
-                                  'translators'         => $translations['General']['TranslatorName'],
+                                  'translators'         => $translations['General']['TranslatorName'] ?? '-',
                                   'percentage_complete' => $percentageComplete . '%',
             );
             $languagesInfo[] = $languageInfo;
@@ -179,12 +187,13 @@ class API extends \Piwik\Plugin\API
     /**
      * Return array of available languages
      *
+     * @param bool $_ignoreConfig
      * @return array Array of array, each containing its ISO language code and name of the language
      */
-    public function getAvailableLanguageNames()
+    public function getAvailableLanguageNames($_ignoreConfig = false)
     {
-        $this->loadAvailableLanguages();
-        return $this->availableLanguageNames;
+        $this->loadAvailableLanguages($_ignoreConfig);
+        return $this->availableLanguageNames[$_ignoreConfig];
     }
 
     /**
@@ -342,19 +351,19 @@ class API extends \Piwik\Plugin\API
         return $lang;
     }
 
-    private function loadAvailableLanguages()
+    private function loadAvailableLanguages($_ignoreConfig = false)
     {
-        if (!is_null($this->availableLanguageNames)) {
+        if (!empty($this->availableLanguageNames[$_ignoreConfig])) {
             return;
         }
 
-        $cacheId = 'availableLanguages';
+        $cacheId = 'availableLanguages' . (int) $_ignoreConfig;
         $cache = PiwikCache::getEagerCache();
 
         if ($cache->contains($cacheId)) {
             $languagesInfo = $cache->fetch($cacheId);
         } else {
-            $languages = $this->getAvailableLanguages();
+            $languages = $this->getAvailableLanguages($_ignoreConfig);
             $languagesInfo = array();
             foreach ($languages as $languageCode) {
                 $data = @file_get_contents(PIWIK_INCLUDE_PATH . "/plugins/Intl/lang/$languageCode.json");
@@ -375,7 +384,7 @@ class API extends \Piwik\Plugin\API
             $cache->save($cacheId, $languagesInfo);
         }
 
-        $this->availableLanguageNames = $languagesInfo;
+        $this->availableLanguageNames[$_ignoreConfig] = $languagesInfo;
     }
 
     private function enableDevelopmentLanguageInDevEnvironment(&$languages)
diff --git a/plugins/LanguagesManager/Commands/CreatePull.php b/plugins/LanguagesManager/Commands/CreatePull.php
deleted file mode 100644
index 683f6e9c2b..0000000000
--- a/plugins/LanguagesManager/Commands/CreatePull.php
+++ /dev/null
@@ -1,233 +0,0 @@
-<?php
-/**
- * Matomo - free/libre analytics platform
- *
- * @link     https://matomo.org
- * @license  http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- *
- */
-
-namespace Piwik\Plugins\LanguagesManager\Commands;
-
-use Piwik\Plugins\LanguagesManager\API;
-use Symfony\Component\Console\Input\ArrayInput;
-use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Console\Input\InputOption;
-use Symfony\Component\Console\Output\OutputInterface;
-
-/**
- */
-class CreatePull extends TranslationBase
-{
-    const GIT_BASE_BRANCH = '4.x-dev';
-
-    protected function configure()
-    {
-        $this->setName('translations:createpull')
-            ->setDescription('Updates translation files')
-            ->addOption('username', 'u', InputOption::VALUE_OPTIONAL, 'Transifex username')
-            ->addOption('password', 'p', InputOption::VALUE_OPTIONAL, 'Transifex password')
-            ->addOption('slug', 's', InputOption::VALUE_OPTIONAL, 'Transifex project slug')
-            ->addOption('plugin', 'P', InputOption::VALUE_OPTIONAL, 'optional name of plugin to update translations for');
-    }
-
-    protected function execute(InputInterface $input, OutputInterface $output)
-    {
-        $changes = shell_exec('git status --porcelain -uno');
-
-        if (!empty($changes)) {
-
-            $output->writeln("You have uncommitted changes. Creating pull request is only available with a clean working directory");
-            return;
-        }
-
-        $unpushedCommits = shell_exec('git log origin/' . self::GIT_BASE_BRANCH . '..HEAD');
-
-        if (!empty($unpushedCommits)) {
-
-            $output->writeln("You have unpushed commits. Creating pull request is only available with a clean working directory");
-            return;
-        }
-
-        chdir(PIWIK_DOCUMENT_ROOT);
-
-        shell_exec('
-            git checkout -f ' . self::GIT_BASE_BRANCH . ' > /dev/null 2>&1
-            git pull > /dev/null 2>&1
-            git submodule init > /dev/null 2>&1
-            git submodule update > /dev/null 2>&1
-        ');
-
-        $plugin = $input->getOption('plugin');
-        if (!empty($plugin)) {
-
-            chdir(PIWIK_DOCUMENT_ROOT.DIRECTORY_SEPARATOR.'plugins'.DIRECTORY_SEPARATOR.$plugin);
-            shell_exec('
-                git checkout ' . self::GIT_BASE_BRANCH . ' > /dev/null 2>&1
-                git pull > /dev/null 2>&1
-            ');
-        }
-
-        // check if branch exists localy and track it if not
-        $branch = shell_exec('git branch | grep translationupdates');
-
-        if (empty($branch)) {
-
-            shell_exec('git checkout -b translationupdates origin/translationupdates');
-        }
-
-        // switch to branch and update it to latest $GIT_BASE_BRANCH
-        shell_exec('
-            git checkout -f translationupdates > /dev/null 2>&1
-            git reset --hard origin/' . self::GIT_BASE_BRANCH . ' > /dev/null 2>&1
-            git push origin translationupdates > /dev/null 2>&1
-        ');
-
-        // update translation files
-        $command = $this->getApplication()->find('translations:update');
-        $arguments = array(
-            'command'    => 'translations:update',
-            '--username' => $input->getOption('username'),
-            '--password' => $input->getOption('password'),
-            '--slug'     => $input->getOption('slug'),
-            '--plugin'   => $plugin
-        );
-        $inputObject = new ArrayInput($arguments);
-        $inputObject->setInteractive($input->isInteractive());
-        $command->run($inputObject, $output);
-
-        shell_exec('git add lang/. > /dev/null 2>&1');
-
-        if (empty($plugin)) {
-            foreach (Update::getPluginsInCore() as $pluginName) {
-                shell_exec(sprintf('git add plugins/%s/lang/. > /dev/null 2>&1', $pluginName));
-            }
-        }
-
-        $changes = shell_exec('git status --porcelain -uno');
-
-        if (empty($changes)) {
-
-            $output->writeln("Nothing changed. Everything is already up to date.");
-            shell_exec('git checkout ' . self::GIT_BASE_BRANCH . ' > /dev/null 2>&1');
-            return;
-        }
-
-        API::unsetInstance(); // reset languagemanager api (to force refresh of data)
-
-        $stats = shell_exec('git diff --numstat HEAD');
-
-        preg_match_all('/([0-9]+)\t([0-9]+)\t[a-zA-Z\/]*lang\/([a-z]{2,3}(?:-[a-z]{2,3})?)\.json/', $stats, $lineChanges);
-
-        $addedLinesSum = 0;
-        if (!empty($lineChanges[1])) {
-            $addedLinesSum = array_sum($lineChanges[1]);
-        }
-
-        $linesSumByLang = array();
-        $lineChangesCount = count($lineChanges[0]);
-        for ($i = 0; $i < $lineChangesCount; $i++) {
-            @$linesSumByLang[$lineChanges[3][$i]] += $lineChanges[1][$i];
-        }
-
-        preg_match_all('/M  [a-zA-Z\/]*lang\/([a-z]{2,3}(?:-[a-z]{2,3})?)\.json/', $changes, $modifiedFiles);
-        preg_match_all('/A  [a-zA-Z\/]*lang\/([a-z]{2,3}(?:-[a-z]{2,3})?)\.json/', $changes, $addedFiles);
-
-        $messages = array();
-
-        $languageCodesTouched = array();
-        if (!empty($addedFiles[1])) {
-            foreach ($addedFiles[1] as $addedFile) {
-                $languageInfo = $this->getLanguageInfoByIsoCode($addedFile);
-                $messages[$addedFile] = sprintf('- Added %s (%s changes / %s translated)\n', $languageInfo['english_name'], $linesSumByLang[$addedFile], $languageInfo['percentage_complete']);
-            }
-            $languageCodesTouched = array_merge($languageCodesTouched, $addedFiles[1]);
-        }
-
-        if (!empty($modifiedFiles[1])) {
-            foreach ($modifiedFiles[1] as $modifiedFile) {
-                if ($linesSumByLang[$modifiedFile]) {
-                    $languageInfo            = $this->getLanguageInfoByIsoCode($modifiedFile);
-                    $messages[$modifiedFile] = sprintf(
-                        '- Updated %s (%s changes / %s translated)\n',
-                        $languageInfo['english_name'],
-                        $linesSumByLang[$modifiedFile],
-                        $languageInfo['percentage_complete']
-                    );
-                    $languageCodesTouched[] = $modifiedFile;
-                }
-            }
-            $languageCodesTouched = array_unique($languageCodesTouched);
-        }
-
-        $message = implode('', $messages);
-
-        $message .= '\n\nHelp us translate Matomo in your language!\nSignup at https://www.transifex.com/matomo/matomo/\nIf you have any questions, get in touch with us at translations@matomo.org';
-
-        $languageCodesTouched = array_unique($languageCodesTouched, SORT_REGULAR);
-
-        $title = sprintf(
-            'Updated %s strings in %u languages (%s)',
-            $addedLinesSum,
-            count($languageCodesTouched),
-            implode(', ', $languageCodesTouched)
-        );
-
-        shell_exec('git commit -m "language update ${pluginName}"');
-        shell_exec('git push');
-        shell_exec('git checkout ' . self::GIT_BASE_BRANCH . ' > /dev/null 2>&1');
-
-        $this->createPullRequest($output, $title, $message);
-    }
-
-    private function getLanguageInfoByIsoCode($isoCode)
-    {
-        $languages = API::getInstance()->getAvailableLanguagesInfo();
-        foreach ($languages as $languageInfo) {
-            if ($languageInfo['code'] == $isoCode) {
-                return $languageInfo;
-            }
-        }
-        return array();
-    }
-
-    private function createPullRequest(OutputInterface $output, $title, $message)
-    {
-        $dialog = $this->getHelperSet()->get('dialog');
-
-        while (true) {
-
-            $username = $dialog->ask($output, 'Please provide your GitHub username (to create a pull request using GitHub API): ');
-
-            $returnCode = shell_exec('curl \
-                 -X POST \
-                 -k \
-                 --silent \
-                 --write-out %{http_code} \
-                 --stderr /dev/null \
-                 -o /dev/null \
-                 -u '.$username.' \
-                 --data "{\"title\":\"[automatic translation update] '.$title.'\",\"body\":\"'.$message.'\",\"head\":\"translationupdates\",\"base\":\"' . self::GIT_BASE_BRANCH . '\"}" \
-                 -H "Accept: application/json" \
-                 https://api.github.com/repos/matomo-org/matomo/pulls');
-
-            switch ($returnCode) {
-                case 401:
-                    $output->writeln("Pull request failed. Bad credentials... Please try again");
-                    continue 2;
-
-                case 422:
-                    $output->writeln("Pull request failed. Unprocessable Entity. Maybe a pull request was already created before.");
-                    return;
-
-                case 201:
-                case 200:
-                    $output->writeln("Pull request successfully created.");
-                    return;
-
-                default:
-                    $output->writeln("Pull request failed... Please try again");
-            }
-        }
-    }
-}
diff --git a/plugins/LanguagesManager/Commands/FetchTranslations.php b/plugins/LanguagesManager/Commands/FetchTranslations.php
index e69fd49761..87dd5076f8 100644
--- a/plugins/LanguagesManager/Commands/FetchTranslations.php
+++ b/plugins/LanguagesManager/Commands/FetchTranslations.php
@@ -12,7 +12,7 @@ namespace Piwik\Plugins\LanguagesManager\Commands;
 use Piwik\Container\StaticContainer;
 use Piwik\Exception\AuthenticationFailedException;
 use Piwik\Plugins\LanguagesManager\API as LanguagesManagerApi;
-use Piwik\Translation\Transifex\API;
+use Piwik\Translation\Weblate\API;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -22,18 +22,16 @@ use Symfony\Component\Console\Output\OutputInterface;
  */
 class FetchTranslations extends TranslationBase
 {
-    const DOWNLOAD_PATH = '/transifex';
+    const DOWNLOAD_PATH = '/weblate';
 
     protected function configure()
     {
         $path = StaticContainer::get('path.tmp') . self::DOWNLOAD_PATH;
 
         $this->setName('translations:fetch')
-             ->setDescription('Fetches translations files from Transifex to ' . $path)
-             ->addOption('username', 'u', InputOption::VALUE_OPTIONAL, 'Transifex username')
-             ->addOption('password', 'p', InputOption::VALUE_OPTIONAL, 'Transifex password')
-             ->addOption('lastupdate', 'l', InputOption::VALUE_OPTIONAL, 'Last time update ran', time()-30*24*3600)
-             ->addOption('slug', 's', InputOption::VALUE_OPTIONAL, 'project slug on transifex', 'matomo')
+             ->setDescription('Fetches translations files from Weblate to ' . $path)
+             ->addOption('token', 't', InputOption::VALUE_OPTIONAL, 'Weblate API token')
+             ->addOption('slug', 's', InputOption::VALUE_OPTIONAL, 'project slug on weblate', 'matomo')
              ->addOption('plugin', 'r', InputOption::VALUE_OPTIONAL, 'Plugin to update');
     }
 
@@ -41,37 +39,35 @@ class FetchTranslations extends TranslationBase
     {
         $output->setDecorated(true);
 
-        $username = $input->getOption('username');
-        $password = $input->getOption('password');
+        $apiToken = $input->getOption('token');
         $plugin = $input->getOption('plugin');
-        $lastUpdate = $input->getOption('lastupdate');
         $slug = $input->getOption('slug');
 
-        $resource = 'matomo-'. ($plugin ? 'plugin-'.strtolower($plugin) : 'base');
+        $resource = $plugin ? 'plugin-'.strtolower($plugin) : 'matomo-base';
 
-        $transifexApi = new API($username, $password, $slug);
+        $weblateApi = new API($apiToken, $slug);
 
         // remove all existing translation files in download path
         $files = glob($this->getDownloadPath() . DIRECTORY_SEPARATOR . '*.json');
         array_map('unlink', $files);
 
-        if (!$transifexApi->resourceExists($resource)) {
-            $output->writeln("Skipping resource $resource as it doesn't exist on Transifex");
+        if (!$weblateApi->resourceExists($resource)) {
+            $output->writeln("Skipping resource $resource as it doesn't exist on Weblate");
             return;
         }
 
-        $output->writeln("Fetching translations from Transifex for resource $resource");
+        $output->writeln("Fetching translations from Weblate for resource $resource");
 
         try {
-            $languages = $transifexApi->getAvailableLanguageCodes();
+            $languages = $weblateApi->getAvailableLanguageCodes();
 
             if (!empty($plugin)) {
                 $languages = array_filter($languages, function ($language) {
-                    return LanguagesManagerApi::getInstance()->isLanguageAvailable(str_replace('_', '-', strtolower($language)));
+                    return LanguagesManagerApi::getInstance()->isLanguageAvailable(str_replace('_', '-', strtolower($language)), true);
                 });
             }
         } catch (AuthenticationFailedException $e) {
-            $availableLanguages = LanguagesManagerApi::getInstance()->getAvailableLanguageNames();
+            $availableLanguages = LanguagesManagerApi::getInstance()->getAvailableLanguageNames(true);
 
             $languageCodes = array();
             foreach ($availableLanguages as $languageInfo) {
@@ -96,20 +92,9 @@ class FetchTranslations extends TranslationBase
 
         $progress->start();
 
-        $statistics = $transifexApi->getStatistics($resource);
-
         foreach ($languages as $language) {
             try {
-                // if we have modification date given from statistics api compare it with given last update time to ignore not update resources
-                if (LanguagesManagerApi::getInstance()->isLanguageAvailable(str_replace('_', '-', strtolower($language))) && isset($statistics->$language)) {
-                    $lastupdated = strtotime($statistics->$language->last_update);
-                    if ($lastUpdate > $lastupdated) {
-                        $progress->advance();
-                        continue;
-                    }
-                }
-
-                $translations = $transifexApi->getTranslations($resource, $language, true);
+                $translations = $weblateApi->getTranslations($resource, $language, true);
                 file_put_contents($this->getDownloadPath() . DIRECTORY_SEPARATOR . str_replace('_', '-', strtolower($language)) . '.json', $translations);
             } catch (\Exception $e) {
                 $output->writeln("Error fetching language file $language: " . $e->getMessage());
diff --git a/plugins/LanguagesManager/Commands/LanguageCodes.php b/plugins/LanguagesManager/Commands/LanguageCodes.php
index 79129107a9..323aa0febb 100644
--- a/plugins/LanguagesManager/Commands/LanguageCodes.php
+++ b/plugins/LanguagesManager/Commands/LanguageCodes.php
@@ -11,6 +11,7 @@ namespace Piwik\Plugins\LanguagesManager\Commands;
 
 use Piwik\Plugins\LanguagesManager\API;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 /**
@@ -20,12 +21,13 @@ class LanguageCodes extends TranslationBase
     protected function configure()
     {
         $this->setName('translations:languagecodes')
+             ->addOption('all', 'a', InputOption::VALUE_NONE, 'Displays all languages (ignores language configuration)')
              ->setDescription('Shows available language codes');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $languages = API::getInstance()->getAvailableLanguageNames();
+        $languages = API::getInstance()->getAvailableLanguageNames($input->getOption('all'));
 
         $languageCodes = array();
         foreach ($languages as $languageInfo) {
diff --git a/plugins/LanguagesManager/Commands/LanguageInfo.php b/plugins/LanguagesManager/Commands/LanguageInfo.php
index 5571db8cc3..ef38f3c2a8 100644
--- a/plugins/LanguagesManager/Commands/LanguageInfo.php
+++ b/plugins/LanguagesManager/Commands/LanguageInfo.php
@@ -11,6 +11,7 @@ namespace Piwik\Plugins\LanguagesManager\Commands;
 
 use Piwik\Plugins\LanguagesManager\API;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 /**
@@ -20,12 +21,13 @@ class LanguageInfo extends TranslationBase
     protected function configure()
     {
         $this->setName('translations:languageinfo')
+             ->addOption('all', 'a', InputOption::VALUE_NONE, 'Displays all languages (ignores language configuration)')
              ->setDescription('Shows available languages info');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $languages = API::getInstance()->getAvailableLanguagesInfo();
+        $languages = API::getInstance()->getAvailableLanguagesInfo(true, $input->getOption('all'));
 
         foreach ($languages as $languageInfo) {
             $output->writeln($languageInfo['code'].'|' . $languageInfo['english_name'] . '|' . $languageInfo['percentage_complete']);
diff --git a/plugins/LanguagesManager/Commands/LanguageNames.php b/plugins/LanguagesManager/Commands/LanguageNames.php
index 7e41d5057a..d30c81141e 100644
--- a/plugins/LanguagesManager/Commands/LanguageNames.php
+++ b/plugins/LanguagesManager/Commands/LanguageNames.php
@@ -11,6 +11,7 @@ namespace Piwik\Plugins\LanguagesManager\Commands;
 
 use Piwik\Plugins\LanguagesManager\API;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 /**
@@ -20,12 +21,13 @@ class LanguageNames extends TranslationBase
     protected function configure()
     {
         $this->setName('translations:languagenames')
+             ->addOption('all', 'a', InputOption::VALUE_NONE, 'Displays all languages (ignores language configuration)')
              ->setDescription('Shows available language names');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $languages = API::getInstance()->getAvailableLanguageNames();
+        $languages = API::getInstance()->getAvailableLanguageNames($input->getOption('all'));
 
         $languageNames = array();
         foreach ($languages as $languageInfo) {
diff --git a/plugins/LanguagesManager/Commands/SetTranslations.php b/plugins/LanguagesManager/Commands/SetTranslations.php
index 6790d40158..4952e62962 100644
--- a/plugins/LanguagesManager/Commands/SetTranslations.php
+++ b/plugins/LanguagesManager/Commands/SetTranslations.php
@@ -43,7 +43,7 @@ class SetTranslations extends TranslationBase
         $languageCode = $input->getOption('code');
         $filename     = $input->getOption('file');
 
-        $languageCodes = (new API())->getAvailableLanguages();
+        $languageCodes = (new API())->getAvailableLanguages(true);
 
         if (empty($languageCode) || !in_array($languageCode, $languageCodes)) {
             $languageCode = $dialog->askAndValidate($output, 'Please provide a valid language code: ', function ($code) use ($languageCodes) {
diff --git a/plugins/LanguagesManager/Commands/Update.php b/plugins/LanguagesManager/Commands/Update.php
index 7fe598985d..4103b27f2b 100644
--- a/plugins/LanguagesManager/Commands/Update.php
+++ b/plugins/LanguagesManager/Commands/Update.php
@@ -28,10 +28,8 @@ class Update extends TranslationBase
     {
         $this->setName('translations:update')
             ->setDescription('Updates translation files')
-            ->addOption('force', 'f', InputOption::VALUE_NONE, 'Force update of all language files')
-            ->addOption('username', 'u', InputOption::VALUE_OPTIONAL, 'Transifex username')
-            ->addOption('password', 'p', InputOption::VALUE_OPTIONAL, 'Transifex password')
-            ->addOption('slug', 's', InputOption::VALUE_OPTIONAL, 'Transifex project slug')
+            ->addOption('token', 't', InputOption::VALUE_OPTIONAL, 'Weblate API token')
+            ->addOption('slug', 's', InputOption::VALUE_OPTIONAL, 'Weblate project slug')
             ->addOption('all', 'a', InputOption::VALUE_NONE, 'Force to update all plugins (even non core). Can not be used with plugin option')
             ->addOption('plugin', 'P', InputOption::VALUE_OPTIONAL, 'optional name of plugin to update translations for');
     }
@@ -45,7 +43,7 @@ class Update extends TranslationBase
         /** @var DialogHelper $dialog */
         $dialog = $this->getHelperSet()->get('dialog');
 
-        $languages = API::getInstance()->getAvailableLanguageNames();
+        $languages = API::getInstance()->getAvailableLanguageNames(true);
 
         $languageCodes = array();
         foreach ($languages as $languageInfo) {
@@ -63,8 +61,6 @@ class Update extends TranslationBase
         if (empty($plugin)) {
             $pluginList = $forceAllPlugins ? self::getAllPlugins() : self::getPluginsInCore();
             array_unshift($pluginList, '');
-        } else {
-            $input->setOption('force', true); // force plugin only updates
         }
 
         foreach ($pluginList as $plugin) {
@@ -220,34 +216,11 @@ class Update extends TranslationBase
         $command = $this->getApplication()->find('translations:fetch');
         $arguments = array(
             'command' => 'translations:fetch',
-            '--username' => $input->getOption('username'),
-            '--password' => $input->getOption('password'),
+            '--token'    => $input->getOption('token'),
             '--slug'     => $input->getOption('slug'),
             '--plugin'   => $plugin
         );
 
-        if ($input->getOption('force')) {
-            $arguments['--lastupdate'] = 1;
-        } else {
-            $lastModDate = strtotime('2015-01-04 00:00:00'); // date of initial transifex setup
-            try {
-                // try to find the language file (of given plugin) with the newest modification date in git log
-                $path = ($plugin ? 'plugins/' . $plugin . '/' : '') . 'lang';
-                $files = explode("\n", trim(shell_exec('git ls-tree -r --name-only HEAD ' . $path)));
-
-                foreach ($files as $file) {
-                    $fileModDate = shell_exec('git log -1 --format="%at" -- ' . $file);
-                    if (basename($file) != 'en.json' && $fileModDate > $lastModDate) {
-                        $lastModDate = $fileModDate;
-                    }
-                }
-            } catch (\Exception $e) {
-            }
-
-            if ($lastModDate != 0) {
-                $arguments['--lastupdate'] = $lastModDate;
-            }
-        }
         $inputObject = new ArrayInput($arguments);
         $inputObject->setInteractive($input->isInteractive());
         $command->run($inputObject, $output);
diff --git a/plugins/LanguagesManager/Commands/Validate.php b/plugins/LanguagesManager/Commands/Validate.php
index 07082ba730..c14273c0c2 100644
--- a/plugins/LanguagesManager/Commands/Validate.php
+++ b/plugins/LanguagesManager/Commands/Validate.php
@@ -24,9 +24,8 @@ class Validate extends TranslationBase
     {
         $this->setName('translations:validate')
             ->setDescription('Validates translation files')
-            ->addOption('username', 'u', InputOption::VALUE_OPTIONAL, 'Transifex username')
-            ->addOption('password', 'p', InputOption::VALUE_OPTIONAL, 'Transifex password')
-            ->addOption('slug', 's', InputOption::VALUE_OPTIONAL, 'Transifex project slug')
+            ->addOption('token', 't', InputOption::VALUE_OPTIONAL, 'Weblate API token')
+            ->addOption('slug', 's', InputOption::VALUE_OPTIONAL, 'Weblate project slug')
             ->addOption('all', 'a', InputOption::VALUE_NONE, 'Force to update all plugins (even non core). Can not be used with plugin option')
             ->addOption('plugin', 'P', InputOption::VALUE_OPTIONAL, 'optional name of plugin to update translations for');
     }
@@ -37,7 +36,7 @@ class Validate extends TranslationBase
 
         $start = microtime(true);
 
-        $languages = API::getInstance()->getAvailableLanguageNames();
+        $languages = API::getInstance()->getAvailableLanguageNames(true);
 
         $languageCodes = array();
         foreach ($languages as $languageInfo) {
@@ -127,12 +126,10 @@ class Validate extends TranslationBase
 
         $command = $this->getApplication()->find('translations:fetch');
         $arguments = array(
-            'command' => 'translations:fetch',
-            '--username' => $input->getOption('username'),
-            '--password' => $input->getOption('password'),
-            '--slug'     => $input->getOption('slug'),
-            '--plugin'   => $plugin,
-            '--lastupdate' => 1
+            'command'  => 'translations:fetch',
+            '--token'  => $input->getOption('token'),
+            '--slug'   => $input->getOption('slug'),
+            '--plugin' => $plugin
         );
 
         $inputObject = new ArrayInput($arguments);
diff --git a/plugins/LanguagesManager/tests/Integration/LanguagesManagerTest.php b/plugins/LanguagesManager/tests/Integration/LanguagesManagerTest.php
index 5b776c7d24..48456239dc 100644
--- a/plugins/LanguagesManager/tests/Integration/LanguagesManagerTest.php
+++ b/plugins/LanguagesManager/tests/Integration/LanguagesManagerTest.php
@@ -107,7 +107,7 @@ class LanguagesManagerTest extends \PHPUnit\Framework\TestCase
             $translationWriter->saveTemporary();
             $this->markTestSkipped(implode("\n", $translationWriter->getFilterMessages()) . "\n"
                 . 'Translation file errors detected in ' . $language . "...\n"
-                . "To synchronise the language files with the english strings, you can manually edit the language files or run the following command may work if you have access to Transifex: \n"
+                . "To synchronise the language files with the english strings, you can manually edit the language files or run the following command may work if you have access to Weblate: \n"
                 . "$ ./console translations:update [--plugin=XYZ] \n"
             );
         }
diff --git a/plugins/LogViewer b/plugins/LogViewer
-Subproject 9c893d29c64f5ca60ed093cf553b897c83a755c
+Subproject 3204e829f960ccfc0a9ddb4b509dea5a4f6529e
diff --git a/plugins/LoginLdap b/plugins/LoginLdap
-Subproject 3fe227f67469ab4e45b61c656db85cd56458d81
+Subproject ca493a3b1147db7626adac4f1baec0e2b0315d8
diff --git a/plugins/MarketingCampaignsReporting b/plugins/MarketingCampaignsReporting
-Subproject c1bfe8f24c30283c2a8e88dcac15ea68fb91d23
+Subproject 67ce24997dad02146dd3333d609eddb3bee314c
diff --git a/plugins/Provider b/plugins/Provider
-Subproject f46d3a6cc4a4715e962f3f893a822d7125d9bd8
+Subproject b04a23f50c6e050904ad83e927a49dfe10b8134
diff --git a/plugins/QueuedTracking b/plugins/QueuedTracking
-Subproject d91ac1a0855ea0ad7c113111364e4f3b4076c47
+Subproject 277f5d0419cff1d560d0274f8545b1088e61f52
diff --git a/plugins/SecurityInfo b/plugins/SecurityInfo
-Subproject aecd4763c9065cde6aafd4f972701831a6f4fde
+Subproject 9164ac73a2cb9c7ef03dc71eb064c455044c462
diff --git a/plugins/TagManager b/plugins/TagManager
-Subproject 629a4132a3324fc1d6ecfa67b1790fef6b01dfc
+Subproject 08b3f02cae322525c001b5dac17b898581a9739
diff --git a/plugins/TasksTimetable b/plugins/TasksTimetable
-Subproject 55914c9a44a1cf35278ad94fb8eadb128fa242d
+Subproject 7287cd7ea0c7320a1f2345923d8794d8f5f2999
diff --git a/plugins/TrackingSpamPrevention b/plugins/TrackingSpamPrevention
-Subproject 86dc97a5b1ba2ff3eacda3cd08ff96ff2820c54
+Subproject 3051e8eb481cf89ce6a1811f805247a6c596f4a
diff --git a/plugins/VisitorGenerator b/plugins/VisitorGenerator
-Subproject 3bb5dcf98aa68de38a9fb1450977aa58fef057d
+Subproject 8cc8882aeee4ea345069d7274ac2e3e913ea958