refs #7567 - use transifex API to fetch translations

3 files changed, 138 insertions, 245 deletions

diff --git a/plugins/LanguagesManager/Commands/FetchFromOTrance.php b/plugins/LanguagesManager/Commands/FetchFromOTrance.php
deleted file mode 100644
index 8802209ced..0000000000
--- a/plugins/LanguagesManager/Commands/FetchFromOTrance.php
+++ /dev/null
@@ -1,182 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- *
- */
-
-namespace Piwik\Plugins\LanguagesManager\Commands;
-
-use Piwik\Container\StaticContainer;
-use Piwik\Unzip;
-use Symfony\Component\Console\Helper\DialogHelper;
-use Symfony\Component\Console\Helper\ProgressHelper;
-use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Console\Input\InputOption;
-use Symfony\Component\Console\Output\OutputInterface;
-
-/**
- */
-class FetchFromOTrance extends TranslationBase
-{
-    const DOWNLOAD_PATH = '/oTrance';
-
-    protected function configure()
-    {
-        $path = StaticContainer::get('path.tmp') . self::DOWNLOAD_PATH;
-
-        $this->setName('translations:fetch')
-             ->setDescription('Fetches translations files from oTrance to ' . $path)
-             ->addOption('username', 'u', InputOption::VALUE_OPTIONAL, 'oTrance username')
-             ->addOption('password', 'p', InputOption::VALUE_OPTIONAL, 'oTrance password')
-             ->addOption('keep-english', 'k', InputOption::VALUE_NONE, 'keep english file');
-    }
-
-    protected function execute(InputInterface $input, OutputInterface $output)
-    {
-        $output->writeln("Starting to fetch latest language pack");
-
-        /** @var DialogHelper $dialog */
-        $dialog = $this->getHelperSet()->get('dialog');
-
-        $cookieFile = self::getDownloadPath() . DIRECTORY_SEPARATOR . 'cookie.txt';
-        @unlink($cookieFile);
-
-        $username = $input->getOption('username');
-        $password = $input->getOption('password');
-
-        while (!file_exists($cookieFile)) {
-            if (empty($username)) {
-                $username = $dialog->ask($output, 'What is your oTrance username? ');
-            }
-
-            if (empty($password)) {
-                $password = $dialog->askHiddenResponse($output, 'What is your oTrance password? ');
-            }
-
-            // send login request to oTrance and save the login cookie
-            $curl = curl_init('http://translations.piwik.org/public/index/login');
-            curl_setopt($curl, CURLOPT_POSTFIELDS, sprintf("user=%s&pass=%s&autologin=1", $username, $password));
-            curl_setopt($curl, CURLOPT_COOKIEJAR, $cookieFile);
-            curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
-            curl_exec($curl);
-            curl_close($curl);
-
-            if (strpos(file_get_contents($cookieFile), 'oTranCe_autologin') !== false) {
-                break;
-            }
-
-            $username = null;
-            $password = null;
-            @unlink($cookieFile);
-            $output->writeln("Invalid oTrance credentials. Please try again...");
-        }
-
-        // send request to create a new download package using the cookie file
-        $createNewPackage = true;
-        if ($input->isInteractive()) {
-            $createNewPackage = $dialog->askConfirmation($output, 'Shall we create a new language pack? ');
-        }
-
-        if ($createNewPackage) {
-
-            $curl = curl_init('http://translations.piwik.org/public/export/update.all');
-            curl_setopt($curl, CURLOPT_COOKIEFILE, $cookieFile);
-            curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
-            curl_exec($curl);
-            curl_close($curl);
-        }
-
-        // request download page to search for available packages
-        $curl = curl_init('http://translations.piwik.org/public/downloads/');
-        curl_setopt($curl, CURLOPT_COOKIEFILE, $cookieFile);
-        curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
-        $response = curl_exec($curl);
-        curl_close($curl);
-
-        preg_match_all('/language\_pack\-[0-9]{8}\-[0-9]{6}\.tar\.gz/i', $response, $matches);
-
-        if (empty($matches[0])) {
-
-            $output->writeln("No packages found for download. Please try again.");
-            return;
-        }
-
-        $downloadPackage = array_shift($matches[0]);
-
-        $continueWithPackage = true;
-        if ($input->isInteractive()) {
-            $continueWithPackage = $dialog->askConfirmation($output, "Found language pack $downloadPackage. Proceed? ");
-        }
-
-        if (!$continueWithPackage) {
-
-            $output->writeln('Aborted.');
-            return;
-        }
-
-        // download language pack
-        $packageHandle = fopen(self::getDownloadPath() . DIRECTORY_SEPARATOR . 'language_pack.tar.gz', 'w');
-        $curl = curl_init('http://translations.piwik.org/public/downloads/download/file/'.$downloadPackage);
-        curl_setopt($curl, CURLOPT_COOKIEFILE, self::getDownloadPath() . DIRECTORY_SEPARATOR . 'cookie.txt');
-        curl_setopt($curl, CURLOPT_FILE, $packageHandle);
-        curl_exec($curl);
-        curl_close($curl);
-
-        @unlink($cookieFile);
-
-        $output->writeln("Extracting package...");
-
-        $unzipper = Unzip::factory('tar.gz', self::getDownloadPath() . DIRECTORY_SEPARATOR . 'language_pack.tar.gz');
-        $unzipper->extract(self::getDownloadPath());
-
-        if (!$input->getOption('keep-english')) {
-            @unlink(self::getDownloadPath() . DIRECTORY_SEPARATOR . 'en.php');
-            @unlink(self::getDownloadPath() . DIRECTORY_SEPARATOR . 'en.json');
-        }
-        @unlink(self::getDownloadPath() . DIRECTORY_SEPARATOR . 'language_pack.tar.gz');
-
-        $filesToConvert = _glob(self::getDownloadPath() . DIRECTORY_SEPARATOR . '*.php');
-
-        $output->writeln("Converting downloaded php files to json");
-
-        /** @var ProgressHelper $progress */
-        $progress = $this->getHelperSet()->get('progress');
-
-        $progress->start($output, count($filesToConvert));
-        foreach ($filesToConvert as $filename) {
-
-            require_once $filename;
-            $basename = explode(".", basename($filename));
-            $nested = array();
-            foreach ($translations as $key => $value) {
-                list($plugin, $nkey) = explode("_", $key, 2);
-                $nested[$plugin][$nkey] = $value;
-            }
-            $translations = $nested;
-            $data = json_encode($translations, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT);
-            $newFile = sprintf("%s/%s.json", self::getDownloadPath(), $basename[0]);
-            file_put_contents($newFile, $data);
-            @unlink($filename);
-
-            $progress->advance();
-        }
-
-        $progress->finish();
-
-        $output->writeln("Finished fetching new language files from oTrance");
-    }
-
-    public static function getDownloadPath()
-    {
-        $path = StaticContainer::get('path.tmp') . self::DOWNLOAD_PATH;
-
-        if (!is_dir($path)) {
-            mkdir($path);
-        }
-
-        return $path;
-    }
-}
diff --git a/plugins/LanguagesManager/Commands/FetchFromTransifex.php b/plugins/LanguagesManager/Commands/FetchFromTransifex.php
new file mode 100644
index 0000000000..aac1d10b2b
--- /dev/null
+++ b/plugins/LanguagesManager/Commands/FetchFromTransifex.php
@@ -0,0 +1,79 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\LanguagesManager\Commands;
+
+use Piwik\Container\StaticContainer;
+use Piwik\Translation\Transifex\API;
+use Symfony\Component\Console\Helper\ProgressHelper;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ */
+class FetchFromTransifex extends TranslationBase
+{
+    const DOWNLOAD_PATH = '/transifex';
+
+    protected function configure()
+    {
+        $path = StaticContainer::get('path.tmp') . self::DOWNLOAD_PATH;
+
+        $this->setName('translations:fetch')
+             ->setDescription('Fetches translations files from oTrance to ' . $path)
+             ->addOption('username', 'u', InputOption::VALUE_OPTIONAL, 'Transifex username')
+             ->addOption('password', 'p', InputOption::VALUE_OPTIONAL, 'Transifex password')
+             ->addOption('plugin', 'r', InputOption::VALUE_OPTIONAL, 'Plugin to update');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $username = $input->getOption('username');
+        $password = $input->getOption('password');
+        $plugin = $input->getOption('plugin');
+
+        $resource = 'piwik-'. ($plugin ? 'plugin-'.strtolower($plugin) : 'base');
+
+        $output->writeln("Fetching translations from Transifex for resource $resource");
+
+        $transifexApi = new API($username, $password);
+
+        $languages = $transifexApi->getAvailableLanguageCodes();
+
+        /** @var ProgressHelper $progress */
+        $progress = $this->getHelperSet()->get('progress');
+
+        $progress->start($output, count($languages));
+
+        foreach ($languages as $language) {
+            try {
+                $translations = $transifexApi->getTranslations($resource, $language, true);
+                file_put_contents($this->getDownloadPath() . DIRECTORY_SEPARATOR . str_replace('_', '-', strtolower($language)) . '.json', $translations);
+            } catch (\Exception $e) {
+                $output->writeln("Error fetching language file $language: " . $e->getMessage());
+            }
+            $progress->advance();
+        }
+
+        $progress->finish();
+        $output->writeln("Finished fetching new language files from Transifex");
+    }
+
+    public static function getDownloadPath()
+    {
+        $path = StaticContainer::get('path.tmp') . self::DOWNLOAD_PATH;
+
+        if (!is_dir($path)) {
+            mkdir($path);
+        }
+
+        return $path;
+    }
+}
diff --git a/plugins/LanguagesManager/Commands/Update.php b/plugins/LanguagesManager/Commands/Update.php
index 4d0c28bd76..f09188e83a 100644
--- a/plugins/LanguagesManager/Commands/Update.php
+++ b/plugins/LanguagesManager/Commands/Update.php
@@ -26,8 +26,8 @@ class Update extends TranslationBase
     {
         $this->setName('translations:update')
             ->setDescription('Updates translation files')
-            ->addOption('username', 'u', InputOption::VALUE_OPTIONAL, 'oTrance username')
-            ->addOption('password', 'p', InputOption::VALUE_OPTIONAL, 'oTrance password')
+            ->addOption('username', 'u', InputOption::VALUE_OPTIONAL, 'Transifex username')
+            ->addOption('password', 'p', InputOption::VALUE_OPTIONAL, 'Transifex password')
             ->addOption('plugin', 'P', InputOption::VALUE_OPTIONAL, 'optional name of plugin to update translations for');
     }
 
@@ -36,16 +36,6 @@ class Update extends TranslationBase
         /** @var DialogHelper $dialog */
         $dialog = $this->getHelperSet()->get('dialog');
 
-        $command = $this->getApplication()->find('translations:fetch');
-        $arguments = array(
-            'command'    => 'translations:fetch',
-            '--username' => $input->getOption('username'),
-            '--password' => $input->getOption('password')
-        );
-        $inputObject = new ArrayInput($arguments);
-        $inputObject->setInteractive($input->isInteractive());
-        $command->run($inputObject, $output);
-
         $languages = API::getInstance()->getAvailableLanguageNames();
 
         $languageCodes = array();
@@ -55,78 +45,70 @@ class Update extends TranslationBase
 
         $plugin = $input->getOption('plugin');
 
-        $files = _glob(FetchFromOTrance::getDownloadPath() . DIRECTORY_SEPARATOR . '*.json');
-
         $output->writeln("Starting to import new language files");
 
         if (!$input->isInteractive()) {
             $output->writeln("(!) Non interactive mode: New languages will be skipped");
         }
 
-        /** @var ProgressHelper $progress */
-        $progress = $this->getHelperSet()->get('progress');
+        $pluginList = array($plugin);
+        if (empty($plugin)) {
+            $pluginList = self::getPluginsInCore();
+            array_unshift($pluginList, '');
+        }
 
-        $progress->start($output, count($files));
+        foreach ($pluginList as $plugin) {
 
-        foreach ($files as $filename) {
+            // fetch base or specific plugin
+            $this->fetchTranslations($input, $output, $plugin);
 
-            $progress->advance();
+            $files = _glob(FetchFromTransifex::getDownloadPath() . DIRECTORY_SEPARATOR . '*.json');
 
-            $code = basename($filename, '.json');
+            /** @var ProgressHelper $progress */
+            $progress = $this->getHelperSet()->get('progress');
 
-            if (!in_array($code, $languageCodes)) {
+            $progress->start($output, count($files));
 
-                if (!empty($plugin)) {
+            foreach ($files as $filename) {
 
-                    continue; # never create a new language for plugin only
-                }
+                $progress->advance();
 
-                $createNewFile = false;
-                if ($input->isInteractive()) {
-                    $createNewFile = $dialog->askConfirmation($output, "\nLanguage $code does not exist. Should it be added? ", false);
-                }
+                $code = basename($filename, '.json');
 
-                if (!$createNewFile) {
+                if (!in_array($code, $languageCodes)) {
 
-                    continue; # do not create a new file for the language
-                }
+                    if (!empty($plugin)) {
+                        continue; # never create a new language for plugin only
+                    }
 
-                @touch(PIWIK_DOCUMENT_ROOT . DIRECTORY_SEPARATOR . 'lang' . DIRECTORY_SEPARATOR . $code . '.json');
-                API::unsetInstance(); // unset language manager instance, so valid names are refetched
-            }
+                    $createNewFile = false;
+                    if ($input->isInteractive()) {
+                        $createNewFile = $dialog->askConfirmation($output, "\nLanguage $code does not exist. Should it be added? ", false);
+                    }
 
-            $command = $this->getApplication()->find('translations:set');
-            $arguments = array(
-                'command'  => 'translations:set',
-                '--code'   => $code,
-                '--file'   => $filename,
-                '--plugin' => $plugin
-            );
-            $inputObject = new ArrayInput($arguments);
-            $inputObject->setInteractive($input->isInteractive());
-            $command->run($inputObject, new NullOutput());
-
-            // update core modules that aren't in their own repo
-            if (empty($plugin)) {
-
-                foreach (self::getPluginsInCore() as $pluginName) {
-
-                    // update translation files
-                    $command = $this->getApplication()->find('translations:set');
-                    $arguments = array(
-                        'command'  => 'translations:set',
-                        '--code'   => $code,
-                        '--file'   => $filename,
-                        '--plugin' => $pluginName
-                    );
-                    $inputObject = new ArrayInput($arguments);
-                    $inputObject->setInteractive($input->isInteractive());
-                    $command->run($inputObject, new NullOutput());
+                    if (!$createNewFile) {
+                        continue; # do not create a new file for the language
+                    }
+
+                    @touch(PIWIK_DOCUMENT_ROOT . DIRECTORY_SEPARATOR . 'lang' . DIRECTORY_SEPARATOR . $code . '.json');
+                    API::unsetInstance(); // unset language manager instance, so valid names are refetched
                 }
+
+                $command = $this->getApplication()->find('translations:set');
+                $arguments = array(
+                    'command' => 'translations:set',
+                    '--code' => $code,
+                    '--file' => $filename,
+                    '--plugin' => $plugin
+                );
+                $inputObject = new ArrayInput($arguments);
+                $inputObject->setInteractive($input->isInteractive());
+                $command->run($inputObject, new NullOutput());
             }
+
+            $progress->finish();
         }
 
-        $progress->finish();
         $output->writeln("Finished.");
     }
 
@@ -154,7 +136,7 @@ class Update extends TranslationBase
         $pluginsNotInCore = array_merge($submodulePlugins, $newPlugins);
 
         $pluginsWithTranslations = glob(sprintf('%s/plugins/*/lang/en.json', PIWIK_INCLUDE_PATH));
-        $pluginsWithTranslations = array_map(function($elem){
+        $pluginsWithTranslations = array_map(function ($elem) {
             return str_replace(array(sprintf('%s/plugins/', PIWIK_INCLUDE_PATH), '/lang/en.json'), '', $elem);
         }, $pluginsWithTranslations);
 
@@ -162,4 +144,18 @@ class Update extends TranslationBase
 
         return $pluginsInCore;
     }
+
+    protected function fetchTranslations($input, $output, $plugin)
+    {
+        $command = $this->getApplication()->find('translations:fetch');
+        $arguments = array(
+            'command' => 'translations:fetch',
+            '--username' => $input->getOption('username'),
+            '--password' => $input->getOption('password'),
+            '--plugin' => $plugin
+        );
+        $inputObject = new ArrayInput($arguments);
+        $inputObject->setInteractive($input->isInteractive());
+        $command->run($inputObject, $output);
+    }
 }