[tools] Upgrade twine to version 0.5.

15 files changed, 222 insertions, 76 deletions

diff --git a/tools/twine/Gemfile b/tools/twine/Gemfile
index e45e65f871..3be9c3cd81 100644
--- a/tools/twine/Gemfile
+++ b/tools/twine/Gemfile
@@ -1,2 +1,2 @@
-source :rubygems
+source "https://rubygems.org"
 gemspec
diff --git a/tools/twine/README.md b/tools/twine/README.md
index 9f307f0747..80e0ec8976 100644
--- a/tools/twine/README.md
+++ b/tools/twine/README.md
@@ -76,6 +76,7 @@ Twine currently supports the following formats for outputting strings:
 * [Android String Resources][androidstrings] (format: android)
 * [Gettext PO Files][gettextpo] (format: gettext)
 * [jquery-localize Language Files][jquerylocalize] (format: jquery)
+* [Django PO Files][djangopo] (format: django)
 
 If you would like to enable twine to create language files in another format, create an appropriate formatter in `lib/twine/formatters`.
 
@@ -166,12 +167,14 @@ Now, whenever you build your application, Xcode will automatically invoke Twine
 
 Many thanks to all of the contributors to the Twine project, including:
 
+* [Blake Watters](https://github.com/blakewatters)
 * [Ishitoya Kentaro](https://github.com/kent013)
 * [Joseph Earl](https://github.com/JosephEarl)
 * [Kevin Everets](https://github.com/keverets)
 * [Kevin Wood](https://github.com/kwood)
 * [Mohammad Hejazi](https://github.com/MohammadHejazi)
 * [Robert Guo](http://www.robertguo.me/)
+* [Shai Shamir](https://github.com/pichirichi)
 
 
 [rubyzip]: http://rubygems.org/gems/rubyzip
@@ -181,3 +184,4 @@ Many thanks to all of the contributors to the Twine project, including:
 [androidstrings]: http://developer.android.com/guide/topics/resources/string-resource.html
 [gettextpo]: http://www.gnu.org/savannah-checkouts/gnu/gettext/manual/html_node/PO-Files.html
 [jquerylocalize]: https://github.com/coderifous/jquery-localize
+[djangopo]: https://docs.djangoproject.com/en/dev/topics/i18n/translation/
diff --git a/tools/twine/lib/twine/formatters.rb b/tools/twine/lib/twine/formatters.rb
index 756df4e86d..50ba4805ff 100644
--- a/tools/twine/lib/twine/formatters.rb
+++ b/tools/twine/lib/twine/formatters.rb
@@ -4,9 +4,10 @@ require 'twine/formatters/apple'
 require 'twine/formatters/flash'
 require 'twine/formatters/gettext'
 require 'twine/formatters/jquery'
+require 'twine/formatters/django'
 
 module Twine
   module Formatters
-    FORMATTERS = [Formatters::Apple, Formatters::Android, Formatters::Gettext, Formatters::JQuery, Formatters::Flash]
+    FORMATTERS = [Formatters::Apple, Formatters::Android, Formatters::Gettext, Formatters::JQuery, Formatters::Flash, Formatters::Django]
   end
 end
diff --git a/tools/twine/lib/twine/formatters/abstract.rb b/tools/twine/lib/twine/formatters/abstract.rb
index f19c8e3506..c9f2df9770 100644
--- a/tools/twine/lib/twine/formatters/abstract.rb
+++ b/tools/twine/lib/twine/formatters/abstract.rb
@@ -14,42 +14,8 @@ module Twine
       end
 
       def iosify_substitutions(str)
-        # 1) use "@" instead of "s" for substituting strings
+        # use "@" instead of "s" for substituting strings
         str.gsub!(/%([0-9\$]*)s/, '%\1@')
-
-        # 2) if substitutions are numbered, see if we can remove the numbering safely
-        expectedSub = 1
-        startFound = false
-        foundSub = 0
-        str.each_char do |c|
-          if startFound
-            if c == "%"
-              # this is a literal %, keep moving
-              startFound = false
-            elsif c.match(/\d/)
-              foundSub *= 10
-              foundSub += Integer(c)
-            elsif c == "$"
-              if expectedSub == foundSub
-                # okay to keep going
-                startFound = false
-                expectedSub += 1
-              else
-                # the numbering appears to be important (or non-existent), leave it alone
-                return str
-              end
-            end
-          elsif c == "%"
-            startFound = true
-            foundSub = 0
-          end
-        end
-
-        # if we got this far, then the numbering (if any) is in order left-to-right and safe to remove
-        if expectedSub > 1
-          str.gsub!(/%\d+\$(.)/, '%\1')
-        end
-
         return str
       end
 
@@ -152,14 +118,23 @@ module Twine
         end
 
         file_name = @options[:file_name] || default_file_name
+        langs_written = []
         Dir.foreach(path) do |item|
-          if File.directory?(path + File::SEPARATOR + item)
+          if item == "." or item == ".."
+            next
+          end
+          item = File.join(path, item)
+          if File.directory?(item)
             lang = determine_language_given_path(item)
             if lang
-              write_file(File.join(path, item, file_name), lang)
+              write_file(File.join(item, file_name), lang)
+              langs_written << lang
             end
           end
         end
+        if langs_written.empty?
+          raise Twine::Error.new("Failed to genertate any files: No languages found at #{path}")
+        end
       end
     end
   end
diff --git a/tools/twine/lib/twine/formatters/android.rb b/tools/twine/lib/twine/formatters/android.rb
index ee6fb6865d..92cbcaa5ea 100644
--- a/tools/twine/lib/twine/formatters/android.rb
+++ b/tools/twine/lib/twine/formatters/android.rb
@@ -40,7 +40,7 @@ module Twine
               lang = match[1]
               lang = LANG_CODES.fetch(lang, lang)
               lang.sub!('-r', '-')
-              return lang =~ /land|port|v\d+|sw\d+/ ? nil : lang
+              return lang
             end
           end
         end
@@ -49,7 +49,7 @@ module Twine
       end
 
       def read_file(path, lang)
-        resources_regex = /<resources>(.*)<\/resources>/m
+        resources_regex = /<resources(?:[^>]*)>(.*)<\/resources>/m
         key_regex = /<string name="(\w+)">/
         comment_regex = /<!-- (.*) -->/
         value_regex = /<string name="\w+">(.*)<\/string>/
@@ -71,6 +71,7 @@ module Twine
                   value.gsub!('\\\'', '\'')
                   value.gsub!('\\"', '"')
                   value = iosify_substitutions(value)
+                  value.gsub!(/(\\u0020)*|(\\u0020)*\z/) { |spaces| ' ' * (spaces.length / 6) }
                 else
                   value = ""
                 end
@@ -129,6 +130,8 @@ module Twine
                   value = CGI.escapeHTML(value)
                   #  3) fix substitutions (e.g. %s/%@)
                   value = androidify_substitutions(value)
+                  #  4) replace beginning and end spaces with \0020. Otherwise Android strips them.
+                  value.gsub!(/\A *| *\z/) { |spaces| '\u0020' * spaces.length }
 
                   comment = row.comment
                   if comment
diff --git a/tools/twine/lib/twine/formatters/django.rb b/tools/twine/lib/twine/formatters/django.rb
new file mode 100644
index 0000000000..4a5352b4a1
--- /dev/null
+++ b/tools/twine/lib/twine/formatters/django.rb
@@ -0,0 +1,143 @@
+module Twine
+  module Formatters
+    class Django < Abstract
+      FORMAT_NAME = 'django'
+      EXTENSION = '.po'
+      DEFAULT_FILE_NAME = 'strings.po'
+
+      def self.can_handle_directory?(path)
+      Dir.entries(path).any? { |item| /^.+\.po$/.match(item) }
+      end
+
+      def default_file_name
+        return DEFAULT_FILE_NAME
+      end
+
+      def determine_language_given_path(path)
+          path_arr = path.split(File::SEPARATOR)
+          path_arr.each do |segment|
+              match = /(..)\.po$/.match(segment)
+              if match
+                  return match[1]
+              end
+          end
+
+        return
+      end
+
+      def read_file(path, lang)
+        comment_regex = /#.? *"(.*)"$/
+        key_regex = /msgid *"(.*)"$/
+        value_regex = /msgstr *"(.*)"$/m
+
+        encoding = Twine::Encoding.encoding_for_path(path)
+        sep = nil
+        if !encoding.respond_to?(:encode)
+          # This code is not necessary in 1.9.3 and does not work as it did in 1.8.7.
+          if encoding.end_with? 'LE'
+            sep = "\x0a\x00"
+          elsif encoding.end_with? 'BE'
+            sep = "\x00\x0a"
+          else
+            sep = "\n"
+          end
+        end
+
+        if encoding.index('UTF-16')
+          mode = "rb:#{encoding}"
+        else
+          mode = "r:#{encoding}"
+        end
+
+        File.open(path, mode) do |f|
+          last_comment = nil
+          while line = (sep) ? f.gets(sep) : f.gets
+            if encoding.index('UTF-16')
+              if line.respond_to? :encode!
+                line.encode!('UTF-8')
+              else
+                require 'iconv'
+                line = Iconv.iconv('UTF-8', encoding, line).join
+              end
+            end
+            if @options[:consume_comments]
+               comment_match = comment_regex.match(line)
+               if comment_match
+                   comment = comment_match[1]
+               end
+            else
+                comment = nil
+            end
+            key_match = key_regex.match(line)
+            if key_match
+                key = key_match[1].gsub('\\"', '"')
+            end
+            value_match = value_regex.match(line)
+            if value_match
+                value = value_match[1].gsub(/"\n"/, '').gsub('\\"', '"')
+            end
+
+
+            if key and key.length > 0 and value and value.length > 0
+                set_translation_for_key(key, lang, value)
+                if comment and comment.length > 0 and !comment.start_with?("--------- ")
+                    set_comment_for_key(key, comment)
+                end
+                comment = nil
+            end
+
+          end
+        end
+      end
+
+      def write_file(path, lang)
+        default_lang = @strings.language_codes[0]
+        encoding = @options[:output_encoding] || 'UTF-8'
+        File.open(path, "w:#{encoding}") do |f|
+          f.puts "##\n # Django Strings File\n # Generated by Twine #{Twine::VERSION}\n # Language: #{lang}\n "
+          @strings.sections.each do |section|
+            printed_section = false
+            section.rows.each do |row|
+              if row.matches_tags?(@options[:tags], @options[:untagged])
+                f.puts ''
+                if !printed_section
+                  if section.name && section.name.length > 0
+                    f.print "#--------- #{section.name} ---------#\n\n"
+                  end
+                  printed_section = true
+                end
+
+                basetrans = row.translated_string_for_lang(default_lang)
+
+                key = row.key
+                key = key.gsub('"', '\\\\"')
+
+                value = row.translated_string_for_lang(lang, default_lang)
+                if value
+                  value = value.gsub('"', '\\\\"')
+
+                  comment = row.comment
+
+                  if comment
+                     comment = comment.gsub('"', '\\\\"')
+                  end
+
+                  if comment && comment.length > 0
+                    f.print "#. #{comment} \n"
+                  end
+
+                  if basetrans && basetrans.length > 0
+                    f.print "# base translation: \"#{basetrans}\"\n"
+                  end
+
+                  f.print "msgid \"#{key}\"\n"
+                  f.print "msgstr \"#{value}\"\n"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/tools/twine/lib/twine/formatters/gettext.rb b/tools/twine/lib/twine/formatters/gettext.rb
index 10ba5e44bf..d76c4c3fb2 100644
--- a/tools/twine/lib/twine/formatters/gettext.rb
+++ b/tools/twine/lib/twine/formatters/gettext.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Twine
   module Formatters
     class Gettext < Abstract
diff --git a/tools/twine/lib/twine/formatters/jquery.rb b/tools/twine/lib/twine/formatters/jquery.rb
index 14f12f4f97..404b62b83d 100644
--- a/tools/twine/lib/twine/formatters/jquery.rb
+++ b/tools/twine/lib/twine/formatters/jquery.rb
@@ -47,21 +47,22 @@ module Twine
           raise Twine::Error.new "You must run 'gem install json' in order to read or write jquery-localize files."
         end
 
+        printed_string = false
         default_lang = @strings.language_codes[0]
         encoding = @options[:output_encoding] || 'UTF-8'
         File.open(path, "w:#{encoding}") do |f|
-          f.puts "/**\n * JQuery Language File\n * Generated by Twine\n * Language: #{lang}\n */"
-          f.puts "{"
+          f.print "{"
 
           @strings.sections.each_with_index do |section, si|
             printed_section = false
             section.rows.each_with_index do |row, ri|
               if row.matches_tags?(@options[:tags], @options[:untagged])
+                if printed_string
+                  f.print ",\n"
+                end
+
                 if !printed_section
-                  f.puts ''
-                  if section.name && section.name.length > 0
-                    f.puts "/* #{section.name} */"
-                  end
+                  f.print "\n"
                   printed_section = true
                 end
 
@@ -71,22 +72,11 @@ module Twine
                 value = row.translated_string_for_lang(lang, default_lang)
                 value = value.gsub('"', '\\\\"')
 
-                comment = row.comment
-                if comment
-                  comment = comment.gsub('*/', '* /')
-                end
-
-                f.print "\"#{key}\":\"#{value}\","
-
-                if comment && comment.length > 0
-                  f.print "  /* #{comment} */\n"
-                else
-                  f.print "\n"
-                end
+                f.print "\"#{key}\":\"#{value}\""
+                printed_string = true
               end
             end
           end
-          f.seek(-2, IO::SEEK_CUR)
           f.puts "\n}"
 
         end
diff --git a/tools/twine/lib/twine/version.rb b/tools/twine/lib/twine/version.rb
index ab2af43847..8df44ab28b 100644
--- a/tools/twine/lib/twine/version.rb
+++ b/tools/twine/lib/twine/version.rb
@@ -1,3 +1,3 @@
 module Twine
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end
diff --git a/tools/twine/test/fixtures/en-1.json b/tools/twine/test/fixtures/en-1.json
index 26c23f4c2c..943acb0414 100644
--- a/tools/twine/test/fixtures/en-1.json
+++ b/tools/twine/test/fixtures/en-1.json
@@ -1,11 +1,4 @@
-/**
- * JQuery Language File
- * Generated by Twine
- * Language: en
- */
 {
-
-/* My Strings */
 "key1":"key1-english",
 "key3":"key3-english",
 "key5":"A new string"
diff --git a/tools/twine/test/fixtures/en-3.xml b/tools/twine/test/fixtures/en-3.xml
new file mode 100644
index 0000000000..4e7746253a
--- /dev/null
+++ b/tools/twine/test/fixtures/en-3.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Android Strings File -->
+<!-- Generated by Twine 0.5.0 -->
+<!-- Language: en -->
+<resources>
+	<!-- SECTION: My Strings -->
+	<string name="string_with_spaces">\u0020string with spaces\u0020\u0020</string>
+</resources>
diff --git a/tools/twine/test/fixtures/test-output-10.txt b/tools/twine/test/fixtures/test-output-10.txt
new file mode 100644
index 0000000000..d08351c7a9
--- /dev/null
+++ b/tools/twine/test/fixtures/test-output-10.txt
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Android Strings File -->
+<!-- Generated by Twine 0.5.0 -->
+<!-- Language: en -->
+<resources>
+	<!-- SECTION: My Strings -->
+	<!-- String ends with space -->
+	<string name="key with space ">string with space\u0020</string>
+</resources>
diff --git a/tools/twine/test/fixtures/test-output-11.txt b/tools/twine/test/fixtures/test-output-11.txt
new file mode 100644
index 0000000000..d1ec0a6fdf
--- /dev/null
+++ b/tools/twine/test/fixtures/test-output-11.txt
@@ -0,0 +1,9 @@
+[[Uncategorized]]
+	[string_with_spaces]
+		en = ` string with spaces  `
+
+[[My Strings]]
+	[key with space ]
+		en = `string with space `
+		tags = tag1
+		comment = String ends with space
diff --git a/tools/twine/test/fixtures/test-output-5.txt b/tools/twine/test/fixtures/test-output-5.txt
index be867e2857..5f5a01dfa3 100644
--- a/tools/twine/test/fixtures/test-output-5.txt
+++ b/tools/twine/test/fixtures/test-output-5.txt
@@ -1,11 +1,4 @@
-/**
- * JQuery Language File
- * Generated by Twine
- * Language: en
- */
 {
-
-/* My Strings */
-"key1":"key1-english",  /* This is a comment */
+"key1":"key1-english",
 "key3":"key3-english"
 }
diff --git a/tools/twine/test/twine_test.rb b/tools/twine/test/twine_test.rb
index 8f61e51408..38ef26e7bb 100644
--- a/tools/twine/test/twine_test.rb
+++ b/tools/twine/test/twine_test.rb
@@ -52,6 +52,14 @@ class TwineTest < Test::Unit::TestCase
     end
   end
 
+  def test_generate_string_file_7
+    Dir.mktmpdir do |dir|
+      output_path = File.join(dir, 'en.xml')
+      Twine::Runner.run(%W(generate-string-file test/fixtures/strings-2.txt #{output_path} -t tag1))
+      assert_equal(ERB.new(File.read('test/fixtures/test-output-10.txt')).result, File.read(output_path))
+    end
+  end
+
   def test_consume_string_file_1
     Dir.mktmpdir do |dir|
       output_path = File.join(dir, 'strings.txt')
@@ -92,6 +100,14 @@ class TwineTest < Test::Unit::TestCase
     end
   end
 
+  def test_consume_string_file_6
+    Dir.mktmpdir do |dir|
+      output_path = File.join(dir, 'strings.txt')
+      Twine::Runner.run(%W(consume-string-file test/fixtures/strings-2.txt test/fixtures/en-3.xml -o #{output_path} -l en -a))
+      assert_equal(File.read('test/fixtures/test-output-11.txt'), File.read(output_path))
+    end
+  end
+
   def test_generate_report_1
     Twine::Runner.run(%w(generate-report test/fixtures/strings-1.txt))
   end