diff options
author | micael.bergeron <micael.bergeron@solutionstlm.com> | 2017-09-04 20:34:15 +0300 |
---|---|---|
committer | micael.bergeron <micael.bergeron@solutionstlm.com> | 2017-09-06 16:01:53 +0300 |
commit | c9aa7932152bd10f6fc7b87b3263922aa4b911b7 (patch) | |
tree | dea1ae4b101ca182eefa16b8b88c8b46308071bf /lib/gitlab/encoding_helper.rb | |
parent | bca72f5906ed38dc231ef066231238758c1cb42d (diff) |
revert to using a simple representation
Diffstat (limited to 'lib/gitlab/encoding_helper.rb')
-rw-r--r-- | lib/gitlab/encoding_helper.rb | 17 |
1 files changed, 12 insertions, 5 deletions
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index c5e173ba55a..21a14141c87 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -24,7 +24,7 @@ module Gitlab # return message if message type is binary detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if binary?(message, detect) + return message.force_encoding("BINARY") if all_binary?(message, detect) if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD # force detected encoding if we have sufficient confidence. @@ -34,14 +34,21 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) rescue => e - byebug encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" end - def binary?(message, detect=nil) - detect ||= CharlockHolmes::EncodingDetector.detect(message) - detect && detect[:type] == :binary && detect[:confidence] == 100 + def all_binary?(data, detect=nil) + detect ||= CharlockHolmes::EncodingDetector.detect(data) + detect && detect[:type] == :binary + end + + def libgit2_binary?(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + all_binary?(data, detect) end def encode_utf8(message) |