Revert "merge charlock_holmes to master"

This reverts commit a25a85b9396cbc07bf2bfcd12d3c0a6685df66d6.

1 files changed, 8 insertions, 6 deletions

diff --git a/lib/utils.rb b/lib/utils.rb
index 8e5d4694cac..2de14761444 100644
--- a/lib/utils.rb
+++ b/lib/utils.rb
@@ -17,13 +17,15 @@ module Utils
   end
 
   module CharEncode
-    def encode(content)
-      content ||= ''
-      detection = CharlockHolmes::EncodingDetector.detect(content)
-      if hash = detection
-        content = CharlockHolmes::Converter.convert(content, hash[:encoding], 'UTF-8') if hash[:encoding]
+    def encode(string)
+      return '' unless string
+      cd = CharDet.detect(string)
+      if cd.confidence > 0.6
+        string.force_encoding(cd.encoding)
       end
-      content
+      string.encode("utf-8", :undef => :replace, :replace => "?", :invalid => :replace)
+    rescue
+      "Invalid Encoding"
     end
   end