diff options
author | Saito <saitowu@gmail.com> | 2011-12-30 14:30:57 +0400 |
---|---|---|
committer | Saito <saitowu@gmail.com> | 2011-12-30 14:30:57 +0400 |
commit | 7279e8c1774f3116dd74ca462447189730f9eff8 (patch) | |
tree | 438c5671951a2a01e1484c3a3bb099a1d26eb162 /lib | |
parent | 5719f02f34339237a2b1faeda30976a404358910 (diff) |
Revert "merge charlock_holmes to master"
This reverts commit a25a85b9396cbc07bf2bfcd12d3c0a6685df66d6.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/utils.rb | 14 |
1 files changed, 8 insertions, 6 deletions
diff --git a/lib/utils.rb b/lib/utils.rb index 8e5d4694cac..2de14761444 100644 --- a/lib/utils.rb +++ b/lib/utils.rb @@ -17,13 +17,15 @@ module Utils end module CharEncode - def encode(content) - content ||= '' - detection = CharlockHolmes::EncodingDetector.detect(content) - if hash = detection - content = CharlockHolmes::Converter.convert(content, hash[:encoding], 'UTF-8') if hash[:encoding] + def encode(string) + return '' unless string + cd = CharDet.detect(string) + if cd.confidence > 0.6 + string.force_encoding(cd.encoding) end - content + string.encode("utf-8", :undef => :replace, :replace => "?", :invalid => :replace) + rescue + "Invalid Encoding" end end |