diff options
author | Saito <saitowu@gmail.com> | 2011-12-29 07:51:33 +0400 |
---|---|---|
committer | Saito <saitowu@gmail.com> | 2011-12-29 07:52:28 +0400 |
commit | a25a85b9396cbc07bf2bfcd12d3c0a6685df66d6 (patch) | |
tree | 753393aee8de4de186679ee672d46beb270f9dbb /lib | |
parent | 6e5caa19856a683e5e9ead5cda907cb4119762d6 (diff) |
merge charlock_holmes to master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/utils.rb | 14 |
1 files changed, 6 insertions, 8 deletions
diff --git a/lib/utils.rb b/lib/utils.rb index 2de14761444..8e5d4694cac 100644 --- a/lib/utils.rb +++ b/lib/utils.rb @@ -17,15 +17,13 @@ module Utils end module CharEncode - def encode(string) - return '' unless string - cd = CharDet.detect(string) - if cd.confidence > 0.6 - string.force_encoding(cd.encoding) + def encode(content) + content ||= '' + detection = CharlockHolmes::EncodingDetector.detect(content) + if hash = detection + content = CharlockHolmes::Converter.convert(content, hash[:encoding], 'UTF-8') if hash[:encoding] end - string.encode("utf-8", :undef => :replace, :replace => "?", :invalid => :replace) - rescue - "Invalid Encoding" + content end end |