diff options
author | Taku Kudo <taku@google.com> | 2018-08-01 15:33:56 +0300 |
---|---|---|
committer | Taku Kudo <taku@google.com> | 2018-08-01 15:33:56 +0300 |
commit | 9f824c82cb7beb78d6dd6cdc278e0344abcb304c (patch) | |
tree | dedb961775d164f6be7a914a49ecc2e22b463741 /src | |
parent | aff191399d9cdb003b8c342527c65769ff62f49c (diff) |
Added u8 literal to UTF8 string.
Diffstat (limited to 'src')
-rw-r--r-- | src/bpe_model_test.cc | 6 | ||||
-rw-r--r-- | src/bpe_model_trainer_test.cc | 8 | ||||
-rw-r--r-- | src/builder_test.cc | 24 | ||||
-rw-r--r-- | src/char_model_test.cc | 2 | ||||
-rw-r--r-- | src/model_interface_test.cc | 14 | ||||
-rw-r--r-- | src/normalizer_test.cc | 26 | ||||
-rw-r--r-- | src/sentencepiece_processor_test.cc | 22 | ||||
-rw-r--r-- | src/trainer_interface_test.cc | 12 | ||||
-rw-r--r-- | src/unicode_script_test.cc | 16 | ||||
-rw-r--r-- | src/unigram_model_test.cc | 16 | ||||
-rw-r--r-- | src/unigram_model_trainer_test.cc | 8 | ||||
-rw-r--r-- | src/util_test.cc | 10 |
12 files changed, 82 insertions, 82 deletions
diff --git a/src/bpe_model_test.cc b/src/bpe_model_test.cc index b3da70c..cc60880 100644 --- a/src/bpe_model_test.cc +++ b/src/bpe_model_test.cc @@ -108,8 +108,8 @@ TEST(BPEModelTest, EncodeTest) { EXPECT_EQ("x", result[0].first); EXPECT_EQ("y", result[1].first); EXPECT_EQ("z", result[2].first); - EXPECT_EQ("東", result[3].first); - EXPECT_EQ("京", result[4].first); + EXPECT_EQ(u8"東", result[3].first); + EXPECT_EQ(u8"京", result[4].first); // User defined result = model.Encode("ABC"); @@ -177,7 +177,7 @@ TEST(BPEModelTest, EncodeAmbiguousTest) { EXPECT_EQ("a", result[2].first); // makes a broken utf-8 - const std::string broken_utf8 = std::string("あ").substr(0, 1); + const std::string broken_utf8 = std::string(u8"あ").substr(0, 1); result = model.Encode(broken_utf8); EXPECT_EQ(1, result.size()); EXPECT_EQ(broken_utf8, result[0].first); diff --git a/src/bpe_model_trainer_test.cc b/src/bpe_model_trainer_test.cc index 01e3864..2061952 100644 --- a/src/bpe_model_trainer_test.cc +++ b/src/bpe_model_trainer_test.cc @@ -111,10 +111,10 @@ TEST(BPETrainerTest, EndToEndTest) { EXPECT_TRUE(tok.empty()); EXPECT_OK(sp.Encode( - "吾輩《わがはい》は猫である。名前はまだ無い。" - "どこで生れたかとんと見当《けんとう》がつかぬ。" - "何でも薄暗いじめじめした所でニャーニャー泣いていた事だけは記憶している" - "。", + u8"吾輩《わがはい》は猫である。名前はまだ無い。" + u8"どこで生れたかとんと見当《けんとう》がつかぬ。" + u8"何でも薄暗いじめじめした所でニャーニャー泣いていた事だけは記憶している" + u8"。", &tok)); EXPECT_EQ(WS " 吾輩 《 わが はい 》 は猫 である 。 名前 はまだ 無い 。 " diff --git a/src/builder_test.cc b/src/builder_test.cc index a3af444..fd2d06c 100644 --- a/src/builder_test.cc +++ b/src/builder_test.cc @@ -66,16 +66,16 @@ TEST(BuilderTest, GetPrecompiledCharsMapTest) { const NormalizerSpec spec = SentencePieceTrainer::GetNormalizerSpec("nmt_nfkc"); const Normalizer normalizer(spec); - EXPECT_EQ(WS "ABC", normalizer.Normalize("ABC")); - EXPECT_EQ(WS "(株)", normalizer.Normalize("㈱")); - EXPECT_EQ(WS "グーグル", normalizer.Normalize("グーグル")); + EXPECT_EQ(WS "ABC", normalizer.Normalize(u8"ABC")); + EXPECT_EQ(WS u8"(株)", normalizer.Normalize(u8"㈱")); + EXPECT_EQ(WS u8"グーグル", normalizer.Normalize(u8"グーグル")); } { const NormalizerSpec spec = SentencePieceTrainer::GetNormalizerSpec("nfkc_cf"); const Normalizer normalizer(spec); - EXPECT_EQ(WS "abc", normalizer.Normalize("ABC")); + EXPECT_EQ(WS "abc", normalizer.Normalize(u8"ABC")); EXPECT_EQ(WS "abc", normalizer.Normalize("ABC")); } @@ -83,7 +83,7 @@ TEST(BuilderTest, GetPrecompiledCharsMapTest) { const NormalizerSpec spec = SentencePieceTrainer::GetNormalizerSpec("nmt_nfkc_cf"); const Normalizer normalizer(spec); - EXPECT_EQ(WS "abc", normalizer.Normalize("ABC")); + EXPECT_EQ(WS "abc", normalizer.Normalize(u8"ABC")); EXPECT_EQ(WS "abc", normalizer.Normalize("ABC")); } @@ -92,9 +92,9 @@ TEST(BuilderTest, GetPrecompiledCharsMapTest) { SentencePieceTrainer::GetNormalizerSpec("identity"); EXPECT_TRUE(spec.precompiled_charsmap().empty()); const Normalizer normalizer(spec); - EXPECT_EQ(WS "ABC", normalizer.Normalize("ABC")); - EXPECT_EQ(WS "㈱", normalizer.Normalize("㈱")); - EXPECT_EQ(WS "グーグル", normalizer.Normalize("グーグル")); + EXPECT_EQ(WS u8"ABC", normalizer.Normalize(u8"ABC")); + EXPECT_EQ(WS u8"㈱", normalizer.Normalize(u8"㈱")); + EXPECT_EQ(WS u8"グーグル", normalizer.Normalize(u8"グーグル")); } } @@ -129,11 +129,11 @@ TEST(BuilderTest, CompileCharsMap) { EXPECT_EQ("ABC", normalizer.Normalize("ABC")); EXPECT_EQ("XY" WS "Z", normalizer.Normalize("xy z")); - EXPECT_EQ("あ", normalizer.Normalize("あ")); - EXPECT_EQ("abc", normalizer.Normalize("あいう")); - EXPECT_EQ("abcえ", normalizer.Normalize("あいうえ")); + EXPECT_EQ(u8"あ", normalizer.Normalize(u8"あ")); + EXPECT_EQ("abc", normalizer.Normalize(u8"あいう")); + EXPECT_EQ("abcえ", normalizer.Normalize(u8"あいうえ")); EXPECT_EQ("ABCabcD", normalizer.Normalize("abcあいうd")); - EXPECT_EQ("abcか", normalizer.Normalize("あいうえおか")); + EXPECT_EQ("abcか", normalizer.Normalize(u8"あいうえおか")); } TEST(BuilderTest, LoadCharsMapTest) { diff --git a/src/char_model_test.cc b/src/char_model_test.cc index e9a3993..ace1b17 100644 --- a/src/char_model_test.cc +++ b/src/char_model_test.cc @@ -93,7 +93,7 @@ TEST(ModelTest, EncodeTest) { EXPECT_EQ("c", result[9].first); // makes a broken utf-8 - const std::string broken_utf8 = std::string("あ").substr(0, 1); + const std::string broken_utf8 = std::string(u8"あ").substr(0, 1); result = model.Encode(broken_utf8); EXPECT_EQ(1, result.size()); EXPECT_EQ(broken_utf8, result[0].first); diff --git a/src/model_interface_test.cc b/src/model_interface_test.cc index 51dbdae..468d8a4 100644 --- a/src/model_interface_test.cc +++ b/src/model_interface_test.cc @@ -187,7 +187,7 @@ TEST(ModelInterfaceTest, InvalidModelTest) { std::string RandomString(int length) { const char kAlphaNum[] = "0123456789" - "!@#$%^&*" + u8"!@#$%^&*" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"; const int kAlphaSize = sizeof(kAlphaNum) - 1; @@ -266,7 +266,7 @@ TEST(ModelInterfaceTest, SplitIntoWordsTest) { } TEST(ModelInterfaceTest, PrefixMatcherTest) { - const PrefixMatcher matcher({"abc", "ab", "xy", "京都"}); + const PrefixMatcher matcher({"abc", "ab", "xy", u8"京都"}); bool found; EXPECT_EQ(1, matcher.PrefixMatch("test", &found)); EXPECT_FALSE(found); @@ -278,14 +278,14 @@ TEST(ModelInterfaceTest, PrefixMatcherTest) { EXPECT_FALSE(found); EXPECT_EQ(2, matcher.PrefixMatch("xyz", &found)); EXPECT_TRUE(found); - EXPECT_EQ(6, matcher.PrefixMatch("京都大学", &found)); + EXPECT_EQ(6, matcher.PrefixMatch(u8"京都大学", &found)); EXPECT_TRUE(found); - EXPECT_EQ(3, matcher.PrefixMatch("東京大学", &found)); + EXPECT_EQ(3, matcher.PrefixMatch(u8"東京大学", &found)); EXPECT_FALSE(found); EXPECT_EQ("", matcher.GlobalReplace("", "")); EXPECT_EQ("", matcher.GlobalReplace("abc", "")); - EXPECT_EQ("--de-pqr", matcher.GlobalReplace("xyabcdeabpqr", "-")); + EXPECT_EQ("--de-pqr", matcher.GlobalReplace("xyabcdeabpqr", u8"-")); } TEST(ModelInterfaceTest, PrefixMatcherWithEmptyTest) { @@ -301,9 +301,9 @@ TEST(ModelInterfaceTest, PrefixMatcherWithEmptyTest) { EXPECT_FALSE(found); EXPECT_EQ(1, matcher.PrefixMatch("xyz", &found)); EXPECT_FALSE(found); - EXPECT_EQ(3, matcher.PrefixMatch("京都大学", &found)); + EXPECT_EQ(3, matcher.PrefixMatch(u8"京都大学", &found)); EXPECT_FALSE(found); - EXPECT_EQ(3, matcher.PrefixMatch("東京大学", &found)); + EXPECT_EQ(3, matcher.PrefixMatch(u8"東京大学", &found)); EXPECT_FALSE(found); EXPECT_EQ("", matcher.GlobalReplace("", "")); diff --git a/src/normalizer_test.cc b/src/normalizer_test.cc index d86ed24..1b74a08 100644 --- a/src/normalizer_test.cc +++ b/src/normalizer_test.cc @@ -41,7 +41,7 @@ TEST(NormalizerTest, NormalizeTest) { // Empty strings. EXPECT_EQ("", normalizer.Normalize("")); EXPECT_EQ("", normalizer.Normalize(" ")); - EXPECT_EQ("", normalizer.Normalize(" ")); + EXPECT_EQ("", normalizer.Normalize(u8" ")); // Sentence with heading/tailing/redundant spaces. EXPECT_EQ(WS "ABC", normalizer.Normalize("ABC")); @@ -53,13 +53,13 @@ TEST(NormalizerTest, NormalizeTest) { EXPECT_EQ(WS "ABC", normalizer.Normalize(" ABC ")); // NFKC char to char normalization. - EXPECT_EQ(WS "123", normalizer.Normalize("①②③")); + EXPECT_EQ(WS "123", normalizer.Normalize(u8"①②③")); // NFKC char to multi-char normalization. - EXPECT_EQ(WS "株式会社", normalizer.Normalize("㍿")); + EXPECT_EQ(WS u8"株式会社", normalizer.Normalize(u8"㍿")); // Half width katakana, character composition happens. - EXPECT_EQ(WS "グーグル", normalizer.Normalize(" グーグル ")); + EXPECT_EQ(WS u8"グーグル", normalizer.Normalize(" グーグル ")); EXPECT_EQ(WS "I" WS "saw" WS "a" WS "girl", normalizer.Normalize(" I saw a girl ")); @@ -73,7 +73,7 @@ TEST(NormalizerTest, NormalizeWithoutDummyPrefixTest) { // Empty strings. EXPECT_EQ("", normalizer.Normalize("")); EXPECT_EQ("", normalizer.Normalize(" ")); - EXPECT_EQ("", normalizer.Normalize(" ")); + EXPECT_EQ("", normalizer.Normalize(u8" ")); // Sentence with heading/tailing/redundant spaces. EXPECT_EQ("ABC", normalizer.Normalize("ABC")); @@ -93,7 +93,7 @@ TEST(NormalizerTest, NormalizeWithoutRemoveExtraWhitespacesTest) { // Empty strings. EXPECT_EQ("", normalizer.Normalize("")); EXPECT_EQ(WS WS WS WS WS WS WS, normalizer.Normalize(" ")); - EXPECT_EQ(WS WS, normalizer.Normalize(" ")); + EXPECT_EQ(WS WS, normalizer.Normalize(u8" ")); // Sentence with heading/tailing/redundant spaces. EXPECT_EQ(WS "ABC", normalizer.Normalize("ABC")); @@ -112,7 +112,7 @@ TEST(NormalizerTest, NormalizeWithoutEscapeWhitespacesTest) { // Empty strings. EXPECT_EQ("", normalizer.Normalize("")); EXPECT_EQ("", normalizer.Normalize(" ")); - EXPECT_EQ("", normalizer.Normalize(" ")); + EXPECT_EQ("", normalizer.Normalize(u8" ")); // Sentence with heading/tailing/redundant spaces. EXPECT_EQ("ABC", normalizer.Normalize("ABC")); @@ -291,7 +291,7 @@ TEST(NormalizerTest, NormalizeFullTest) { { const std::string input = " グーグル "; // halfwidth katakana normalizer.Normalize(input, &output, &n2i); - EXPECT_EQ(WS "グーグル", output); + EXPECT_EQ(WS u8"グーグル", output); const std::vector<size_t> expected = {1, 1, 1, // WS (3byte) 1, 1, 1, // グ 7, 7, 7, // ー @@ -302,7 +302,7 @@ TEST(NormalizerTest, NormalizeFullTest) { } { - const std::string input = "①②③"; + const std::string input = u8"①②③"; normalizer.Normalize(input, &output, &n2i); EXPECT_EQ(WS "123", output); const std::vector<size_t> expected = {0, 0, 0, // WS (3byte) @@ -314,17 +314,17 @@ TEST(NormalizerTest, NormalizeFullTest) { } { - const std::string input = "㍿"; + const std::string input = u8"㍿"; normalizer.Normalize(input, &output, &n2i); - EXPECT_EQ(WS "株式会社", output); + EXPECT_EQ(WS u8"株式会社", output); const std::vector<size_t> expected = {0, 0, 0, // WS (3byte) 0, 0, 0, // 株 0, 0, 0, // 式 0, 0, 0, // 会 0, 0, 0, // 社 3}; - // When "株式" is one piece, this has no alignment to the input. - // Sentencepieces which includes the last character ("会社" or "社") + // When u8"株式" is one piece, this has no alignment to the input. + // Sentencepieces which includes the last character (u8"会社" or u8"社") // have the alignment to the input. EXPECT_EQ(expected, n2i); } diff --git a/src/sentencepiece_processor_test.cc b/src/sentencepiece_processor_test.cc index 3a78c87..4e6797d 100644 --- a/src/sentencepiece_processor_test.cc +++ b/src/sentencepiece_processor_test.cc @@ -259,23 +259,23 @@ TEST(SentencepieceProcessorTest, EncodeTest) { // Halfwidth to Fullwidith katakana normalization. { auto mock = MakeUnique<MockModel>(); - const EncodeResult result = {{WS "グー", 3}, {"グル", 4}, {"</s>", 2}}; - const absl::string_view input = WS "グーグル"; + const EncodeResult result = {{WS u8"グー", 3}, {u8"グル", 4}, {"</s>", 2}}; + const absl::string_view input = WS u8"グーグル"; mock->SetEncodeResult(input, result); sp.SetModel(std::move(mock)); std::vector<std::string> output; - EXPECT_OK(sp.Encode("グーグル", &output)); + EXPECT_OK(sp.Encode(u8"グーグル", &output)); EXPECT_EQ(GetSpVec(result), output); SentencePieceText spt; - EXPECT_OK(sp.Encode("グーグル", &spt)); + EXPECT_OK(sp.Encode(u8"グーグル", &spt)); EXPECT_EQ(3, spt.pieces_size()); for (int i = 0; i < 3; ++i) { EXPECT_EQ(result[i].first, spt.pieces(i).piece()); } - EXPECT_EQ("グー", spt.pieces(0).surface()); - EXPECT_EQ("グル", spt.pieces(1).surface()); + EXPECT_EQ(u8"グー", spt.pieces(0).surface()); + EXPECT_EQ(u8"グル", spt.pieces(1).surface()); EXPECT_EQ("", spt.pieces(2).surface()); EXPECT_EQ(3, spt.pieces(0).id()); @@ -293,23 +293,23 @@ TEST(SentencepieceProcessorTest, EncodeTest) { // One to many normalization. { auto mock = MakeUnique<MockModel>(); - const EncodeResult result = {{WS "株式", 3}, {"会社", 4}, {"</s>", 2}}; - const absl::string_view input = WS "株式会社"; + const EncodeResult result = {{WS u8"株式", 3}, {u8"会社", 4}, {"</s>", 2}}; + const absl::string_view input = WS u8"株式会社"; mock->SetEncodeResult(input, result); sp.SetModel(std::move(mock)); std::vector<std::string> output; - EXPECT_OK(sp.Encode("㍿", &output)); + EXPECT_OK(sp.Encode(u8"㍿", &output)); EXPECT_EQ(GetSpVec(result), output); SentencePieceText spt; - EXPECT_OK(sp.Encode("㍿", &spt)); + EXPECT_OK(sp.Encode(u8"㍿", &spt)); EXPECT_EQ(3, spt.pieces_size()); for (int i = 0; i < 3; ++i) { EXPECT_EQ(result[i].first, spt.pieces(i).piece()); } EXPECT_EQ("", spt.pieces(0).surface()); - EXPECT_EQ("㍿", spt.pieces(1).surface()); + EXPECT_EQ(u8"㍿", spt.pieces(1).surface()); EXPECT_EQ("", spt.pieces(2).surface()); EXPECT_EQ(3, spt.pieces(0).id()); diff --git a/src/trainer_interface_test.cc b/src/trainer_interface_test.cc index 03e038f..10baba1 100644 --- a/src/trainer_interface_test.cc +++ b/src/trainer_interface_test.cc @@ -49,9 +49,9 @@ TEST(TrainerInterfaceTest, IsValidSentencePieceTest) { EXPECT_FALSE(IsValid(WS "a" WS)); EXPECT_FALSE(IsValid("a" WS "b")); EXPECT_FALSE(IsValid("a" WS "b" WS)); - EXPECT_TRUE(IsValid("あいう")); - EXPECT_TRUE(IsValid("グーグル")); // "ー" is a part of Katakana - EXPECT_TRUE(IsValid("食べる")); + EXPECT_TRUE(IsValid(u8"あいう")); + EXPECT_TRUE(IsValid(u8"グーグル")); // u8"ー" is a part of Katakana + EXPECT_TRUE(IsValid(u8"食べる")); EXPECT_FALSE(IsValid("漢字ABC")); // mixed CJK scripts EXPECT_FALSE(IsValid("F1")); EXPECT_TRUE(IsValid("$10")); // $ and 1 are both "common" script. @@ -70,9 +70,9 @@ TEST(TrainerInterfaceTest, IsValidSentencePieceTest) { EXPECT_FALSE(IsValid("a" WS "b" WS)); trainer_spec.set_split_by_unicode_script(false); - EXPECT_TRUE(IsValid("あいう")); - EXPECT_TRUE(IsValid("グーグル")); - EXPECT_TRUE(IsValid("食べる")); + EXPECT_TRUE(IsValid(u8"あいう")); + EXPECT_TRUE(IsValid(u8"グーグル")); + EXPECT_TRUE(IsValid(u8"食べる")); EXPECT_TRUE(IsValid("漢字ABC")); EXPECT_TRUE(IsValid("F1")); EXPECT_TRUE(IsValid("$10")); diff --git a/src/unicode_script_test.cc b/src/unicode_script_test.cc index 6a79cf1..18492ab 100644 --- a/src/unicode_script_test.cc +++ b/src/unicode_script_test.cc @@ -27,17 +27,17 @@ ScriptType GetScriptType(absl::string_view s) { } TEST(UnicodeScript, GetScriptTypeTest) { - EXPECT_EQ(U_Han, GetScriptType("京")); - EXPECT_EQ(U_Han, GetScriptType("太")); - EXPECT_EQ(U_Hiragana, GetScriptType("い")); - EXPECT_EQ(U_Katakana, GetScriptType("グ")); - EXPECT_EQ(U_Common, GetScriptType("ー")); + EXPECT_EQ(U_Han, GetScriptType(u8"京")); + EXPECT_EQ(U_Han, GetScriptType(u8"太")); + EXPECT_EQ(U_Hiragana, GetScriptType(u8"い")); + EXPECT_EQ(U_Katakana, GetScriptType(u8"グ")); + EXPECT_EQ(U_Common, GetScriptType(u8"ー")); EXPECT_EQ(U_Latin, GetScriptType("a")); EXPECT_EQ(U_Latin, GetScriptType("A")); EXPECT_EQ(U_Common, GetScriptType("0")); - EXPECT_EQ(U_Common, GetScriptType("$")); - EXPECT_EQ(U_Common, GetScriptType("@")); - EXPECT_EQ(U_Common, GetScriptType("-")); + EXPECT_EQ(U_Common, GetScriptType(u8"$")); + EXPECT_EQ(U_Common, GetScriptType(u8"@")); + EXPECT_EQ(U_Common, GetScriptType(u8"-")); } } // namespace unicode_script } // namespace sentencepiece diff --git a/src/unigram_model_test.cc b/src/unigram_model_test.cc index 66c9c75..a6f2d64 100644 --- a/src/unigram_model_test.cc +++ b/src/unigram_model_test.cc @@ -87,19 +87,19 @@ TEST(LatticeTest, InsertTest) { EXPECT_EQ("A", node[0]->piece); EXPECT_EQ("B", node[1]->piece); - EXPECT_EQ("あ", node[2]->piece); - EXPECT_EQ("い", node[3]->piece); + EXPECT_EQ(u8"あ", node[2]->piece); + EXPECT_EQ(u8"い", node[3]->piece); EXPECT_EQ("AB", node[4]->piece); EXPECT_EQ("Bあ", node[5]->piece); - EXPECT_EQ("あい", node[6]->piece); + EXPECT_EQ(u8"あい", node[6]->piece); EXPECT_EQ("A", node[0]->piece); EXPECT_EQ("B", node[1]->piece); - EXPECT_EQ("あ", node[2]->piece); - EXPECT_EQ("い", node[3]->piece); + EXPECT_EQ(u8"あ", node[2]->piece); + EXPECT_EQ(u8"い", node[3]->piece); EXPECT_EQ("AB", node[4]->piece); EXPECT_EQ("Bあ", node[5]->piece); - EXPECT_EQ("あい", node[6]->piece); + EXPECT_EQ(u8"あい", node[6]->piece); EXPECT_EQ(0, node[0]->pos); EXPECT_EQ(1, node[1]->pos); @@ -569,8 +569,8 @@ TEST(UnigramModelTest, EncodeTest) { EXPECT_EQ("x", result[0].first); EXPECT_EQ("y", result[1].first); EXPECT_EQ("z", result[2].first); - EXPECT_EQ("東", result[3].first); - EXPECT_EQ("京", result[4].first); + EXPECT_EQ(u8"東", result[3].first); + EXPECT_EQ(u8"京", result[4].first); // User defined result = model.Encode("ABC"); diff --git a/src/unigram_model_trainer_test.cc b/src/unigram_model_trainer_test.cc index c7164fa..153b3aa 100644 --- a/src/unigram_model_trainer_test.cc +++ b/src/unigram_model_trainer_test.cc @@ -65,10 +65,10 @@ TEST(UnigramTrainerTest, EndToEndTest) { EXPECT_TRUE(tok.empty()); EXPECT_OK(sp.Encode( - "吾輩《わがはい》は猫である。名前はまだ無い。" - "どこで生れたかとんと見当《けんとう》がつかぬ。" - "何でも薄暗いじめじめした所でニャーニャー泣いていた事だけは記憶している" - "。", + u8"吾輩《わがはい》は猫である。名前はまだ無い。" + u8"どこで生れたかとんと見当《けんとう》がつかぬ。" + u8"何でも薄暗いじめじめした所でニャーニャー泣いていた事だけは記憶している" + u8"。", &tok)); EXPECT_EQ(WS " 吾輩 《 わが はい 》 は 猫 である 。 名前 はまだ 無い 。 " diff --git a/src/util_test.cc b/src/util_test.cc index ba5e224..6772b4a 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -178,7 +178,7 @@ TEST(UtilTest, JoinTest) { tokens.push_back("a"); tokens.push_back("test"); EXPECT_EQ(string_util::Join(tokens, " "), "this is a test"); - EXPECT_EQ(string_util::Join(tokens, ":"), "this:is:a:test"); + EXPECT_EQ(string_util::Join(tokens, u8":"), "this:is:a:test"); EXPECT_EQ(string_util::Join(tokens, ""), "thisisatest"); tokens[2] = ""; EXPECT_EQ(string_util::Join(tokens, " "), "this is test"); @@ -191,7 +191,7 @@ TEST(UtilTest, JoinIntTest) { tokens.push_back(-4); tokens.push_back(5); EXPECT_EQ(string_util::Join(tokens, " "), "10 2 -4 5"); - EXPECT_EQ(string_util::Join(tokens, ":"), "10:2:-4:5"); + EXPECT_EQ(string_util::Join(tokens, u8":"), "10:2:-4:5"); EXPECT_EQ(string_util::Join(tokens, ""), "102-45"); } @@ -274,7 +274,7 @@ TEST(UtilTest, ItoaTest) { TEST(UtilTest, OneCharLenTest) { EXPECT_EQ(1, string_util::OneCharLen("abc")); - EXPECT_EQ(3, string_util::OneCharLen("テスト")); + EXPECT_EQ(3, string_util::OneCharLen(u8"テスト")); } TEST(UtilTest, DecodeUTF8Test) { @@ -438,8 +438,8 @@ TEST(UtilTest, UnicodeTextToUTF8Test) { ut = string_util::UTF8ToUnicodeText("test"); EXPECT_EQ("test", string_util::UnicodeTextToUTF8(ut)); - ut = string_util::UTF8ToUnicodeText("テスト"); - EXPECT_EQ("テスト", string_util::UnicodeTextToUTF8(ut)); + ut = string_util::UTF8ToUnicodeText(u8"テスト"); + EXPECT_EQ(u8"テスト", string_util::UnicodeTextToUTF8(ut)); ut = string_util::UTF8ToUnicodeText("これはtest"); EXPECT_EQ("これはtest", string_util::UnicodeTextToUTF8(ut)); |