Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/marian-nmt/sentencepiece.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTaku Kudo <taku@google.com>2018-08-05 05:33:17 +0300
committerTaku Kudo <taku@google.com>2018-08-05 05:33:17 +0300
commit9de12b5e7d680853f73ad6aab6603ea3e74ba8b0 (patch)
tree7bd56caa1824f6050348d6f20ca2bb4c1dca59be /src/sentencepiece_processor_test.cc
parentc93b4d15cdfe33dcab3e091d141882d9ce18deeb (diff)
Enable unittest on Windows
Diffstat (limited to 'src/sentencepiece_processor_test.cc')
-rw-r--r--src/sentencepiece_processor_test.cc22
1 files changed, 11 insertions, 11 deletions
diff --git a/src/sentencepiece_processor_test.cc b/src/sentencepiece_processor_test.cc
index 30bdf8f..09d6e1e 100644
--- a/src/sentencepiece_processor_test.cc
+++ b/src/sentencepiece_processor_test.cc
@@ -259,23 +259,23 @@ TEST(SentencepieceProcessorTest, EncodeTest) {
// Halfwidth to Fullwidith katakana normalization.
{
auto mock = MakeUnique<MockModel>();
- const EncodeResult result = {{WS u8"グー", 3}, {u8"グル", 4}, {"</s>", 2}};
- const absl::string_view input = WS u8"グーグル";
+ const EncodeResult result = {{WS "グー", 3}, {"グル", 4}, {"</s>", 2}};
+ const absl::string_view input = WS "グーグル";
mock->SetEncodeResult(input, result);
sp.SetModel(std::move(mock));
std::vector<std::string> output;
- EXPECT_OK(sp.Encode(u8"グーグル", &output));
+ EXPECT_OK(sp.Encode("グーグル", &output));
EXPECT_EQ(GetSpVec(result), output);
SentencePieceText spt;
- EXPECT_OK(sp.Encode(u8"グーグル", &spt));
+ EXPECT_OK(sp.Encode("グーグル", &spt));
EXPECT_EQ(3, spt.pieces_size());
for (int i = 0; i < 3; ++i) {
EXPECT_EQ(result[i].first, spt.pieces(i).piece());
}
- EXPECT_EQ(u8"グー", spt.pieces(0).surface());
- EXPECT_EQ(u8"グル", spt.pieces(1).surface());
+ EXPECT_EQ("グー", spt.pieces(0).surface());
+ EXPECT_EQ("グル", spt.pieces(1).surface());
EXPECT_EQ("", spt.pieces(2).surface());
EXPECT_EQ(3, spt.pieces(0).id());
@@ -293,23 +293,23 @@ TEST(SentencepieceProcessorTest, EncodeTest) {
// One to many normalization.
{
auto mock = MakeUnique<MockModel>();
- const EncodeResult result = {{WS u8"株式", 3}, {u8"会社", 4}, {"</s>", 2}};
- const absl::string_view input = WS u8"株式会社";
+ const EncodeResult result = {{WS "株式", 3}, {"会社", 4}, {"</s>", 2}};
+ const absl::string_view input = WS "株式会社";
mock->SetEncodeResult(input, result);
sp.SetModel(std::move(mock));
std::vector<std::string> output;
- EXPECT_OK(sp.Encode(u8"㍿", &output));
+ EXPECT_OK(sp.Encode("㍿", &output));
EXPECT_EQ(GetSpVec(result), output);
SentencePieceText spt;
- EXPECT_OK(sp.Encode(u8"㍿", &spt));
+ EXPECT_OK(sp.Encode("㍿", &spt));
EXPECT_EQ(3, spt.pieces_size());
for (int i = 0; i < 3; ++i) {
EXPECT_EQ(result[i].first, spt.pieces(i).piece());
}
EXPECT_EQ("", spt.pieces(0).surface());
- EXPECT_EQ(u8"㍿", spt.pieces(1).surface());
+ EXPECT_EQ("㍿", spt.pieces(1).surface());
EXPECT_EQ("", spt.pieces(2).surface());
EXPECT_EQ(3, spt.pieces(0).id());