Add Tokenize method to get collection of strings from a string.

2 files changed, 32 insertions, 6 deletions

diff --git a/base/base_tests/string_utils_test.cpp b/base/base_tests/string_utils_test.cpp
index 9d416c287b..304d6b2ce8 100644
--- a/base/base_tests/string_utils_test.cpp
+++ b/base/base_tests/string_utils_test.cpp
@@ -3,10 +3,13 @@
 #include "base/string_utils.hpp"
 #include "base/logging.hpp"
 
-#include "std/iomanip.hpp"
-#include "std/fstream.hpp"
 #include "std/bind.hpp"
+#include "std/fstream.hpp"
+#include "std/iomanip.hpp"
+#include "std/map.hpp"
+#include "std/type_traits.hpp"
 #include "std/unordered_map.hpp"
+#include "std/vector.hpp"
 
 
 /// internal function in base
@@ -206,17 +209,17 @@ UNIT_TEST(to_uint)
 
   s = "-2";
   TEST(!strings::to_uint(s, i), ());
-  
+
   s = "0";
   TEST(strings::to_uint(s, i), ());
   TEST_EQUAL(0, i, ());
-  
+
   s = "123456789123456789123456789";
   TEST(!strings::to_uint(s, i), ());
-  
+
   s = "labuda";
   TEST(!strings::to_uint(s, i), ());
-  
+
   s = "AF";
   TEST(strings::to_uint(s, i, 16), ());
   TEST_EQUAL(175, i, ());
@@ -449,6 +452,18 @@ UNIT_TEST(SimpleTokenizer)
   }
 }
 
+UNIT_TEST(Tokenize)
+{
+  {
+    initializer_list<string> expected{"acb", "def", "ghi"};
+    TEST_EQUAL(strings::Tokenize<vector>("acb def ghi", " "), vector<string>(expected), ());
+    TEST_EQUAL(strings::Tokenize<set>("acb def ghi", " "), set<string>(expected), ());
+  }
+  {
+    static_assert(is_same<vector<string>, decltype(strings::Tokenize("", ""))>::value);
+  }
+}
+
 UNIT_TEST(LastUniChar)
 {
   TEST_EQUAL(strings::LastUniChar(""), 0, ());
diff --git a/base/string_utils.hpp b/base/string_utils.hpp
index 561ec20fe1..c7645767a7 100644
--- a/base/string_utils.hpp
+++ b/base/string_utils.hpp
@@ -306,6 +306,17 @@ void Tokenize(string const & str, char const * delims, TFunctor && f)
   }
 }
 
+template <template <typename ...> class Collection = vector>
+Collection<string> Tokenize(string const & str, char const * delims)
+{
+  Collection<string> c;
+  Tokenize(str, delims, [&c](string const & str)
+  {
+    c.insert(end(c), str);
+  });
+  return c;
+}
+
 /// Splits a string by the delimiter, keeps empty parts, on an empty string returns an empty vector.
 /// Does not support quoted columns, newlines in columns and escaped quotes.
 void ParseCSVRow(string const & s, char const delimiter, vector<string> & target);