diff options
author | Yuri Gorshenin <y@maps.me> | 2017-04-18 17:37:14 +0300 |
---|---|---|
committer | Yuri Gorshenin <y@maps.me> | 2017-04-19 15:42:12 +0300 |
commit | a0c121bc255aabbafb8eede636e787245e00ef33 (patch) | |
tree | 4bd373793bc0690aac89b9428b0a4832f50a42f7 /search | |
parent | 02483b4948479ec463c737b6db77dbc3db015079 (diff) |
[search] Single json array is replaced by lines of json objects.
Diffstat (limited to 'search')
-rw-r--r-- | search/search_quality/assessment_tool/main_model.cpp | 4 | ||||
-rw-r--r-- | search/search_quality/features_collector_tool/features_collector_tool.cpp | 8 | ||||
-rwxr-xr-x | search/search_quality/gen-samples.lisp | 2 | ||||
-rw-r--r-- | search/search_quality/sample.cpp | 44 | ||||
-rw-r--r-- | search/search_quality/sample.hpp | 4 | ||||
-rw-r--r-- | search/search_quality/search_quality_tests/sample_test.cpp | 82 |
6 files changed, 39 insertions, 105 deletions
diff --git a/search/search_quality/assessment_tool/main_model.cpp b/search/search_quality/assessment_tool/main_model.cpp index 066f1c8284..f3e9107491 100644 --- a/search/search_quality/assessment_tool/main_model.cpp +++ b/search/search_quality/assessment_tool/main_model.cpp @@ -58,7 +58,7 @@ void MainModel::Open(string const & path) } vector<search::Sample> samples; - if (!search::Sample::DeserializeFromJSON(contents, samples)) + if (!search::Sample::DeserializeFromJSONLines(contents, samples)) { m_view->ShowError("Can't parse samples: " + path); return; @@ -95,7 +95,7 @@ void MainModel::SaveAs(string const & path) search::FeatureLoader loader(m_index); string contents; - search::Sample::SerializeToJSON(m_contexts.MakeSamples(loader), contents); + search::Sample::SerializeToJSONLines(m_contexts.MakeSamples(loader), contents); { ofstream ofs(path); diff --git a/search/search_quality/features_collector_tool/features_collector_tool.cpp b/search/search_quality/features_collector_tool/features_collector_tool.cpp index 598f11c913..d72b218863 100644 --- a/search/search_quality/features_collector_tool/features_collector_tool.cpp +++ b/search/search_quality/features_collector_tool/features_collector_tool.cpp @@ -127,10 +127,10 @@ int main(int argc, char * argv[]) auto infoGetter = CountryInfoReader::CreateCountryInfoReader(platform); infoGetter->InitAffiliationsInfo(&storage.GetAffiliations()); - string jsonStr; + string lines; if (FLAGS_json_in.empty()) { - GetContents(cin, jsonStr); + GetContents(cin, lines); } else { @@ -140,11 +140,11 @@ int main(int argc, char * argv[]) cerr << "Can't open input json file." << endl; return -1; } - GetContents(ifs, jsonStr); + GetContents(ifs, lines); } vector<Sample> samples; - if (!Sample::DeserializeFromJSON(jsonStr, samples)) + if (!Sample::DeserializeFromJSONLines(lines, samples)) { cerr << "Can't parse input json file." << endl; return -1; diff --git a/search/search_quality/gen-samples.lisp b/search/search_quality/gen-samples.lisp index bc843e4fd8..a61fa6cd8b 100755 --- a/search/search_quality/gen-samples.lisp +++ b/search/search_quality/gen-samples.lisp @@ -129,4 +129,4 @@ exec /usr/bin/env sbcl --noinform --quit --load "$0" --end-toplevel-options "$@" (loop for sample in *samples* summing (length (slot-value sample 'results)))) -(format t "~a~%" (json:encode-json-to-string (reverse *samples*))) +(format t "~{~a~%~}" (mapcar #'json:encode-json-to-string (reverse *samples*))) diff --git a/search/search_quality/sample.cpp b/search/search_quality/sample.cpp index 2d0c532fe8..65a80fb697 100644 --- a/search/search_quality/sample.cpp +++ b/search/search_quality/sample.cpp @@ -128,35 +128,37 @@ bool Sample::operator==(Sample const & rhs) const } // static -bool Sample::DeserializeFromJSON(string const & jsonStr, std::vector<Sample> & samples) +bool Sample::DeserializeFromJSONLines(string const & lines, std::vector<Sample> & samples) { - try - { - my::Json root(jsonStr.c_str()); - if (!json_is_array(root.get())) - MYTHROW(my::Json::Exception, ("The field", "samples", "must contain a json array.")); - size_t numSamples = json_array_size(root.get()); - samples.resize(numSamples); - for (size_t i = 0; i < numSamples; ++i) - samples[i].DeserializeFromJSONImpl(json_array_get(root.get(), i)); - return true; - } - catch (my::Json::Exception const & e) + istringstream is(lines); + string line; + vector<Sample> result; + + while (getline(is, line)) { - LOG(LERROR, ("Can't parse samples:", e.Msg(), jsonStr)); + if (line.empty()) + continue; + + Sample sample; + if (!sample.DeserializeFromJSON(line)) + return false; + result.emplace_back(move(sample)); } - return false; + + samples.insert(samples.end(), result.begin(), result.end()); + return true; } // static -void Sample::SerializeToJSON(std::vector<Sample> const & samples, std::string & jsonStr) +void Sample::SerializeToJSONLines(std::vector<Sample> const & samples, std::string & lines) { - auto array = my::NewJSONArray(); for (auto const & sample : samples) - json_array_append_new(array.get(), sample.SerializeToJSON().release()); - std::unique_ptr<char, JSONFreeDeleter> buffer( - json_dumps(array.get(), JSON_COMPACT | JSON_ENSURE_ASCII)); - jsonStr.assign(buffer.get()); + { + unique_ptr<char, JSONFreeDeleter> buffer( + json_dumps(sample.SerializeToJSON().get(), JSON_COMPACT | JSON_ENSURE_ASCII)); + lines.append(buffer.get()); + lines.push_back('\n'); + } } void Sample::DeserializeFromJSONImpl(json_t * root) diff --git a/search/search_quality/sample.hpp b/search/search_quality/sample.hpp index e82f34f9f6..3f3c325024 100644 --- a/search/search_quality/sample.hpp +++ b/search/search_quality/sample.hpp @@ -41,8 +41,8 @@ struct Sample bool DeserializeFromJSON(std::string const & jsonStr); my::JSONPtr SerializeToJSON() const; - static bool DeserializeFromJSON(std::string const & jsonStr, std::vector<Sample> & samples); - static void SerializeToJSON(std::vector<Sample> const & samples, std::string & jsonStr); + static bool DeserializeFromJSONLines(std::string const & lines, std::vector<Sample> & samples); + static void SerializeToJSONLines(std::vector<Sample> const & samples, std::string & lines); bool operator<(Sample const & rhs) const; diff --git a/search/search_quality/search_quality_tests/sample_test.cpp b/search/search_quality/search_quality_tests/sample_test.cpp index 0e9569ca70..f4f6fcaa6a 100644 --- a/search/search_quality/search_quality_tests/sample_test.cpp +++ b/search/search_quality/search_quality_tests/sample_test.cpp @@ -127,81 +127,13 @@ UNIT_CLASS_TEST(SampleTest, BadViewport) UNIT_CLASS_TEST(SampleTest, Arrays) { - auto const jsonStr = R"EOF( - [ - { - "query": "cuba", - "locale": "en", - "position": { - "x": 37.618706, - "y": 99.53730574302003 - }, - "viewport": { - "minx": 37.1336, - "miny": 67.1349, - "maxx": 38.0314, - "maxy": 67.7348 - }, - "results": [ - { - "name": "Cuba", - "relevancy": "relevant", - "types": [ - "place-country" - ], - "position": { - "x": -80.832886, - "y": 15.521132748163712 - }, - "houseNumber": "" - } - ] - }, - { - "query": "riga", - "locale": "en", - "position": { - "x": 37.65376, - "y": 98.51110651930014 - }, - "viewport": { - "minx": 37.5064, - "miny": 67.0476, - "maxx": 37.7799, - "maxy": 67.304 - }, - "results": [ - { - "name": "R\u012bga", - "relevancy": "vital", - "types": [ - "place-city-capital-2" - ], - "position": { - "x": 24.105186, - "y": 107.7819569220319 - }, - "houseNumber": "" - }, - { - "name": "R\u012bga", - "relevancy": "vital", - "types": [ - "place-city-capital-2" - ], - "position": { - "x": 24.105186, - "y": 107.7819569220319 - }, - "houseNumber": "" - } - ] - } - ] + auto const lines = R"EOF( + {"query": "cuba", "locale": "en", "position": {"x": 37.618706, "y": 99.53730574302003}, "viewport": {"minx": 37.1336, "miny": 67.1349, "maxx": 38.0314, "maxy": 67.7348}, "results": [{"name": "Cuba", "relevancy": "relevant", "types": ["place-country"], "position": {"x": -80.832886, "y": 15.521132748163712}, "houseNumber": ""}]} + {"query": "riga", "locale": "en", "position": {"x": 37.65376, "y": 98.51110651930014}, "viewport": {"minx": 37.5064, "miny": 67.0476, "maxx": 37.7799, "maxy": 67.304}, "results": [{"name": "R\u012bga", "relevancy": "vital", "types": ["place-city-capital-2"], "position": {"x": 24.105186, "y": 107.7819569220319}, "houseNumber": ""}, {"name": "R\u012bga", "relevancy": "vital", "types": ["place-city-capital-2"], "position": {"x": 24.105186, "y": 107.7819569220319}, "houseNumber": ""}]} )EOF"; vector<Sample> samples; - TEST(Sample::DeserializeFromJSON(jsonStr, samples), ()); + TEST(Sample::DeserializeFromJSONLines(lines, samples), ()); vector<Sample> expected = {m_cuba, m_riga}; @@ -215,11 +147,11 @@ UNIT_CLASS_TEST(SampleTest, SerDes) { vector<Sample> expected = {m_cuba, m_riga}; - std::string buffer; - Sample::SerializeToJSON(expected, buffer); + std::string lines; + Sample::SerializeToJSONLines(expected, lines); vector<Sample> actual; - TEST(Sample::DeserializeFromJSON(buffer, actual), ()); + TEST(Sample::DeserializeFromJSONLines(lines, actual), ()); sort(expected.begin(), expected.end()); sort(actual.begin(), actual.end()); |