diff options
author | Hieu Hoang <hieuhoang@gmail.com> | 2011-12-20 13:44:45 +0400 |
---|---|---|
committer | Hieu Hoang <hieuhoang@gmail.com> | 2011-12-20 13:44:45 +0400 |
commit | a6229dd52f17ccf749b120a85fa2bd7c9bdbab9a (patch) | |
tree | 5fca9c331cef9cb3e96b4602617e2c0430b9284b /lm | |
parent | 96217a0985259f585cc166fc53b6201a180da1b5 (diff) |
move some functions to .hh file for easier integration with other products
Diffstat (limited to 'lm')
-rw-r--r-- | lm/ngram_query.cc | 97 | ||||
-rw-r--r-- | lm/ngram_query.hh | 91 |
2 files changed, 99 insertions, 89 deletions
diff --git a/lm/ngram_query.cc b/lm/ngram_query.cc index 1b2cd5db3..19a63fb25 100644 --- a/lm/ngram_query.cc +++ b/lm/ngram_query.cc @@ -1,90 +1,9 @@ -#include "lm/enumerate_vocab.hh" -#include "lm/model.hh" +#include "lm/ngram_query.hh" -#include <cstdlib> -#include <fstream> -#include <iostream> -#include <string> - -#include <ctype.h> -#if !defined(_WIN32) && !defined(_WIN64) -#include <sys/resource.h> -#include <sys/time.h> -#endif - -#if !defined(_WIN32) && !defined(_WIN64) -float FloatSec(const struct timeval &tv) { - return static_cast<float>(tv.tv_sec) + (static_cast<float>(tv.tv_usec) / 1000000000.0); -} -#endif - -void PrintUsage(const char *message) { -#if !defined(_WIN32) && !defined(_WIN64) - struct rusage usage; - if (getrusage(RUSAGE_SELF, &usage)) { - perror("getrusage"); - return; - } - std::cerr << message; - std::cerr << "user\t" << FloatSec(usage.ru_utime) << "\nsys\t" << FloatSec(usage.ru_stime) << '\n'; - - // Linux doesn't set memory usage :-(. - std::ifstream status("/proc/self/status", std::ios::in); - std::string line; - while (getline(status, line)) { - if (!strncmp(line.c_str(), "VmRSS:\t", 7)) { - std::cerr << "rss " << (line.c_str() + 7) << '\n'; - break; - } - } -#endif -} - -template <class Model> void Query(const Model &model, bool sentence_context) { - PrintUsage("Loading statistics:\n"); - typename Model::State state, out; - lm::FullScoreReturn ret; - std::string word; - - while (std::cin) { - state = sentence_context ? model.BeginSentenceState() : model.NullContextState(); - float total = 0.0; - bool got = false; - unsigned int oov = 0; - while (std::cin >> word) { - got = true; - lm::WordIndex vocab = model.GetVocabulary().Index(word); - if (vocab == 0) ++oov; - ret = model.FullScore(state, vocab, out); - total += ret.prob; - std::cout << word << '=' << vocab << ' ' << static_cast<unsigned int>(ret.ngram_length) << ' ' << ret.prob << '\t'; - state = out; - char c; - while (true) { - c = std::cin.get(); - if (!std::cin) break; - if (c == '\n') break; - if (!isspace(c)) { - std::cin.unget(); - break; - } - } - if (c == '\n') break; - } - if (!got && !std::cin) break; - if (sentence_context) { - ret = model.FullScore(state, model.GetVocabulary().EndSentence(), out); - total += ret.prob; - std::cout << "</s>=" << model.GetVocabulary().EndSentence() << ' ' << static_cast<unsigned int>(ret.ngram_length) << ' ' << ret.prob << '\t'; - } - std::cout << "Total: " << total << " OOV: " << oov << '\n'; - } - PrintUsage("After queries:\n"); -} template <class Model> void Query(const char *name) { lm::ngram::Config config; - Model model(name, config); + Model model(name, config, std::cin, std::cout); Query(model); } @@ -100,19 +19,19 @@ int main(int argc, char *argv[]) { if (lm::ngram::RecognizeBinary(argv[1], model_type)) { switch(model_type) { case lm::ngram::HASH_PROBING: - Query<lm::ngram::ProbingModel>(argv[1], sentence_context); + Query<lm::ngram::ProbingModel>(argv[1], sentence_context, std::cin, std::cout); break; case lm::ngram::TRIE_SORTED: - Query<lm::ngram::TrieModel>(argv[1], sentence_context); + Query<lm::ngram::TrieModel>(argv[1], sentence_context, std::cin, std::cout); break; case lm::ngram::QUANT_TRIE_SORTED: - Query<lm::ngram::QuantTrieModel>(argv[1], sentence_context); + Query<lm::ngram::QuantTrieModel>(argv[1], sentence_context, std::cin, std::cout); break; case lm::ngram::ARRAY_TRIE_SORTED: - Query<lm::ngram::ArrayTrieModel>(argv[1], sentence_context); + Query<lm::ngram::ArrayTrieModel>(argv[1], sentence_context, std::cin, std::cout); break; case lm::ngram::QUANT_ARRAY_TRIE_SORTED: - Query<lm::ngram::QuantArrayTrieModel>(argv[1], sentence_context); + Query<lm::ngram::QuantArrayTrieModel>(argv[1], sentence_context, std::cin, std::cout); break; case lm::ngram::HASH_SORTED: default: @@ -120,7 +39,7 @@ int main(int argc, char *argv[]) { abort(); } } else { - Query<lm::ngram::ProbingModel>(argv[1], sentence_context); + Query<lm::ngram::ProbingModel>(argv[1], sentence_context, std::cin, std::cout); } PrintUsage("Total time including destruction:\n"); diff --git a/lm/ngram_query.hh b/lm/ngram_query.hh new file mode 100644 index 000000000..6bf6ec48b --- /dev/null +++ b/lm/ngram_query.hh @@ -0,0 +1,91 @@ +#ifndef LM_NGRAM_QUERY__ +#define LM_NGRAM_QUERY__ + +#include "lm/enumerate_vocab.hh" +#include "lm/model.hh" + +#include <cstdlib> +#include <fstream> +#include <iostream> +#include <string> + +#include <ctype.h> +#if !defined(_WIN32) && !defined(_WIN64) +#include <sys/resource.h> +#include <sys/time.h> +#endif + +#if !defined(_WIN32) && !defined(_WIN64) +float FloatSec(const struct timeval &tv) { + return static_cast<float>(tv.tv_sec) + (static_cast<float>(tv.tv_usec) / 1000000000.0); +} +#endif + +void PrintUsage(const char *message) { +#if !defined(_WIN32) && !defined(_WIN64) + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage)) { + perror("getrusage"); + return; + } + std::cerr << message; + std::cerr << "user\t" << FloatSec(usage.ru_utime) << "\nsys\t" << FloatSec(usage.ru_stime) << '\n'; + + // Linux doesn't set memory usage :-(. + std::ifstream status("/proc/self/status", std::ios::in); + std::string line; + while (getline(status, line)) { + if (!strncmp(line.c_str(), "VmRSS:\t", 7)) { + std::cerr << "rss " << (line.c_str() + 7) << '\n'; + break; + } + } +#endif +} + +template <class Model> void Query(const Model &model, bool sentence_context, std::istream &inStream, std::ostream &outStream) { + PrintUsage("Loading statistics:\n"); + typename Model::State state, out; + lm::FullScoreReturn ret; + std::string word; + + while (inStream) { + state = sentence_context ? model.BeginSentenceState() : model.NullContextState(); + float total = 0.0; + bool got = false; + unsigned int oov = 0; + while (inStream >> word) { + got = true; + lm::WordIndex vocab = model.GetVocabulary().Index(word); + if (vocab == 0) ++oov; + ret = model.FullScore(state, vocab, out); + total += ret.prob; + outStream << word << '=' << vocab << ' ' << static_cast<unsigned int>(ret.ngram_length) << ' ' << ret.prob << '\t'; + state = out; + char c; + while (true) { + c = inStream.get(); + if (!inStream) break; + if (c == '\n') break; + if (!isspace(c)) { + inStream.unget(); + break; + } + } + if (c == '\n') break; + } + if (!got && !inStream) break; + if (sentence_context) { + ret = model.FullScore(state, model.GetVocabulary().EndSentence(), out); + total += ret.prob; + outStream << "</s>=" << model.GetVocabulary().EndSentence() << ' ' << static_cast<unsigned int>(ret.ngram_length) << ' ' << ret.prob << '\t'; + } + outStream << "Total: " << total << " OOV: " << oov << '\n'; + } + PrintUsage("After queries:\n"); +} + + +#endif // LM_NGRAM_QUERY__ + + |