#include "lm/enumerate_vocab.hh" #include "lm/model.hh" #include #include #include #include #include #if !defined(_WIN32) && !defined(_WIN64) #include #include #endif #if !defined(_WIN32) && !defined(_WIN64) float FloatSec(const struct timeval &tv) { return static_cast(tv.tv_sec) + (static_cast(tv.tv_usec) / 1000000000.0); } #endif void PrintUsage(const char *message) { #if !defined(_WIN32) && !defined(_WIN64) struct rusage usage; if (getrusage(RUSAGE_SELF, &usage)) { perror("getrusage"); return; } std::cerr << message; std::cerr << "user\t" << FloatSec(usage.ru_utime) << "\nsys\t" << FloatSec(usage.ru_stime) << '\n'; // Linux doesn't set memory usage :-(. std::ifstream status("/proc/self/status", std::ios::in); std::string line; while (getline(status, line)) { if (!strncmp(line.c_str(), "VmRSS:\t", 7)) { std::cerr << "rss " << (line.c_str() + 7) << '\n'; break; } } #endif } template void Query(const Model &model, bool sentence_context) { PrintUsage("Loading statistics:\n"); typename Model::State state, out; lm::FullScoreReturn ret; std::string word; while (std::cin) { state = sentence_context ? model.BeginSentenceState() : model.NullContextState(); float total = 0.0; bool got = false; unsigned int oov = 0; while (std::cin >> word) { got = true; lm::WordIndex vocab = model.GetVocabulary().Index(word); if (vocab == 0) ++oov; ret = model.FullScore(state, vocab, out); total += ret.prob; std::cout << word << '=' << vocab << ' ' << static_cast(ret.ngram_length) << ' ' << ret.prob << '\t'; state = out; char c; while (true) { c = std::cin.get(); if (!std::cin) break; if (c == '\n') break; if (!isspace(c)) { std::cin.unget(); break; } } if (c == '\n') break; } if (!got && !std::cin) break; if (sentence_context) { ret = model.FullScore(state, model.GetVocabulary().EndSentence(), out); total += ret.prob; std::cout << "=" << model.GetVocabulary().EndSentence() << ' ' << static_cast(ret.ngram_length) << ' ' << ret.prob << '\t'; } std::cout << "Total: " << total << " OOV: " << oov << '\n'; } PrintUsage("After queries:\n"); } template void Query(const char *name) { lm::ngram::Config config; Model model(name, config); Query(model); } int main(int argc, char *argv[]) { if (!(argc == 2 || (argc == 3 && !strcmp(argv[2], "null")))) { std::cerr << "Usage: " << argv[0] << " lm_file [null]" << std::endl; std::cerr << "Input is wrapped in and unless null is passed." << std::endl; return 1; } try { bool sentence_context = (argc == 2); lm::ngram::ModelType model_type; if (lm::ngram::RecognizeBinary(argv[1], model_type)) { switch(model_type) { case lm::ngram::HASH_PROBING: Query(argv[1], sentence_context); break; case lm::ngram::TRIE_SORTED: Query(argv[1], sentence_context); break; case lm::ngram::QUANT_TRIE_SORTED: Query(argv[1], sentence_context); break; case lm::ngram::ARRAY_TRIE_SORTED: Query(argv[1], sentence_context); break; case lm::ngram::QUANT_ARRAY_TRIE_SORTED: Query(argv[1], sentence_context); break; case lm::ngram::HASH_SORTED: default: std::cerr << "Unrecognized kenlm model type " << model_type << std::endl; abort(); } } else { Query(argv[1], sentence_context); } PrintUsage("Total time including destruction:\n"); } catch (const std::exception &e) { std::cerr << e.what() << std::endl; return 1; } return 0; }