diff options
Diffstat (limited to 'lm/builder/lmplz_main.cc')
| -rw-r--r-- | lm/builder/lmplz_main.cc | 36 |
1 files changed, 31 insertions, 5 deletions
diff --git a/lm/builder/lmplz_main.cc b/lm/builder/lmplz_main.cc index e1ae2d417..265dd2164 100644 --- a/lm/builder/lmplz_main.cc +++ b/lm/builder/lmplz_main.cc @@ -33,7 +33,6 @@ std::vector<uint64_t> ParsePruning(const std::vector<std::string> ¶m, std::s // convert to vector of integers std::vector<uint64_t> prune_thresholds; prune_thresholds.reserve(order); - std::cerr << "Pruning "; for (std::vector<std::string>::const_iterator it(param.begin()); it != param.end(); ++it) { try { prune_thresholds.push_back(boost::lexical_cast<uint64_t>(*it)); @@ -66,6 +65,18 @@ std::vector<uint64_t> ParsePruning(const std::vector<std::string> ¶m, std::s return prune_thresholds; } +lm::builder::Discount ParseDiscountFallback(const std::vector<std::string> ¶m) { + lm::builder::Discount ret; + UTIL_THROW_IF(param.size() > 3, util::Exception, "Specify at most three fallback discounts: 1, 2, and 3+"); + UTIL_THROW_IF(param.empty(), util::Exception, "Fallback discounting enabled, but no discount specified"); + ret.amount[0] = 0.0; + for (unsigned i = 0; i < 3; ++i) { + float discount = boost::lexical_cast<float>(param[i < param.size() ? i : (param.size() - 1)]); + UTIL_THROW_IF(discount < 0.0 || discount > static_cast<float>(i+1), util::Exception, "The discount for count " << (i+1) << " was parsed as " << discount << " which is not in the range [0, " << (i+1) << "]."); + ret.amount[i + 1] = discount; + } + return ret; +} } // namespace @@ -77,7 +88,11 @@ int main(int argc, char *argv[]) { std::string text, arpa; std::vector<std::string> pruning; - + std::vector<std::string> discount_fallback; + std::vector<std::string> discount_fallback_default; + discount_fallback_default.push_back("0.5"); + discount_fallback_default.push_back("1"); + discount_fallback_default.push_back("1.5"); options.add_options() ("help,h", po::bool_switch(), "Show this help message") @@ -86,7 +101,7 @@ int main(int argc, char *argv[]) { ->required() #endif , "Order of the model") - ("interpolate_unigrams", po::bool_switch(&pipeline.initial_probs.interpolate_unigrams), "Interpolate the unigrams (default: emulate SRILM by not interpolating)") + ("interpolate_unigrams", po::value<bool>(&pipeline.initial_probs.interpolate_unigrams)->default_value(true)->implicit_value(true), "Interpolate the unigrams (default) as opposed to giving lots of mass to <unk> like SRI. If you want SRI's behavior with a large <unk> and the old lmplz default, use --interpolate_unigrams 0.") ("skip_symbols", po::bool_switch(), "Treat <s>, </s>, and <unk> as whitespace instead of throwing an exception") ("temp_prefix,T", po::value<std::string>(&pipeline.sort.temp_prefix)->default_value("/tmp/lm"), "Temporary file prefix") ("memory,S", SizeOption(pipeline.sort.total_memory, util::GuessPhysicalMemory() ? "80%" : "1G"), "Sorting memory") @@ -99,7 +114,9 @@ int main(int argc, char *argv[]) { ("verbose_header", po::bool_switch(&pipeline.verbose_header), "Add a verbose header to the ARPA file that includes information such as token count, smoothing type, etc.") ("text", po::value<std::string>(&text), "Read text from a file instead of stdin") ("arpa", po::value<std::string>(&arpa), "Write ARPA to a file instead of stdout") - ("prune", po::value<std::vector<std::string> >(&pruning)->multitoken(), "Prune n-grams with count less than or equal to the given threshold. Specify one value for each order i.e. 0 0 1 to prune singleton trigrams and above. The sequence of values must be non-decreasing and the last value applies to any remaining orders. Unigram pruning is not implemented, so the first value must be zero. Default is to not prune, which is equivalent to --prune 0."); + ("collapse_values", po::bool_switch(&pipeline.output_q), "Collapse probability and backoff into a single value, q that yields the same sentence-level probabilities. See http://kheafield.com/professional/edinburgh/rest_paper.pdf for more details, including a proof.") + ("prune", po::value<std::vector<std::string> >(&pruning)->multitoken(), "Prune n-grams with count less than or equal to the given threshold. Specify one value for each order i.e. 0 0 1 to prune singleton trigrams and above. The sequence of values must be non-decreasing and the last value applies to any remaining orders. Unigram pruning is not implemented, so the first value must be zero. Default is to not prune, which is equivalent to --prune 0.") + ("discount_fallback", po::value<std::vector<std::string> >(&discount_fallback)->multitoken()->implicit_value(discount_fallback_default, "0.5 1 1.5"), "The closed-form estimate for Kneser-Ney discounts does not work without singletons or doubletons. It can also fail if these values are out of range. This option falls back to user-specified discounts when the closed-form estimate fails. Note that this option is generally a bad idea: you should deduplicate your corpus instead. However, class-based models need custom discounts because they lack singleton unigrams. Provide up to three discounts (for adjusted counts 1, 2, and 3+), which will be applied to all orders where the closed-form estimates fail."); po::variables_map vm; po::store(po::parse_command_line(argc, argv, options), vm); @@ -143,7 +160,7 @@ int main(int argc, char *argv[]) { #endif if (pipeline.vocab_size_for_unk && !pipeline.initial_probs.interpolate_unigrams) { - std::cerr << "--vocab_pad requires --interpolate_unigrams" << std::endl; + std::cerr << "--vocab_pad requires --interpolate_unigrams be on" << std::endl; return 1; } @@ -153,6 +170,15 @@ int main(int argc, char *argv[]) { pipeline.disallowed_symbol_action = lm::THROW_UP; } + if (vm.count("discount_fallback")) { + pipeline.discount.fallback = ParseDiscountFallback(discount_fallback); + pipeline.discount.bad_action = lm::COMPLAIN; + } else { + // Unused, just here to prevent the compiler from complaining about uninitialized. + pipeline.discount.fallback = lm::builder::Discount(); + pipeline.discount.bad_action = lm::THROW_UP; + } + // parse pruning thresholds. These depend on order, so it is not done as a notifier. pipeline.prune_thresholds = ParsePruning(pruning, pipeline.order); |