diff options
author | Kenneth Heafield <github@kheafield.com> | 2015-09-10 18:04:09 +0300 |
---|---|---|
committer | Kenneth Heafield <github@kheafield.com> | 2015-09-10 18:04:09 +0300 |
commit | 5732129bd45bb70be70eb38ad36949216c07d3bc (patch) | |
tree | d4c8703343b42682ad9ca7124371e6308cfe7682 /lm | |
parent | 37f7326057d7e7af95548684ef997b9fee1a55ca (diff) |
Update kenlm
Diffstat (limited to 'lm')
-rw-r--r-- | lm/CMakeLists.txt | 4 | ||||
-rw-r--r-- | lm/builder/CMakeLists.txt | 4 | ||||
-rw-r--r-- | lm/builder/interpolate.cc | 1 | ||||
-rw-r--r-- | lm/builder/pipeline.cc | 3 | ||||
-rw-r--r-- | lm/filter/CMakeLists.txt | 2 | ||||
-rw-r--r-- | lm/kenlm_benchmark_main.cc | 4 |
6 files changed, 10 insertions, 8 deletions
diff --git a/lm/CMakeLists.txt b/lm/CMakeLists.txt index 195fc730c..5fca22c71 100644 --- a/lm/CMakeLists.txt +++ b/lm/CMakeLists.txt @@ -71,7 +71,7 @@ foreach(exe ${EXE_LIST}) add_executable(${exe} ${exe}_main.cc $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_util>) # Link the executable against boost - target_link_libraries(${exe} ${Boost_LIBRARIES}) + target_link_libraries(${exe} ${Boost_LIBRARIES} pthread) # Group executables together set_target_properties(${exe} PROPERTIES FOLDER executables) @@ -104,7 +104,7 @@ if(BUILD_TESTING) set_target_properties(${test} PROPERTIES COMPILE_FLAGS -DBOOST_TEST_DYN_LINK) # Link the executable against boost - target_link_libraries(${test} ${Boost_LIBRARIES}) + target_link_libraries(${test} ${Boost_LIBRARIES} pthread) # model_test requires an extra command line parameter if ("${test}" STREQUAL "model_test") diff --git a/lm/builder/CMakeLists.txt b/lm/builder/CMakeLists.txt index d84a7f7da..01b415da2 100644 --- a/lm/builder/CMakeLists.txt +++ b/lm/builder/CMakeLists.txt @@ -45,7 +45,7 @@ add_library(kenlm_builder OBJECT ${KENLM_BUILDER_SOURCE}) add_executable(lmplz lmplz_main.cc $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_common> $<TARGET_OBJECTS:kenlm_builder> $<TARGET_OBJECTS:kenlm_util>) # Link the executable against boost -target_link_libraries(lmplz ${Boost_LIBRARIES}) +target_link_libraries(lmplz ${Boost_LIBRARIES} pthread) # Group executables together set_target_properties(lmplz PROPERTIES FOLDER executables) @@ -68,7 +68,7 @@ if(BUILD_TESTING) set_target_properties(${test} PROPERTIES COMPILE_FLAGS "-DBOOST_TEST_DYN_LINK -DBOOST_PROGRAM_OPTIONS_DYN_LINK") # Link the executable against boost - target_link_libraries(${test} ${Boost_LIBRARIES}) + target_link_libraries(${test} ${Boost_LIBRARIES} pthread) # Specify command arguments for how to run each unit test # diff --git a/lm/builder/interpolate.cc b/lm/builder/interpolate.cc index 6374bcf04..a62ef43d2 100644 --- a/lm/builder/interpolate.cc +++ b/lm/builder/interpolate.cc @@ -9,6 +9,7 @@ #include "util/fixed_array.hh" #include "util/murmur_hash.hh" +#include <iostream> #include <cassert> #include <cmath> diff --git a/lm/builder/pipeline.cc b/lm/builder/pipeline.cc index 69972e278..64e30f724 100644 --- a/lm/builder/pipeline.cc +++ b/lm/builder/pipeline.cc @@ -191,6 +191,9 @@ class Master { chains_.clear(); std::cerr << "Chain sizes:"; for (std::size_t i = 0; i < config_.order; ++i) { + // Always have enough for at least one record. + // This was crashing if e.g. there was no 5-gram. + assignments[i] = std::max(assignments[i], block_count[i] * NGram<BuildingPayload>::TotalSize(i + 1)); std::cerr << ' ' << (i+1) << ":" << assignments[i]; chains_.push_back(util::stream::ChainConfig(NGram<BuildingPayload>::TotalSize(i + 1), block_count[i], assignments[i])); } diff --git a/lm/filter/CMakeLists.txt b/lm/filter/CMakeLists.txt index 4e791cef8..d4616cc32 100644 --- a/lm/filter/CMakeLists.txt +++ b/lm/filter/CMakeLists.txt @@ -52,7 +52,7 @@ foreach(exe ${EXE_LIST}) add_executable(${exe} ${exe}_main.cc $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_filter> $<TARGET_OBJECTS:kenlm_util>) # Link the executable against boost - target_link_libraries(${exe} ${Boost_LIBRARIES}) + target_link_libraries(${exe} ${Boost_LIBRARIES} pthread) # Group executables together set_target_properties(${exe} PROPERTIES FOLDER executables) diff --git a/lm/kenlm_benchmark_main.cc b/lm/kenlm_benchmark_main.cc index d8b659139..1704ec6c9 100644 --- a/lm/kenlm_benchmark_main.cc +++ b/lm/kenlm_benchmark_main.cc @@ -31,9 +31,7 @@ template <class Model, class Width> void QueryFromBytes(const Model &model, int Width kEOS = model.GetVocabulary().EndSentence(); Width buf[4096]; float sum = 0.0; - while (true) { - std::size_t got = util::ReadOrEOF(fd_in, buf, sizeof(buf)); - if (!got) break; + while (std::size_t got = util::ReadOrEOF(fd_in, buf, sizeof(buf))) { UTIL_THROW_IF2(got % sizeof(Width), "File size not a multiple of vocab id size " << sizeof(Width)); got /= sizeof(Width); // Do even stuff first. |