Update kenlm

6 files changed, 10 insertions, 8 deletions

diff --git a/lm/CMakeLists.txt b/lm/CMakeLists.txt
index 195fc730c..5fca22c71 100644
--- a/lm/CMakeLists.txt
+++ b/lm/CMakeLists.txt
@@ -71,7 +71,7 @@ foreach(exe ${EXE_LIST})
   add_executable(${exe} ${exe}_main.cc $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_util>)
 
   # Link the executable against boost
-  target_link_libraries(${exe} ${Boost_LIBRARIES})
+  target_link_libraries(${exe} ${Boost_LIBRARIES} pthread)
 
   # Group executables together
   set_target_properties(${exe} PROPERTIES FOLDER executables)
@@ -104,7 +104,7 @@ if(BUILD_TESTING)
       set_target_properties(${test} PROPERTIES COMPILE_FLAGS -DBOOST_TEST_DYN_LINK)
   
       # Link the executable against boost
-      target_link_libraries(${test} ${Boost_LIBRARIES})
+      target_link_libraries(${test} ${Boost_LIBRARIES} pthread)
 
       # model_test requires an extra command line parameter
       if ("${test}" STREQUAL "model_test")
diff --git a/lm/builder/CMakeLists.txt b/lm/builder/CMakeLists.txt
index d84a7f7da..01b415da2 100644
--- a/lm/builder/CMakeLists.txt
+++ b/lm/builder/CMakeLists.txt
@@ -45,7 +45,7 @@ add_library(kenlm_builder OBJECT ${KENLM_BUILDER_SOURCE})
 add_executable(lmplz lmplz_main.cc $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_common> $<TARGET_OBJECTS:kenlm_builder> $<TARGET_OBJECTS:kenlm_util>)
 
 # Link the executable against boost
-target_link_libraries(lmplz ${Boost_LIBRARIES})
+target_link_libraries(lmplz ${Boost_LIBRARIES} pthread)
 
 # Group executables together
 set_target_properties(lmplz PROPERTIES FOLDER executables)
@@ -68,7 +68,7 @@ if(BUILD_TESTING)
       set_target_properties(${test} PROPERTIES COMPILE_FLAGS "-DBOOST_TEST_DYN_LINK -DBOOST_PROGRAM_OPTIONS_DYN_LINK")
   
       # Link the executable against boost
-      target_link_libraries(${test} ${Boost_LIBRARIES})
+      target_link_libraries(${test} ${Boost_LIBRARIES} pthread)
       
       # Specify command arguments for how to run each unit test
       #
diff --git a/lm/builder/interpolate.cc b/lm/builder/interpolate.cc
index 6374bcf04..a62ef43d2 100644
--- a/lm/builder/interpolate.cc
+++ b/lm/builder/interpolate.cc
@@ -9,6 +9,7 @@
 #include "util/fixed_array.hh"
 #include "util/murmur_hash.hh"
 
+#include <iostream>
 #include <cassert>
 #include <cmath>
 
diff --git a/lm/builder/pipeline.cc b/lm/builder/pipeline.cc
index 69972e278..64e30f724 100644
--- a/lm/builder/pipeline.cc
+++ b/lm/builder/pipeline.cc
@@ -191,6 +191,9 @@ class Master {
       chains_.clear();
       std::cerr << "Chain sizes:";
       for (std::size_t i = 0; i < config_.order; ++i) {
+        // Always have enough for at least one record.
+        // This was crashing if e.g. there was no 5-gram.
+        assignments[i] = std::max(assignments[i], block_count[i] * NGram<BuildingPayload>::TotalSize(i + 1));
         std::cerr << ' ' << (i+1) << ":" << assignments[i];
         chains_.push_back(util::stream::ChainConfig(NGram<BuildingPayload>::TotalSize(i + 1), block_count[i], assignments[i]));
       }
diff --git a/lm/filter/CMakeLists.txt b/lm/filter/CMakeLists.txt
index 4e791cef8..d4616cc32 100644
--- a/lm/filter/CMakeLists.txt
+++ b/lm/filter/CMakeLists.txt
@@ -52,7 +52,7 @@ foreach(exe ${EXE_LIST})
 	add_executable(${exe} ${exe}_main.cc $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_filter> $<TARGET_OBJECTS:kenlm_util>)
 
   # Link the executable against boost
-  target_link_libraries(${exe} ${Boost_LIBRARIES})
+  target_link_libraries(${exe} ${Boost_LIBRARIES} pthread)
 
   # Group executables together
   set_target_properties(${exe} PROPERTIES FOLDER executables)
diff --git a/lm/kenlm_benchmark_main.cc b/lm/kenlm_benchmark_main.cc
index d8b659139..1704ec6c9 100644
--- a/lm/kenlm_benchmark_main.cc
+++ b/lm/kenlm_benchmark_main.cc
@@ -31,9 +31,7 @@ template <class Model, class Width> void QueryFromBytes(const Model &model, int
   Width kEOS = model.GetVocabulary().EndSentence();
   Width buf[4096];
   float sum = 0.0;
-  while (true) {
-    std::size_t got = util::ReadOrEOF(fd_in, buf, sizeof(buf));
-    if (!got) break;
+  while (std::size_t got = util::ReadOrEOF(fd_in, buf, sizeof(buf))) {
     UTIL_THROW_IF2(got % sizeof(Width), "File size not a multiple of vocab id size " << sizeof(Width));
     got /= sizeof(Width);
     // Do even stuff first.