diff options
Diffstat (limited to 'lm/read_arpa.cc')
-rw-r--r-- | lm/read_arpa.cc | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/lm/read_arpa.cc b/lm/read_arpa.cc index 4723ab3a2..9ea087985 100644 --- a/lm/read_arpa.cc +++ b/lm/read_arpa.cc @@ -46,8 +46,14 @@ uint64_t ReadCount(const std::string &from) { void ReadARPACounts(util::FilePiece &in, std::vector<uint64_t> &number) { number.clear(); - StringPiece line; - while (IsEntirelyWhiteSpace(line = in.ReadLine())) {} + StringPiece line = in.ReadLine(); + // In general, ARPA files can have arbitrary text before "\data\" + // But in KenLM, we require such lines to start with "#", so that + // we can do stricter error checking + while (IsEntirelyWhiteSpace(line) || line.starts_with("#")) { + line = in.ReadLine(); + } + if (line != "\\data\\") { if ((line.size() >= 2) && (line.data()[0] == 0x1f) && (static_cast<unsigned char>(line.data()[1]) == 0x8b)) { UTIL_THROW(FormatLoadException, "Looks like a gzip file. If this is an ARPA file, pipe " << in.FileName() << " through zcat. If this already in binary format, you need to decompress it because mmap doesn't work on top of gzip."); |