diff options
author | Kenneth Heafield <github@kheafield.com> | 2014-01-28 04:51:35 +0400 |
---|---|---|
committer | Kenneth Heafield <github@kheafield.com> | 2014-01-28 04:51:35 +0400 |
commit | 14e02978fcbbe9ec6ed3faa232fcb3f30664f40e (patch) | |
tree | 0b0d19a9c1ba1020ae6b3f1230b8b5b9e6a945f9 /lm/search_trie.hh | |
parent | b68a906fdd97c344ce7d6fa90a2db5a79c5853e6 (diff) |
KenLM 5cc905bc2d214efa7de2db56a9a672b749a95591
Avoid unspecified behavior of mmap when a file is resized reported by Christian Hardmeier
Fixes for Mavericks and a workaround for Boost's broken semaphore
Clean clang compile (of kenlm)
Merged some of 744376b3fbebc41c4a270bf549826d5eb9219ae0 but also undid some of it because it was just masking a fundaemntal problem with pread rather than working around windows limitations
Diffstat (limited to 'lm/search_trie.hh')
-rw-r--r-- | lm/search_trie.hh | 18 |
1 files changed, 8 insertions, 10 deletions
diff --git a/lm/search_trie.hh b/lm/search_trie.hh index 763fd1a72..299262a5d 100644 --- a/lm/search_trie.hh +++ b/lm/search_trie.hh @@ -17,13 +17,13 @@ namespace lm { namespace ngram { -struct Backing; +class BinaryFormat; class SortedVocabulary; namespace trie { template <class Quant, class Bhiksha> class TrieSearch; class SortedFiles; -template <class Quant, class Bhiksha> void BuildTrie(SortedFiles &files, std::vector<uint64_t> &counts, const Config &config, TrieSearch<Quant, Bhiksha> &out, Quant &quant, const SortedVocabulary &vocab, Backing &backing); +template <class Quant, class Bhiksha> void BuildTrie(SortedFiles &files, std::vector<uint64_t> &counts, const Config &config, TrieSearch<Quant, Bhiksha> &out, Quant &quant, SortedVocabulary &vocab, BinaryFormat &backing); template <class Quant, class Bhiksha> class TrieSearch { public: @@ -39,11 +39,11 @@ template <class Quant, class Bhiksha> class TrieSearch { static const unsigned int kVersion = 1; - static void UpdateConfigFromBinary(int fd, const std::vector<uint64_t> &counts, Config &config) { - Quant::UpdateConfigFromBinary(fd, counts, config); - util::AdvanceOrThrow(fd, Quant::Size(counts.size(), config) + Unigram::Size(counts[0])); + static void UpdateConfigFromBinary(const BinaryFormat &file, const std::vector<uint64_t> &counts, uint64_t offset, Config &config) { + Quant::UpdateConfigFromBinary(file, offset, config); // Currently the unigram pointers are not compresssed, so there will only be a header for order > 2. - if (counts.size() > 2) Bhiksha::UpdateConfigFromBinary(fd, config); + if (counts.size() > 2) + Bhiksha::UpdateConfigFromBinary(file, offset + Quant::Size(counts.size(), config) + Unigram::Size(counts[0]), config); } static uint64_t Size(const std::vector<uint64_t> &counts, const Config &config) { @@ -60,9 +60,7 @@ template <class Quant, class Bhiksha> class TrieSearch { uint8_t *SetupMemory(uint8_t *start, const std::vector<uint64_t> &counts, const Config &config); - void LoadedBinary(); - - void InitializeFromARPA(const char *file, util::FilePiece &f, std::vector<uint64_t> &counts, const Config &config, SortedVocabulary &vocab, Backing &backing); + void InitializeFromARPA(const char *file, util::FilePiece &f, std::vector<uint64_t> &counts, const Config &config, SortedVocabulary &vocab, BinaryFormat &backing); unsigned char Order() const { return middle_end_ - middle_begin_ + 2; @@ -103,7 +101,7 @@ template <class Quant, class Bhiksha> class TrieSearch { } private: - friend void BuildTrie<Quant, Bhiksha>(SortedFiles &files, std::vector<uint64_t> &counts, const Config &config, TrieSearch<Quant, Bhiksha> &out, Quant &quant, const SortedVocabulary &vocab, Backing &backing); + friend void BuildTrie<Quant, Bhiksha>(SortedFiles &files, std::vector<uint64_t> &counts, const Config &config, TrieSearch<Quant, Bhiksha> &out, Quant &quant, SortedVocabulary &vocab, BinaryFormat &backing); // Middles are managed manually so we can delay construction and they don't have to be copyable. void FreeMiddles() { |