diff options
author | Tetsuo Kiso <tetsuo-s@is.naist.jp> | 2012-03-19 22:43:04 +0400 |
---|---|---|
committer | Tetsuo Kiso <tetsuo-s@is.naist.jp> | 2012-03-19 22:43:04 +0400 |
commit | 525f06452cf84a0d04dcfd38d7368b9c1f0231e4 (patch) | |
tree | e157b02445380e3b726d4a1784911b244aef35f1 /mert/Ngram.h | |
parent | 2b28072f7a9e651a0e87fb84ec5c8956e2bbed47 (diff) |
Change the Encoder class to Vocabulary.
- Introduce the namespace to avoid naming collisions. The class name
is used in KenLM.
- Add the unit test.
Diffstat (limited to 'mert/Ngram.h')
-rw-r--r-- | mert/Ngram.h | 55 |
1 files changed, 0 insertions, 55 deletions
diff --git a/mert/Ngram.h b/mert/Ngram.h index d2c5f3932..4b64c327b 100644 --- a/mert/Ngram.h +++ b/mert/Ngram.h @@ -5,61 +5,6 @@ #include <map> #include <string> -/** - * A map to manage vocaburaries. - */ -class Encoder { - public: - typedef std::map<std::string, int>::iterator iterator; - typedef std::map<std::string, int>::const_iterator const_iterator; - - Encoder() {} - virtual ~Encoder() {} - - /** Returns the assiged id for given "token". */ - int Encode(const std::string& token) { - iterator it = m_vocab.find(token); - int encoded_token; - if (it == m_vocab.end()) { - // Add an new entry to the vocaburary. - encoded_token = static_cast<int>(m_vocab.size()); - m_vocab[token] = encoded_token; - } else { - encoded_token = it->second; - } - return encoded_token; - } - - /** - * Return true iff the specified "str" is found in the container. - */ - bool Lookup(const std::string&str , int* v) const { - const_iterator it = m_vocab.find(str); - if (it == m_vocab.end()) return false; - *v = it->second; - return true; - } - - void clear() { m_vocab.clear(); } - - bool empty() const { return m_vocab.empty(); } - - size_t size() const { return m_vocab.size(); } - - iterator find(const std::string& str) { return m_vocab.find(str); } - const_iterator find(const std::string& str) const { return m_vocab.find(str); } - - int& operator[](const std::string& str) { return m_vocab[str]; } - - iterator begin() { return m_vocab.begin(); } - const_iterator begin() const { return m_vocab.begin(); } - iterator end() { return m_vocab.end(); } - const_iterator end() const { return m_vocab.end(); } - - private: - std::map<std::string, int> m_vocab; -}; - /** A simple STL-std::map based n-gram counts. Basically, we provide * typical accessors and mutaors, but we intentionally does not allow * erasing elements. |