diff options
author | Kenneth Heafield <github@kheafield.com> | 2013-04-25 22:42:30 +0400 |
---|---|---|
committer | Kenneth Heafield <github@kheafield.com> | 2013-04-25 22:42:30 +0400 |
commit | f1d366381033c0caae18f8d15305ded38734bdbf (patch) | |
tree | 22b0cbd3acc337a995701629bf9facbe179f5618 /moses/FactorCollection.cpp | |
parent | 8a1e944bb428a0af9f6c82c26e5633361ce4052c (diff) |
Back FactorCollection with a memory pool. Less memory for large vocabularies.
Diffstat (limited to 'moses/FactorCollection.cpp')
-rw-r--r-- | moses/FactorCollection.cpp | 34 |
1 files changed, 8 insertions, 26 deletions
diff --git a/moses/FactorCollection.cpp b/moses/FactorCollection.cpp index 849830f4d..969bb39d1 100644 --- a/moses/FactorCollection.cpp +++ b/moses/FactorCollection.cpp @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <string> #include "FactorCollection.h" #include "Util.h" +#include "util/pool.hh" using namespace std; @@ -36,42 +37,23 @@ FactorCollection FactorCollection::s_instance; const Factor *FactorCollection::AddFactor(const StringPiece &factorString) { -// Sorry this is so complicated. Can't we just require everybody to use Boost >= 1.42? The issue is that I can't check BOOST_VERSION unless we have Boost. -#ifdef WITH_THREADS - -#if BOOST_VERSION < 104200 FactorFriend to_ins; - to_ins.in.m_string.assign(factorString.data(), factorString.size()); -#endif // BOOST_VERSION + to_ins.in.m_string = factorString; + to_ins.in.m_id = m_factorId; + // If we're threaded, hope a read-only lock is sufficient. +#ifdef WITH_THREADS { // read=lock scope boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock); -#if BOOST_VERSION >= 104200 - // If this line doesn't compile, upgrade your Boost. - Set::const_iterator i = m_set.find(factorString, HashFactor(), EqualsFactor()); -#else // BOOST_VERSION Set::const_iterator i = m_set.find(to_ins); -#endif // BOOST_VERSION if (i != m_set.end()) return &i->in; } boost::unique_lock<boost::shared_mutex> lock(m_accessLock); -#if BOOST_VERSION >= 104200 - FactorFriend to_ins; - to_ins.in.m_string.assign(factorString.data(), factorString.size()); -#endif // BOOST_VERSION - -#else // WITH_THREADS - -#if BOOST_VERSION >= 104200 - Set::const_iterator i = m_set.find(factorString, HashFactor(), EqualsFactor()); - if (i != m_set.end()) return &i->in; -#endif - FactorFriend to_ins; - to_ins.in.m_string.assign(factorString.data(), factorString.size()); - #endif // WITH_THREADS - to_ins.in.m_id = m_factorId; std::pair<Set::iterator, bool> ret(m_set.insert(to_ins)); if (ret.second) { + ret.first->in.m_string.set( + memcpy(m_string_backing.Allocate(factorString.size()), factorString.data(), factorString.size()), + factorString.size()); m_factorId++; } return &ret.first->in; |