diff options
author | bhaddow <bhaddow@1f5c12ca-751b-0410-a591-d2e778427230> | 2011-10-14 11:40:53 +0400 |
---|---|---|
committer | bhaddow <bhaddow@1f5c12ca-751b-0410-a591-d2e778427230> | 2011-10-14 11:40:53 +0400 |
commit | 718c55d3aa8fcec6a610f9e8787333e178cb2223 (patch) | |
tree | e8a07e4c67c8fce683c696834b4d87b642565fa6 /mert/FeatureStats.cpp | |
parent | 57bf51fd05ce79ad4e776f59280cd06315001029 (diff) |
Reducing memory usage for sparse features
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@4355 1f5c12ca-751b-0410-a591-d2e778427230
Diffstat (limited to 'mert/FeatureStats.cpp')
-rw-r--r-- | mert/FeatureStats.cpp | 77 |
1 files changed, 72 insertions, 5 deletions
diff --git a/mert/FeatureStats.cpp b/mert/FeatureStats.cpp index 738d27121..719f9ae3e 100644 --- a/mert/FeatureStats.cpp +++ b/mert/FeatureStats.cpp @@ -6,11 +6,80 @@ * */ +#include <cmath> #include <fstream> #include "FeatureStats.h" #define AVAILABLE_ 8; +SparseVector::name2id_t SparseVector::name2id_; +SparseVector::id2name_t SparseVector::id2name_; + +FeatureStatsType SparseVector::get(string name) const { + name2id_t::const_iterator name2id_iter = name2id_.find(name); + if (name2id_iter == name2id_.end()) return 0; + size_t id = name2id_iter->second; + return get(id); + +} + +FeatureStatsType SparseVector::get(size_t id) const { + fvector_t::const_iterator fvector_iter = fvector_.find(id); + if (fvector_iter == fvector_.end()) return 0; + return fvector_iter->second; +} + +void SparseVector::set(string name, FeatureStatsType value) { + name2id_t::const_iterator name2id_iter = name2id_.find(name); + size_t id = 0; + if (name2id_iter == name2id_.end()) { + id = id2name_.size(); + id2name_.push_back(name); + name2id_[name] = id; + } else { + id = name2id_iter->second; + } + fvector_[id] = value; +} + +void SparseVector::write(ostream& out) const { + for (fvector_t::const_iterator i = fvector_.begin(); i != fvector_.end(); ++i) { + if (abs((float)(i->second)) < 0.00001) continue; + string name = id2name_[i->first]; + out << name << " " << i->second << " "; + } +} + +void SparseVector::clear() { + fvector_.clear(); +} + +size_t SparseVector::size() const { + return fvector_.size(); +} + +SparseVector& SparseVector::operator-=(const SparseVector& rhs) { + //All the elements that have values in *this + for (fvector_t::iterator i = fvector_.begin(); i != fvector_.end(); ++i) { + fvector_[i->first] = i->second - rhs.get(i->first); + } + + //Any elements in rhs, that have no value in *this + for (fvector_t::const_iterator i = rhs.fvector_.begin(); + i != rhs.fvector_.end(); ++i) { + if (fvector_.find(i->first) == fvector_.end()) { + fvector_[i->first] = -(i->second); + } + } + return *this; +} + +SparseVector operator-(const SparseVector& lhs, const SparseVector& rhs) { + SparseVector res(lhs); + res -= rhs; + return res; +} + FeatureStats::FeatureStats() { @@ -64,7 +133,7 @@ void FeatureStats::add(FeatureStatsType v) void FeatureStats::addSparse(string name, FeatureStatsType v) { - map_[name]=v; + map_.set(name,v); } void FeatureStats::set(std::string &theString) @@ -151,9 +220,7 @@ ostream& operator<<(ostream& o, const FeatureStats& e) o << e.get(i) << " "; } // sparse features - const sparse_featstats_t &sparse = e.getSparse(); - for(sparse_featstats_t::const_iterator i = sparse.begin(); i != sparse.end(); i++) { - o << i->first << i->second << " "; - } + e.getSparse().write(o); + return o; } |