Added posibility to weight words in sempos

1 files changed, 50 insertions, 1 deletions

diff --git a/mert/SemposScorer.cpp b/mert/SemposScorer.cpp
index 5dee29c3a..62a70c510 100644
--- a/mert/SemposScorer.cpp
+++ b/mert/SemposScorer.cpp
@@ -3,20 +3,28 @@
 #include <algorithm>
 #include <vector>
 #include <stdexcept>
+#include <fstream>
 
 #include "Util.h"
+#include "SemposOverlapping.h"
 
 using namespace std;
 
 SemposScorer::SemposScorer(const string& config)
   : StatisticsBasedScorer("SEMPOS", config),
-    m_ovr(SemposOverlappingFactory::GetOverlapping(getConfig("overlapping", "cap-micro"))),
+    m_ovr(SemposOverlappingFactory::GetOverlapping(getConfig("overlapping", "cap-micro"),this)),
     m_enable_debug(false)
 {
   const string& debugSwitch = getConfig("debug", "0");
   if (debugSwitch == "1") m_enable_debug = true;
 
   m_semposMap.clear();
+
+  string weightsfile = getConfig("weightsfile", "");
+  if (weightsfile != "")
+  {
+    loadWeights(weightsfile);
+  }
 }
 
 SemposScorer::~SemposScorer() {}
@@ -128,3 +136,44 @@ int SemposScorer::encodeSempos(const string& sempos)
     return it->second;
   }
 }
+
+float SemposScorer::weight(int item) const
+{
+    std::map<int,float>::const_iterator it = weightsMap.find(item);
+    if (it == weightsMap.end())
+    {
+        return 1.0f;
+    }
+    else
+    {
+        return it->second;
+    }
+}
+
+void SemposScorer::loadWeights(const string& weightsfile)
+{
+    string line;
+    ifstream myfile;
+    myfile.open(weightsfile.c_str(), ifstream::in);
+    if (myfile.is_open())
+    {
+        while ( myfile.good() )
+        {
+            getline (myfile,line);
+            vector<string> fields;
+            if (line == "") continue;
+            split(line, '\t', fields);
+            if (fields.size() != 2) throw std::runtime_error("Bad format of a row in weights file.");
+            int encoded = encodeString(fields[0]);
+            float weight = atof(fields[1].c_str());
+            weightsMap[encoded] = weight;
+        }
+        myfile.close();
+    }
+    else
+    {
+        cerr << "Unable to open file "<< weightsfile << endl;
+        exit(1);
+    }    
+
+}