SourceLabelsPhraseProperty

3 files changed, 198 insertions, 0 deletions

diff --git a/moses/PP/Factory.cpp b/moses/PP/Factory.cpp
index a7e06b392..0cc891a82 100644
--- a/moses/PP/Factory.cpp
+++ b/moses/PP/Factory.cpp
@@ -5,6 +5,7 @@
 #include <vector>
 
 #include "moses/PP/CountsPhraseProperty.h"
+#include "moses/PP/SourceLabelsPhraseProperty.h"
 #include "moses/PP/TreeStructurePhraseProperty.h"
 
 namespace Moses
@@ -52,6 +53,7 @@ PhrasePropertyFactory::PhrasePropertyFactory()
 #define MOSES_PNAME2(name, type) Add(name, new DefaultPhrasePropertyCreator< type >());
 
   MOSES_PNAME2("Counts", CountsPhraseProperty);
+  MOSES_PNAME2("SourceLabels", SourceLabelsPhraseProperty);
   MOSES_PNAME2("Tree",TreeStructurePhraseProperty);
 
 }
diff --git a/moses/PP/SourceLabelsPhraseProperty.cpp b/moses/PP/SourceLabelsPhraseProperty.cpp
new file mode 100644
index 000000000..6f18480c5
--- /dev/null
+++ b/moses/PP/SourceLabelsPhraseProperty.cpp
@@ -0,0 +1,125 @@
+#include "moses/PP/SourceLabelsPhraseProperty.h"
+#include <iostream>
+#include <cstdio>
+#include <cstdlib>
+#include <sstream>
+#include <string>
+#include <queue>
+#include <assert.h>
+#include <limits>
+#include "util/exception.hh"
+
+namespace Moses
+{
+
+void SourceLabelsPhraseProperty::ProcessValue()
+{
+  std::istringstream tokenizer(m_value);
+
+  if (! (tokenizer >> m_nNTs)) { // first token: number of non-terminals (incl. left-hand side)
+    UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read number of non-terminals. Flawed property?");
+  }
+  assert( m_nNTs > 0 );
+
+  if (! (tokenizer >> m_totalCount)) { // second token: overall rule count
+    UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read overall rule count. Flawed property?");
+  }
+  assert( m_totalCount > 0.0 );
+
+
+
+  // read source-labelled rule items 
+
+  std::priority_queue<float> ruleLabelledCountsPQ;
+
+  while (tokenizer.peek() != EOF) {
+    try {
+
+      SourceLabelsPhrasePropertyItem item;
+      size_t numberOfLHSsGivenRHS = std::numeric_limits<std::size_t>::max();
+
+      if (m_nNTs == 1) {
+
+        item.m_sourceLabelsRHSCount = m_totalCount;
+
+      } else { // rule has right-hand side non-terminals, i.e. it's a hierarchical rule
+
+        for (size_t i=0; i<m_nNTs-1; ++i) { // RHS source non-terminal labels
+          size_t sourceLabelRHS;
+          if (! (tokenizer >> sourceLabelRHS) ) { // RHS source non-terminal label
+            UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read right-hand side label index. Flawed property?");
+          }
+          item.m_sourceLabelsRHS.push_back(sourceLabelRHS);
+        }
+
+        if (! (tokenizer >> item.m_sourceLabelsRHSCount)) {
+          UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read right-hand side count. Flawed property?");
+        }
+
+        if (! (tokenizer >> numberOfLHSsGivenRHS)) {
+          UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read number of left-hand sides. Flawed property?");
+        }
+      }
+
+      for (size_t i=0; i<numberOfLHSsGivenRHS && tokenizer.peek()!=EOF; ++i) { // LHS source non-terminal labels seen with this RHS
+        size_t sourceLabelLHS;
+        if (! (tokenizer >> sourceLabelLHS)) { // LHS source non-terminal label
+          UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read left-hand side label index. Flawed property?");
+        }
+        float ruleSourceLabelledCount;
+        if (! (tokenizer >> ruleSourceLabelledCount)) {
+          UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read count. Flawed property?");
+        }
+        item.m_sourceLabelsLHSList.push_back( std::make_pair(sourceLabelLHS,ruleSourceLabelledCount) );
+        ruleLabelledCountsPQ.push(ruleSourceLabelledCount);
+      }
+
+      m_sourceLabelItems.push_back(item);
+
+    } catch (const std::exception &e) {
+      UTIL_THROW2("SourceLabelsPhraseProperty: Read error. Flawed property?");
+    }
+  }
+
+  // keep only top N label vectors
+  const size_t N=50;
+
+  if (ruleLabelledCountsPQ.size() > N) {
+
+    float topNRuleLabelledCount = std::numeric_limits<int>::max();
+    for (size_t i=0; !ruleLabelledCountsPQ.empty() && i<N; ++i) {
+      topNRuleLabelledCount = ruleLabelledCountsPQ.top();
+      ruleLabelledCountsPQ.pop();
+    }
+
+    size_t nKept=0;
+    std::list<SourceLabelsPhrasePropertyItem>::iterator itemIter=m_sourceLabelItems.begin();
+    while (itemIter!=m_sourceLabelItems.end()) {
+      if (itemIter->m_sourceLabelsRHSCount < topNRuleLabelledCount) {
+        itemIter = m_sourceLabelItems.erase(itemIter);
+      } else {
+        std::list< std::pair<size_t,float> >::iterator itemLHSIter=(itemIter->m_sourceLabelsLHSList).begin();
+        while (itemLHSIter!=(itemIter->m_sourceLabelsLHSList).end()) {
+          if (itemLHSIter->second < topNRuleLabelledCount) {
+            itemLHSIter = (itemIter->m_sourceLabelsLHSList).erase(itemLHSIter);
+          } else {
+            if (nKept >= N) {
+              itemLHSIter = (itemIter->m_sourceLabelsLHSList).erase(itemLHSIter,(itemIter->m_sourceLabelsLHSList).end());
+            } else {
+              ++nKept;
+              ++itemLHSIter;
+            }
+          }
+        }
+        if ((itemIter->m_sourceLabelsLHSList).empty()) {
+          itemIter = m_sourceLabelItems.erase(itemIter);
+        } else {
+          ++itemIter;
+        }
+      }
+    }
+  }
+};
+
+} // namespace Moses
+
diff --git a/moses/PP/SourceLabelsPhraseProperty.h b/moses/PP/SourceLabelsPhraseProperty.h
new file mode 100644
index 000000000..049d88f17
--- /dev/null
+++ b/moses/PP/SourceLabelsPhraseProperty.h
@@ -0,0 +1,71 @@
+
+#pragma once
+
+#include "moses/PP/PhraseProperty.h"
+#include <string>
+#include <list>
+
+namespace Moses
+{
+
+// Note that we require label tokens (strings) in the corresponding property values of phrase table entries 
+// to be replaced beforehand by indices (size_t) of a label vocabulary. (TODO: change that?)
+
+class SourceLabelsPhrasePropertyItem
+{
+friend class SourceLabelsPhraseProperty;
+
+public:
+  SourceLabelsPhrasePropertyItem() {};
+
+  float GetSourceLabelsRHSCount() const
+  {
+    return m_sourceLabelsRHSCount;
+  };
+
+  const std::list<size_t> &GetSourceLabelsRHS() const
+  {
+    return m_sourceLabelsRHS;
+  };
+
+  const std::list< std::pair<size_t,float> > &GetSourceLabelsLHSList() const
+  {
+    return m_sourceLabelsLHSList;
+  };
+
+private:
+  float m_sourceLabelsRHSCount;
+  std::list<size_t> m_sourceLabelsRHS; // should be of size nNTs-1 (empty if initial rule, i.e. no right-hand side non-terminals)
+  std::list< std::pair<size_t,float> > m_sourceLabelsLHSList; // list of left-hand sides for this right-hand side, with counts
+};
+
+
+class SourceLabelsPhraseProperty : public PhraseProperty
+{
+public:
+  SourceLabelsPhraseProperty(const std::string &value) :  PhraseProperty(value) {};
+
+  virtual void ProcessValue();
+
+  size_t GetNumberOfNonTerminals() const {
+    return m_nNTs;
+  }
+
+  float GetTotalCount() const {
+    return m_totalCount;
+  }
+
+  const std::list<SourceLabelsPhrasePropertyItem> &GetSourceLabelItems() const {
+    return m_sourceLabelItems;
+  };
+
+protected:
+
+  size_t m_nNTs;
+  float m_totalCount;
+
+  std::list<SourceLabelsPhrasePropertyItem> m_sourceLabelItems;
+};
+
+} // namespace Moses
+