added support tp specify translation options via xml for chart decoder

4 files changed, 131 insertions, 10 deletions

diff --git a/moses/src/ChartManager.cpp b/moses/src/ChartManager.cpp
index a0737ffcb..79a181952 100644
--- a/moses/src/ChartManager.cpp
+++ b/moses/src/ChartManager.cpp
@@ -29,6 +29,7 @@
 #include "ChartTrellisPathList.h"
 #include "StaticData.h"
 #include "DecodeStep.h"
+#include "TreeInput.h"
 
 using namespace std;
 using namespace Moses;
@@ -79,6 +80,8 @@ void ChartManager::ProcessSentence()
   VERBOSE(2,"Decoding: " << endl);
   //ChartHypothesis::ResetHypoCount();
 
+  AddXmlChartOptions();
+
   // MAIN LOOP
   size_t size = m_source.GetSize();
   for (size_t width = 1; width <= size; ++width) {
@@ -122,6 +125,28 @@ void ChartManager::ProcessSentence()
   }
 }
 
+void ChartManager::AddXmlChartOptions() {
+  TreeInput const &source = dynamic_cast<TreeInput const&>(m_source);
+  const std::vector <ChartTranslationOption*> xmlChartOptionsList = source.GetXmlChartTranslationOptions();
+  IFVERBOSE(2) { cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl; }
+  if (xmlChartOptionsList.size() == 0) return;
+
+  for(std::vector<ChartTranslationOption*>::const_iterator i = xmlChartOptionsList.begin();
+      i != xmlChartOptionsList.end(); ++i) {
+    ChartTranslationOption* opt = *i;
+
+    Moses::Scores wordPenaltyScore(1, -0.434294482); // TODO what is this number?
+    opt->GetTargetPhraseCollection().GetCollection()[0]->SetScore((ScoreProducer*)m_system->GetWordPenaltyProducer(), wordPenaltyScore);
+
+    const WordsRange &range = opt->GetSourceWordsRange();
+    RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
+    ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
+    hypo->CalcScore();
+    ChartCell &cell = m_hypoStackColl.Get(range);
+    cell.AddHypothesis(hypo);
+  }
+}
+
 const ChartHypothesis *ChartManager::GetBestHypothesis() const
 {
   size_t size = m_source.GetSize();
diff --git a/moses/src/ChartManager.h b/moses/src/ChartManager.h
index dbcd42b6c..d59975c61 100644
--- a/moses/src/ChartManager.h
+++ b/moses/src/ChartManager.h
@@ -65,6 +65,7 @@ public:
   ChartManager(InputType const& source, const TranslationSystem* system);
   ~ChartManager();
   void ProcessSentence();
+  void AddXmlChartOptions();
   const ChartHypothesis *GetBestHypothesis() const;
   void CalcNBest(size_t count, ChartTrellisPathList &ret,bool onlyDistinct=0) const;
 
diff --git a/moses/src/TreeInput.cpp b/moses/src/TreeInput.cpp
index d6e0f1e78..a0b04b800 100644
--- a/moses/src/TreeInput.cpp
+++ b/moses/src/TreeInput.cpp
@@ -20,7 +20,7 @@ namespace Moses
  * \param reorderingConstraint reordering constraint zones specified by xml
  * \param walls reordering constraint walls specified by xml
  */
-bool TreeInput::ProcessAndStripXMLTags(string &line, std::vector<XMLParseOutput> &sourceLabels)
+bool TreeInput::ProcessAndStripXMLTags(string &line, std::vector<XMLParseOutput> &sourceLabels, std::vector<XmlOption*> &xmlOptions)
 {
   //parse XML markup in translation line
 
@@ -41,6 +41,10 @@ bool TreeInput::ProcessAndStripXMLTags(string &line, std::vector<XMLParseOutput>
   string cleanLine; // return string (text without xml)
   size_t wordPos = 0; // position in sentence (in terms of number of words)
 
+  // keep this handy for later
+  const vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
+  const string &factorDelimiter = StaticData::Instance().GetFactorDelimiter();
+
   // loop through the tokens
   for (size_t xmlTokenPos = 0 ; xmlTokenPos < xmlTokens.size() ; xmlTokenPos++) {
     // not a xml tag, but regular text (may contain many words)
@@ -145,14 +149,63 @@ bool TreeInput::ProcessAndStripXMLTags(string &line, std::vector<XMLParseOutput>
           return false;
         }
 
-        WordsRange range(startPos,endPos-1);
-        // specified translations -> vector of phrases
-        // multiple translations may be specified, separated by "||"
-        vector<string> altTexts = TokenizeMultiCharSeparator(ParseXmlTagAttribute(tagContent,"label"), "||");
-        CHECK(altTexts.size() == 1);
+	// may be either a input span label ("label"), or a specified output translation "translation"
+        string label = ParseXmlTagAttribute(tagContent,"label");
+        string translation = ParseXmlTagAttribute(tagContent,"translation");
+
+        // specified label
+        if (translation.length() == 0 && label.length() > 0) {
+          WordsRange range(startPos,endPos-1); // really?
+          XMLParseOutput item(label, range);
+          sourceLabels.push_back(item);
+        }
 
-        XMLParseOutput item(altTexts[0], range);
-        sourceLabels.push_back(item);
+        // specified translations -> vector of phrases, separated by "||"
+        if (translation.length() > 0 && StaticData::Instance().GetXmlInputType() != XmlIgnore) {
+          vector<string> altTexts = TokenizeMultiCharSeparator(translation, "||");
+          vector<string> altLabel = TokenizeMultiCharSeparator(label, "||");
+          vector<string> altProbs = TokenizeMultiCharSeparator(ParseXmlTagAttribute(tagContent,"prob"), "||");
+	  //TRACE_ERR("number of translations: " << altTexts.size() << endl);
+          for (size_t i=0; i<altTexts.size(); ++i) {
+            // set target phrase
+            TargetPhrase targetPhrase(Output);
+            targetPhrase.CreateFromString(outputFactorOrder,altTexts[i],factorDelimiter);
+
+            // set constituent label
+	    string targetLHSstr;
+            if (altLabel.size() > i && altLabel[i].size() > 0) {
+              targetLHSstr = altLabel[i];
+            }
+            else {
+              const UnknownLHSList &lhsList = StaticData::Instance().GetUnknownLHS();
+              UnknownLHSList::const_iterator iterLHS = lhsList.begin();
+              targetLHSstr = iterLHS->first;
+            }
+            Word targetLHS(true);
+            targetLHS.CreateFromString(Output, outputFactorOrder, targetLHSstr, true);
+            CHECK(targetLHS.GetFactor(0) != NULL);
+            targetPhrase.SetTargetLHS(targetLHS);
+
+            // get probability
+            float probValue = 1;
+            if (altProbs.size() > i && altProbs[i].size() > 0) {
+              probValue = Scan<float>(altProbs[i]);
+            }
+            // convert from prob to log-prob
+            float scoreValue = FloorScore(TransformScore(probValue));
+            targetPhrase.SetScore(scoreValue);
+
+            // set span and create XmlOption
+            WordsRange range(startPos+1,endPos);
+            XmlOption *option = new XmlOption(range,targetPhrase);
+            CHECK(option);
+            xmlOptions.push_back(option);
+
+            VERBOSE(2,"xml translation = [" << range << "] " << targetLHSstr << " -> " << altTexts[i] << " prob: " << probValue << endl);
+          }
+          altTexts.clear();
+          altProbs.clear();
+        }
       }
     }
   }
@@ -179,7 +232,8 @@ int TreeInput::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
   //line = Trim(line);
 
   std::vector<XMLParseOutput> sourceLabels;
-  ProcessAndStripXMLTags(line, sourceLabels);
+  std::vector<XmlOption*> xmlOptionsList;
+  ProcessAndStripXMLTags(line, sourceLabels, xmlOptionsList);
 
   // do words 1st - hack
   stringstream strme;
@@ -211,6 +265,42 @@ int TreeInput::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
     }
   }
 
+  // XML Options
+
+  //only fill the vector if we are parsing XML
+  if (staticData.GetXmlInputType() != XmlPassThrough ) {
+    //TODO: needed to handle exclusive
+    //for (size_t i=0; i<GetSize(); i++) {
+    //  m_xmlCoverageMap.push_back(false);
+    //}
+
+    //iterXMLOpts will be empty for XmlIgnore
+    //look at each column
+    for(std::vector<XmlOption*>::const_iterator iterXmlOpts = xmlOptionsList.begin();
+        iterXmlOpts != xmlOptionsList.end(); iterXmlOpts++) {
+
+      const XmlOption *xmlOption = *iterXmlOpts;
+      TargetPhrase *targetPhrase = new TargetPhrase(xmlOption->targetPhrase);
+      *targetPhrase = xmlOption->targetPhrase; // copy everything
+      WordsRange *range = new WordsRange(xmlOption->range);
+      const StackVec emptyStackVec; // hmmm... maybe dangerous, but it is never consulted
+
+      TargetPhraseCollection *tpc = new TargetPhraseCollection;
+      tpc->Add(targetPhrase);
+
+      ChartTranslationOption *transOpt = new ChartTranslationOption(*tpc, emptyStackVec, *range, 0.0f);
+      m_xmlChartOptionsList.push_back(transOpt);
+
+      //TODO: needed to handle exclusive
+      //for(size_t j=transOpt->GetSourceWordsRange().GetStartPos(); j<=transOpt->GetSourceWordsRange().GetEndPos(); j++) {
+      //  m_xmlCoverageMap[j]=true;
+      //}
+
+      delete xmlOption;
+    }
+
+  }
+
   return 1;
 }
 
diff --git a/moses/src/TreeInput.h b/moses/src/TreeInput.h
index a6242d97e..e460ad6b3 100644
--- a/moses/src/TreeInput.h
+++ b/moses/src/TreeInput.h
@@ -4,6 +4,7 @@
 
 #include <vector>
 #include "Sentence.h"
+#include "ChartTranslationOption.h"
 
 namespace Moses
 {
@@ -25,6 +26,7 @@ class TreeInput : public Sentence
 
 protected:
   std::vector<std::vector<NonTerminalSet> > m_sourceChart;
+  std::vector <ChartTranslationOption*> m_xmlChartOptionsList;
 
   void AddChartLabel(size_t startPos, size_t endPos, const std::string &label
                      ,const std::vector<FactorType>& factorOrder);
@@ -34,7 +36,7 @@ protected:
     return m_sourceChart[startPos][endPos - startPos];
   }
 
-  bool ProcessAndStripXMLTags(std::string &line, std::vector<XMLParseOutput> &sourceLabels);
+  bool ProcessAndStripXMLTags(std::string &line, std::vector<XMLParseOutput> &sourceLabels, std::vector<XmlOption*> &res);
 
 public:
   TreeInput()
@@ -57,6 +59,9 @@ public:
     return m_sourceChart[startPos][endPos - startPos];
   }
 
+  std::vector <ChartTranslationOption*> GetXmlChartTranslationOptions() const {
+    return m_xmlChartOptionsList;
+  };
 };
 
 }