Merge from miramerge.

Reverted ChartHypothesis as it breaks chart-decoding.

3 files changed, 84 insertions, 35 deletions

diff --git a/moses-chart-cmd/src/IOWrapper.cpp b/moses-chart-cmd/src/IOWrapper.cpp
index cf90b877b..ce31bc192 100644
--- a/moses-chart-cmd/src/IOWrapper.cpp
+++ b/moses-chart-cmd/src/IOWrapper.cpp
@@ -47,6 +47,9 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "ChartHypothesis.h"
 #include "DotChart.h"
 
+#include <boost/algorithm/string.hpp>
+#include "FeatureVector.h"
+
 
 using namespace std;
 using namespace Moses;
@@ -345,7 +348,7 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha
     // print the surface factor of the translation
     out << translationId << " ||| ";
     OutputSurface(out, outputPhrase, m_outputFactorOrder, false);
-    out << " |||";
+    out << " ||| ";
 
     // print the scores in a hardwired order
     // before each model type, the corresponding command-line-like name must be emitted
@@ -362,26 +365,23 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha
       }
     }
 
-
     std::string lastName = "";
 
     // translation components
     const vector<PhraseDictionaryFeature*>& pds = system->GetPhraseDictionaries();
     if (pds.size() > 0) {
-
       for( size_t i=0; i<pds.size(); i++ ) {
-	size_t pd_numinputscore = pds[i]->GetNumInputScores();
-	vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( pds[i] );
-	for (size_t j = 0; j<scores.size(); ++j){
-
-	  if (labeledOutput && (i == 0) ){
-	    if ((j == 0) || (j == pd_numinputscore)){
-	      lastName =  pds[i]->GetScoreProducerWeightShortName(j);
-	      out << " " << lastName << ":";
-	    }
-	  }
-	  out << " " << scores[j];
-	}
+      	size_t pd_numinputscore = pds[i]->GetNumInputScores();
+      	vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( pds[i] );
+      	for (size_t j = 0; j<scores.size(); ++j){
+      		if (labeledOutput && (i == 0) ){
+      			if ((j == 0) || (j == pd_numinputscore)){
+      				lastName =  pds[i]->GetScoreProducerWeightShortName(j);
+      				out << " " << lastName << ":";
+      			}
+      		}
+      		out << " " << scores[j];
+      	}
       }
     }
 
@@ -393,26 +393,36 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha
     // generation
     const vector<GenerationDictionary*>& gds = system->GetGenerationDictionaries();
     if (gds.size() > 0) {
-
       for( size_t i=0; i<gds.size(); i++ ) {
-	size_t pd_numinputscore = gds[i]->GetNumInputScores();
-	vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( gds[i] );
-	for (size_t j = 0; j<scores.size(); ++j){
-
-	  if (labeledOutput && (i == 0) ){
-	    if ((j == 0) || (j == pd_numinputscore)){
-	      lastName =  gds[i]->GetScoreProducerWeightShortName(j);
-	      out << " " << lastName << ":";
-	    }
-	  }
-	  out << " " << scores[j];
-	}
+      	size_t pd_numinputscore = gds[i]->GetNumInputScores();
+      	vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( gds[i] );
+      	for (size_t j = 0; j<scores.size(); ++j){
+      		if (labeledOutput && (i == 0) ){
+      			if ((j == 0) || (j == pd_numinputscore)){
+      				lastName =  gds[i]->GetScoreProducerWeightShortName(j);
+      				out << " " << lastName << ":";
+      			}
+      		}
+      		out << " " << scores[j];
+      	}
       }
     }
 
+    // output sparse features
+    lastName = "";
+    const vector<const StatefulFeatureFunction*>& sff = system->GetStatefulFeatureFunctions();
+    for( size_t i=0; i<sff.size(); i++ )
+    	if (sff[i]->GetNumScoreComponents() == ScoreProducer::unlimited)
+    		OutputSparseFeatureScores( out, path, sff[i], lastName );
+
+    const vector<const StatelessFeatureFunction*>& slf = system->GetStatelessFeatureFunctions();
+    for( size_t i=0; i<slf.size(); i++ )
+    	if (sff[i]->GetNumScoreComponents() == ScoreProducer::unlimited)
+    		OutputSparseFeatureScores( out, path, slf[i], lastName );
+
 
     // total
-    out << " |||" << path.GetTotalScore();
+    out << " ||| " << path.GetTotalScore();
 
     /*
     if (includeAlignment) {
@@ -443,6 +453,32 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha
   m_nBestOutputCollector->Write(translationId, out.str());
 }
 
+void IOWrapper::OutputSparseFeatureScores( std::ostream& out, const ChartTrellisPath &path, const FeatureFunction *ff, std::string &lastName )
+{
+  const StaticData &staticData = StaticData::Instance();
+  bool labeledOutput = staticData.IsLabeledNBestList();
+  const FVector scores = path.GetScoreBreakdown().GetVectorForProducer( ff );
+
+  // report weighted aggregate
+  if (! ff->GetSparseFeatureReporting()) {
+  	const FVector &weights = staticData.GetAllWeights().GetScoresVector();
+  	if (labeledOutput && !boost::contains(ff->GetScoreProducerDescription(), ":"))
+  		out << " " << ff->GetScoreProducerWeightShortName() << ":";
+    out << " " << scores.inner_product(weights);
+  }
+
+  // report each feature
+  else {
+  	for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++) {
+  		if (i->second != 0) { // do not report zero-valued features
+  			if (labeledOutput)
+  				out << " " << i->first << ":";
+        out << " " << i->second;
+      }
+    }
+  }
+}
+
 void IOWrapper::FixPrecision(std::ostream &stream, size_t size)
 {
   stream.setf(std::ios::fixed);
diff --git a/moses-chart-cmd/src/IOWrapper.h b/moses-chart-cmd/src/IOWrapper.h
index 5936e7405..058ee0712 100644
--- a/moses-chart-cmd/src/IOWrapper.h
+++ b/moses-chart-cmd/src/IOWrapper.h
@@ -44,6 +44,8 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "OutputCollector.h"
 #include "ChartHypothesis.h"
 
+#include "ChartTrellisPath.h"
+
 namespace Moses
 {
 class FactorCollection;
@@ -82,6 +84,7 @@ public:
   void OutputBestHypo(const Moses::ChartHypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors);
   void OutputBestHypo(const std::vector<const Moses::Factor*>&  mbrBestHypo, long translationId, bool reportSegmentation, bool reportAllFactors);
   void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartHypothesis *bestHypo, const Moses::TranslationSystem* system, long translationId);
+  void OutputSparseFeatureScores(std::ostream& out, const Moses::ChartTrellisPath &path, const Moses::FeatureFunction *ff, std::string &lastName);
   void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, long translationId);
   void Backtrack(const Moses::ChartHypothesis *hypo);
 
diff --git a/moses-chart-cmd/src/Main.cpp b/moses-chart-cmd/src/Main.cpp
index 2c9002720..bec974ee8 100644
--- a/moses-chart-cmd/src/Main.cpp
+++ b/moses-chart-cmd/src/Main.cpp
@@ -165,18 +165,25 @@ bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source
 }
 static void PrintFeatureWeight(const FeatureFunction* ff)
 {
-
   size_t numScoreComps = ff->GetNumScoreComponents();
   if (numScoreComps != ScoreProducer::unlimited) {
     vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
-    for (size_t i = 0; i < numScoreComps; ++i) {
+    for (size_t i = 0; i < numScoreComps; ++i) 
       cout << ff->GetScoreProducerDescription() <<  " "
            << ff->GetScoreProducerWeightShortName() << " "
            << values[i] << endl;
-    }
-  } else {
-    cout << ff->GetScoreProducerDescription() << " " <<
-         ff->GetScoreProducerWeightShortName() << " sparse" <<  endl;
+  }
+}
+
+static void PrintSparseFeatureWeight(const FeatureFunction* ff)
+{
+  if (ff->GetNumScoreComponents() == ScoreProducer::unlimited) {
+    if (ff->GetSparseProducerWeight() == 1)
+      cout << ff->GetScoreProducerDescription() << " " <<
+	ff->GetScoreProducerWeightShortName() << " sparse" << endl;
+    else
+      cout << ff->GetScoreProducerDescription() << " " <<
+	ff->GetScoreProducerWeightShortName() << " " << ff->GetSparseProducerWeight() << endl;
   }
 }
 
@@ -201,6 +208,9 @@ static void ShowWeights()
   for (size_t i = 0; i < slf.size(); ++i) {
     PrintFeatureWeight(slf[i]);
   }
+  for (size_t i = 0; i < sff.size(); ++i) {
+    PrintSparseFeatureWeight(sff[i]);
+  }
 }