From 23ca29a2ea13bfa12210f4f278785aefdd45a672 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Fri, 5 Dec 2014 17:59:53 +0000 Subject: add Decode to API framework --- contrib/server/mosesserver.cpp | 4 ++-- mira/Decoder.cpp | 4 ++-- moses-cmd/LatticeMBRGrid.cpp | 2 +- moses/BaseManager.h | 1 + moses/ChartManager.cpp | 46 +++++++++++++++++++++++++++++++++++++++++- moses/ChartManager.h | 7 ++++++- moses/IOWrapper.cpp | 33 ------------------------------ moses/IOWrapper.h | 2 -- moses/Incremental.cpp | 6 +++++- moses/Incremental.h | 4 +++- moses/Manager.cpp | 2 +- moses/Manager.h | 2 +- moses/TranslationTask.cpp | 16 ++++----------- 13 files changed, 71 insertions(+), 58 deletions(-) diff --git a/contrib/server/mosesserver.cpp b/contrib/server/mosesserver.cpp index 129c32a73..a2e7b9183 100644 --- a/contrib/server/mosesserver.cpp +++ b/contrib/server/mosesserver.cpp @@ -283,7 +283,7 @@ public: stringstream in(source + "\n"); tinput.Read(in,inputFactorOrder); ChartManager manager(tinput); - manager.ProcessSentence(); + manager.Decode(); const ChartHypothesis *hypo = manager.GetBestHypothesis(); outputChartHypo(out,hypo); if (addGraphInfo) { @@ -302,7 +302,7 @@ public: stringstream in(source + "\n"); sentence.Read(in,inputFactorOrder); Manager manager(sentence, staticData.GetSearchAlgorithm()); - manager.ProcessSentence(); + manager.Decode(); const Hypothesis* hypo = manager.GetBestHypothesis(); vector alignInfo; diff --git a/mira/Decoder.cpp b/mira/Decoder.cpp index 5f5337312..64b6a2e5f 100644 --- a/mira/Decoder.cpp +++ b/mira/Decoder.cpp @@ -144,7 +144,7 @@ vector< vector > MosesDecoder::runDecoder(const std::string& source { // run the decoder m_manager = new Moses::Manager(*m_sentence, search); - m_manager->ProcessSentence(); + m_manager->Decode(); TrellisPathList nBestList; m_manager->CalcNBest(nBestSize, nBestList, distinct); @@ -221,7 +221,7 @@ vector< vector > MosesDecoder::runChartDecoder(const std::string& s { // run the decoder m_chartManager = new ChartManager(*m_sentence); - m_chartManager->ProcessSentence(); + m_chartManager->Decode(); ChartKBestExtractor::KBestVec nBestList; m_chartManager->CalcNBest(nBestSize, nBestList, distinct); diff --git a/moses-cmd/LatticeMBRGrid.cpp b/moses-cmd/LatticeMBRGrid.cpp index c4faa5776..100cb0e96 100644 --- a/moses-cmd/LatticeMBRGrid.cpp +++ b/moses-cmd/LatticeMBRGrid.cpp @@ -182,7 +182,7 @@ int main(int argc, char* argv[]) source->SetTranslationId(lineCount); Manager manager(*source, staticData.GetSearchAlgorithm()); - manager.ProcessSentence(); + manager.Decode(); TrellisPathList nBestList; manager.CalcNBest(nBestSize, nBestList,true); //grid search diff --git a/moses/BaseManager.h b/moses/BaseManager.h index 11c594d8c..70316274d 100644 --- a/moses/BaseManager.h +++ b/moses/BaseManager.h @@ -46,6 +46,7 @@ protected: } public: + virtual void Decode() = 0; // outputs virtual void OutputNBest(OutputCollector *collector) const = 0; virtual void OutputLatticeSamples(OutputCollector *collector) const = 0; diff --git a/moses/ChartManager.cpp b/moses/ChartManager.cpp index d2269c3bf..f6f5f341c 100644 --- a/moses/ChartManager.cpp +++ b/moses/ChartManager.cpp @@ -64,7 +64,7 @@ ChartManager::~ChartManager() } //! decode the sentence. This contains the main laps. Basically, the CKY++ algorithm -void ChartManager::ProcessSentence() +void ChartManager::Decode() { VERBOSE(1,"Translating: " << m_source << endl); @@ -597,6 +597,18 @@ void ChartManager::OutputDetailedTranslationReport( OutputTranslationOptions(out, applicationContext, hypo, sentence, translationId); collector->Write(translationId, out.str()); + + //DIMw + const StaticData &staticData = StaticData::Instance(); + + if (staticData.IsDetailedAllTranslationReportingEnabled()) { + const Sentence &sentence = dynamic_cast(m_source); + size_t nBestSize = staticData.GetNBestSize(); + std::vector > nBestList; + CalcNBest(nBestSize, nBestList, staticData.GetDistinctNBest()); + OutputDetailedAllTranslationReport(collector, nBestList, sentence, translationId); + } + } void ChartManager::OutputTranslationOptions(std::ostream &out, @@ -753,4 +765,36 @@ void ChartManager::OutputSearchGraph(OutputCollector *collector) const } } +//DIMw +void ChartManager::OutputDetailedAllTranslationReport( + OutputCollector *collector, + const std::vector > &nBestList, + const Sentence &sentence, + long translationId) const +{ + std::ostringstream out; + ApplicationContext applicationContext; + + const ChartCellCollection& cells = GetChartCellCollection(); + size_t size = GetSource().GetSize(); + for (size_t width = 1; width <= size; ++width) { + for (size_t startPos = 0; startPos <= size-width; ++startPos) { + size_t endPos = startPos + width - 1; + WordsRange range(startPos, endPos); + const ChartCell& cell = cells.Get(range); + const HypoList* hyps = cell.GetAllSortedHypotheses(); + out << "Chart Cell [" << startPos << ".." << endPos << "]" << endl; + HypoList::const_iterator iter; + size_t c = 1; + for (iter = hyps->begin(); iter != hyps->end(); ++iter) { + out << "----------------Item " << c++ << " ---------------------" + << endl; + OutputTranslationOptions(out, applicationContext, *iter, + sentence, translationId); + } + } + } + collector->Write(translationId, out.str()); +} + } // namespace Moses diff --git a/moses/ChartManager.h b/moses/ChartManager.h index 8b0cf4ff8..814be1e60 100644 --- a/moses/ChartManager.h +++ b/moses/ChartManager.h @@ -96,11 +96,16 @@ private: const ChartHypothesis *hypo, const Sentence &sentence, long translationId) const; + void OutputDetailedAllTranslationReport( + OutputCollector *collector, + const std::vector > &nBestList, + const Sentence &sentence, + long translationId) const; public: ChartManager(InputType const& source); ~ChartManager(); - void ProcessSentence(); + void Decode(); void AddXmlChartOptions(); const ChartHypothesis *GetBestHypothesis() const; void CalcNBest(size_t n, std::vector > &nBestList, bool onlyDistinct=false) const; diff --git a/moses/IOWrapper.cpp b/moses/IOWrapper.cpp index 9e3e0f57f..ee7b12db5 100644 --- a/moses/IOWrapper.cpp +++ b/moses/IOWrapper.cpp @@ -533,39 +533,6 @@ void IOWrapper::OutputSurface(std::ostream &out, const Phrase &phrase, const std -//DIMw -void IOWrapper::OutputDetailedAllTranslationReport( - const std::vector > &nBestList, - const ChartManager &manager, - const Sentence &sentence, - long translationId) -{ - std::ostringstream out; - ApplicationContext applicationContext; - - const ChartCellCollection& cells = manager.GetChartCellCollection(); - size_t size = manager.GetSource().GetSize(); - for (size_t width = 1; width <= size; ++width) { - for (size_t startPos = 0; startPos <= size-width; ++startPos) { - size_t endPos = startPos + width - 1; - WordsRange range(startPos, endPos); - const ChartCell& cell = cells.Get(range); - const HypoList* hyps = cell.GetAllSortedHypotheses(); - out << "Chart Cell [" << startPos << ".." << endPos << "]" << endl; - HypoList::const_iterator iter; - size_t c = 1; - for (iter = hyps->begin(); iter != hyps->end(); ++iter) { - out << "----------------Item " << c++ << " ---------------------" - << endl; - OutputTranslationOptions(out, applicationContext, *iter, - sentence, translationId); - } - } - } - UTIL_THROW_IF2(m_detailedTranslationCollector == NULL, - "No output file for details specified"); - m_detailedTranslationCollector->Write(translationId, out.str()); -} ////////////////////////////////////////////////////////////////////////// /*** diff --git a/moses/IOWrapper.h b/moses/IOWrapper.h index 3244be44c..59af4600d 100644 --- a/moses/IOWrapper.h +++ b/moses/IOWrapper.h @@ -177,8 +177,6 @@ public: void OutputBestNone(long translationId); - void OutputDetailedAllTranslationReport(const std::vector > &nBestList, const Moses::ChartManager &manager, const Moses::Sentence &sentence, long translationId); - // phrase-based void OutputBestSurface(std::ostream &out, const Moses::Hypothesis *hypo, const std::vector &outputFactorOrder, char reportSegmentation, bool reportAllFactors); void OutputLatticeMBRNBest(std::ostream& out, const std::vector& solutions,long translationId); diff --git a/moses/Incremental.cpp b/moses/Incremental.cpp index bb20d0640..91663d7de 100644 --- a/moses/Incremental.cpp +++ b/moses/Incremental.cpp @@ -273,9 +273,13 @@ template void Manager::LMCallback(const lm::ngram::Qu template void Manager::LMCallback(const lm::ngram::ArrayTrieModel &model, const std::vector &words); template void Manager::LMCallback(const lm::ngram::QuantArrayTrieModel &model, const std::vector &words); -const std::vector &Manager::ProcessSentence() +void Manager::Decode() { LanguageModel::GetFirstLM().IncrementalCallback(*this); +} + +const std::vector &Manager::GetNBest() const +{ return *completed_nbest_; } diff --git a/moses/Incremental.h b/moses/Incremental.h index 0bb8e0b62..ea4b3feda 100644 --- a/moses/Incremental.h +++ b/moses/Incremental.h @@ -30,7 +30,9 @@ public: template void LMCallback(const Model &model, const std::vector &words); - const std::vector &ProcessSentence(); + void Decode(); + + const std::vector &GetNBest() const; // Call to get the same value as ProcessSentence returned. const std::vector &Completed() const { diff --git a/moses/Manager.cpp b/moses/Manager.cpp index 0edb2a09a..0b5f70dc4 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -79,7 +79,7 @@ Manager::~Manager() * Main decoder loop that translates a sentence by expanding * hypotheses stack by stack, until the end of the sentence. */ -void Manager::ProcessSentence() +void Manager::Decode() { // initialize statistics ResetSentenceStats(m_source); diff --git a/moses/Manager.h b/moses/Manager.h index 7a99fe122..ef32c4a9a 100644 --- a/moses/Manager.h +++ b/moses/Manager.h @@ -151,7 +151,7 @@ public: ~Manager(); const TranslationOptionCollection* getSntTranslationOptions(); - void ProcessSentence(); + void Decode(); const Hypothesis *GetBestHypothesis() const; const Hypothesis *GetActualBestHypothesis() const; void CalcNBest(size_t count, TrellisPathList &ret,bool onlyDistinct=0) const; diff --git a/moses/TranslationTask.cpp b/moses/TranslationTask.cpp index 3dd76f55e..e72811377 100644 --- a/moses/TranslationTask.cpp +++ b/moses/TranslationTask.cpp @@ -83,7 +83,7 @@ void TranslationTask::RunPb() initTime.start(); Manager manager(*m_source,staticData.GetSearchAlgorithm()); VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Initialize search took " << initTime << " seconds total" << endl); - manager.ProcessSentence(); + manager.Decode(); // we are done with search, let's look what we got Timer additionalReportingTime; @@ -301,7 +301,8 @@ void TranslationTask::RunChart() if (staticData.GetSearchAlgorithm() == ChartIncremental) { Incremental::Manager manager(*m_source); - const std::vector &nbest = manager.ProcessSentence(); + manager.Decode(); + const std::vector &nbest = manager.GetNBest(); if (!nbest.empty()) { m_ioWrapper.OutputBestHypo(nbest[0], translationId); @@ -318,7 +319,7 @@ void TranslationTask::RunChart() } ChartManager manager(*m_source); - manager.ProcessSentence(); + manager.Decode(); UTIL_THROW_IF2(staticData.UseMBR(), "Cannot use MBR"); @@ -340,15 +341,6 @@ void TranslationTask::RunChart() manager.OutputDetailedTreeFragmentsTranslationReport(m_ioWrapper.GetDetailTreeFragmentsOutputCollector()); manager.OutputUnknowns(m_ioWrapper.GetUnknownsCollector()); - //DIMw - if (staticData.IsDetailedAllTranslationReportingEnabled()) { - const Sentence &sentence = dynamic_cast(*m_source); - size_t nBestSize = staticData.GetNBestSize(); - std::vector > nBestList; - manager.CalcNBest(nBestSize, nBestList, staticData.GetDistinctNBest()); - m_ioWrapper.OutputDetailedAllTranslationReport(nBestList, manager, sentence, translationId); - } - // n-best manager.OutputNBest(m_ioWrapper.GetNBestOutputCollector()); -- cgit v1.2.3