diff options
Diffstat (limited to 'moses')
314 files changed, 179 insertions, 144 deletions
diff --git a/moses/src/AlignmentInfo.cpp b/moses/src/AlignmentInfo.cpp index 959e7591f..959e7591f 100755..100644 --- a/moses/src/AlignmentInfo.cpp +++ b/moses/src/AlignmentInfo.cpp diff --git a/moses/src/AlignmentInfo.h b/moses/src/AlignmentInfo.h index 12c5ee2e9..12c5ee2e9 100755..100644 --- a/moses/src/AlignmentInfo.h +++ b/moses/src/AlignmentInfo.h diff --git a/moses/src/AlignmentInfoCollection.cpp b/moses/src/AlignmentInfoCollection.cpp index aa411fc63..aa411fc63 100755..100644 --- a/moses/src/AlignmentInfoCollection.cpp +++ b/moses/src/AlignmentInfoCollection.cpp diff --git a/moses/src/AlignmentInfoCollection.h b/moses/src/AlignmentInfoCollection.h index a6c90f135..a6c90f135 100755..100644 --- a/moses/src/AlignmentInfoCollection.h +++ b/moses/src/AlignmentInfoCollection.h diff --git a/moses/src/BilingualDynSuffixArray.cpp b/moses/src/BilingualDynSuffixArray.cpp index a2d5e3e90..a2d5e3e90 100755..100644 --- a/moses/src/BilingualDynSuffixArray.cpp +++ b/moses/src/BilingualDynSuffixArray.cpp diff --git a/moses/src/BilingualDynSuffixArray.h b/moses/src/BilingualDynSuffixArray.h index 1543c3709..1543c3709 100755..100644 --- a/moses/src/BilingualDynSuffixArray.h +++ b/moses/src/BilingualDynSuffixArray.h diff --git a/moses/src/BitmapContainer.cpp b/moses/src/BitmapContainer.cpp index c80f3b542..c80f3b542 100755..100644 --- a/moses/src/BitmapContainer.cpp +++ b/moses/src/BitmapContainer.cpp diff --git a/moses/src/BitmapContainer.h b/moses/src/BitmapContainer.h index 987440750..987440750 100755..100644 --- a/moses/src/BitmapContainer.h +++ b/moses/src/BitmapContainer.h diff --git a/moses/src/BleuScoreFeature.cpp b/moses/src/BleuScoreFeature.cpp index a0f4df200..8ba979742 100755..100644 --- a/moses/src/BleuScoreFeature.cpp +++ b/moses/src/BleuScoreFeature.cpp @@ -22,11 +22,28 @@ int BleuScoreState::Compare(const FFState& o) const if (&o == this) return 0; + const StaticData &staticData = StaticData::Instance(); + SearchAlgorithm searchAlgorithm = staticData.GetSearchAlgorithm(); + bool chartDecoding = (searchAlgorithm == ChartDecoding); + if (chartDecoding) + return 0; + const BleuScoreState& other = dynamic_cast<const BleuScoreState&>(o); int c = m_words.Compare(other.m_words); if (c != 0) return c; + /*for(size_t i = 0; i < m_ngram_counts.size(); i++) { + if (m_ngram_counts[i] < other.m_ngram_counts[i]) + return -1; + if (m_ngram_counts[i] > other.m_ngram_counts[i]) + return 1; + if (m_ngram_matches[i] < other.m_ngram_matches[i]) + return -1; + if (m_ngram_matches[i] > other.m_ngram_matches[i]) + return 1; + }*/ + return 0; } std::ostream& operator<<(std::ostream& out, const BleuScoreState& state) { diff --git a/moses/src/BleuScoreFeature.h b/moses/src/BleuScoreFeature.h index fc7aa2e39..5bae3a1fb 100755..100644 --- a/moses/src/BleuScoreFeature.h +++ b/moses/src/BleuScoreFeature.h @@ -123,7 +123,7 @@ public: FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection* accumulator) const; - + bool Enabled() const { return m_enabled; } float CalculateBleu(BleuScoreState*) const; float CalculateBleu(Phrase translation) const; const FFState* EmptyHypothesisState(const InputType&) const; diff --git a/moses/src/CellCollection.h b/moses/src/CellCollection.h index 5b99fe0fe..5b99fe0fe 100755..100644 --- a/moses/src/CellCollection.h +++ b/moses/src/CellCollection.h diff --git a/moses/src/ChartCell.cpp b/moses/src/ChartCell.cpp index 21c5dd68a..21c5dd68a 100755..100644 --- a/moses/src/ChartCell.cpp +++ b/moses/src/ChartCell.cpp diff --git a/moses/src/ChartCell.h b/moses/src/ChartCell.h index ab9e5b1ea..ab9e5b1ea 100755..100644 --- a/moses/src/ChartCell.h +++ b/moses/src/ChartCell.h diff --git a/moses/src/ChartCellCollection.cpp b/moses/src/ChartCellCollection.cpp index 894bb352c..894bb352c 100755..100644 --- a/moses/src/ChartCellCollection.cpp +++ b/moses/src/ChartCellCollection.cpp diff --git a/moses/src/ChartCellCollection.h b/moses/src/ChartCellCollection.h index 6627b6b67..6627b6b67 100755..100644 --- a/moses/src/ChartCellCollection.h +++ b/moses/src/ChartCellCollection.h diff --git a/moses/src/ChartCellLabel.h b/moses/src/ChartCellLabel.h index 4feeac322..4feeac322 100755..100644 --- a/moses/src/ChartCellLabel.h +++ b/moses/src/ChartCellLabel.h diff --git a/moses/src/ChartCellLabelSet.h b/moses/src/ChartCellLabelSet.h index 2a359758d..2a359758d 100755..100644 --- a/moses/src/ChartCellLabelSet.h +++ b/moses/src/ChartCellLabelSet.h diff --git a/moses/src/ChartHypothesis.cpp b/moses/src/ChartHypothesis.cpp index 88b930243..88b930243 100755..100644 --- a/moses/src/ChartHypothesis.cpp +++ b/moses/src/ChartHypothesis.cpp diff --git a/moses/src/ChartHypothesis.h b/moses/src/ChartHypothesis.h index aa663a140..aa663a140 100755..100644 --- a/moses/src/ChartHypothesis.h +++ b/moses/src/ChartHypothesis.h diff --git a/moses/src/ChartHypothesisCollection.cpp b/moses/src/ChartHypothesisCollection.cpp index 48a108aab..48a108aab 100755..100644 --- a/moses/src/ChartHypothesisCollection.cpp +++ b/moses/src/ChartHypothesisCollection.cpp diff --git a/moses/src/ChartHypothesisCollection.h b/moses/src/ChartHypothesisCollection.h index ebf1c6002..ebf1c6002 100755..100644 --- a/moses/src/ChartHypothesisCollection.h +++ b/moses/src/ChartHypothesisCollection.h diff --git a/moses/src/ChartManager.cpp b/moses/src/ChartManager.cpp index ad8f0dc84..77b703007 100755..100644 --- a/moses/src/ChartManager.cpp +++ b/moses/src/ChartManager.cpp @@ -185,7 +185,8 @@ void ChartManager::CalcNBest(size_t count, ChartTrellisPathList &ret, bool onlyD // Get all complete translations Word *w = new Word(); - w->CreateFromString(Output, staticData.GetOutputFactorOrder(), "TOP", true); + //w->CreateFromString(Output, staticData.GetOutputFactorOrder(), "TOP", true); + w->CreateFromString(Output, staticData.GetOutputFactorOrder(), "S", true); const HypoList topHypos = lastCell.GetSortedHypotheses(*w); // Create a ChartTrellisDetour for each complete translation and add it to the queue diff --git a/moses/src/ChartManager.h b/moses/src/ChartManager.h index dbcd42b6c..dbcd42b6c 100755..100644 --- a/moses/src/ChartManager.h +++ b/moses/src/ChartManager.h diff --git a/moses/src/ChartRuleLookupManager.cpp b/moses/src/ChartRuleLookupManager.cpp index 63e941429..63e941429 100755..100644 --- a/moses/src/ChartRuleLookupManager.cpp +++ b/moses/src/ChartRuleLookupManager.cpp diff --git a/moses/src/ChartRuleLookupManager.h b/moses/src/ChartRuleLookupManager.h index 2166f28f5..2166f28f5 100755..100644 --- a/moses/src/ChartRuleLookupManager.h +++ b/moses/src/ChartRuleLookupManager.h diff --git a/moses/src/ChartRuleLookupManagerMemory.cpp b/moses/src/ChartRuleLookupManagerMemory.cpp index aab3dfcce..aab3dfcce 100755..100644 --- a/moses/src/ChartRuleLookupManagerMemory.cpp +++ b/moses/src/ChartRuleLookupManagerMemory.cpp diff --git a/moses/src/ChartRuleLookupManagerMemory.h b/moses/src/ChartRuleLookupManagerMemory.h index a9273499c..a9273499c 100755..100644 --- a/moses/src/ChartRuleLookupManagerMemory.h +++ b/moses/src/ChartRuleLookupManagerMemory.h diff --git a/moses/src/ChartRuleLookupManagerOnDisk.cpp b/moses/src/ChartRuleLookupManagerOnDisk.cpp index 17e57f73c..010973900 100755..100644 --- a/moses/src/ChartRuleLookupManagerOnDisk.cpp +++ b/moses/src/ChartRuleLookupManagerOnDisk.cpp @@ -41,7 +41,6 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk( const WordPenaltyProducer *wpProducer, const std::vector<FactorType> &inputFactorsVec, const std::vector<FactorType> &outputFactorsVec, - const std::vector<float> &weight, const std::string &filePath) : ChartRuleLookupManager(sentence, cellColl) , m_dictionary(dictionary) @@ -50,7 +49,6 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk( , m_wpProducer(wpProducer) , m_inputFactorsVec(inputFactorsVec) , m_outputFactorsVec(outputFactorsVec) - , m_weight(weight) , m_filePath(filePath) { CHECK(m_expandableDottedRuleListVec.size() == 0); @@ -241,12 +239,11 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(m_dictionary.GetTableLimit(), m_dbWrapper); - std::vector<float> weightT = staticData.GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(); + std::vector<float> weightT = staticData.GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(m_dictionary.GetDictIndex()); targetPhraseCollection = tpcollBerkeleyDb->ConvertToMoses(m_inputFactorsVec ,m_outputFactorsVec ,m_dictionary - //, m_weight ,weightT ,m_wpProducer ,*m_languageModels diff --git a/moses/src/ChartRuleLookupManagerOnDisk.h b/moses/src/ChartRuleLookupManagerOnDisk.h index ba87dbbf8..31f44bdfe 100755..100644 --- a/moses/src/ChartRuleLookupManagerOnDisk.h +++ b/moses/src/ChartRuleLookupManagerOnDisk.h @@ -44,7 +44,6 @@ public: const WordPenaltyProducer *wpProducer, const std::vector<FactorType> &inputFactorsVec, const std::vector<FactorType> &outputFactorsVec, - const std::vector<float> &weight, const std::string &filePath); ~ChartRuleLookupManagerOnDisk(); @@ -60,7 +59,6 @@ private: const WordPenaltyProducer *m_wpProducer; const std::vector<FactorType> &m_inputFactorsVec; const std::vector<FactorType> &m_outputFactorsVec; - const std::vector<float> &m_weight; const std::string &m_filePath; std::vector<DottedRuleStackOnDisk*> m_expandableDottedRuleListVec; std::map<UINT64, const TargetPhraseCollection*> m_cache; diff --git a/moses/src/ChartTranslationOption.cpp b/moses/src/ChartTranslationOption.cpp index 6aa04d189..6aa04d189 100755..100644 --- a/moses/src/ChartTranslationOption.cpp +++ b/moses/src/ChartTranslationOption.cpp diff --git a/moses/src/ChartTranslationOption.h b/moses/src/ChartTranslationOption.h index 116eff037..116eff037 100755..100644 --- a/moses/src/ChartTranslationOption.h +++ b/moses/src/ChartTranslationOption.h diff --git a/moses/src/ChartTranslationOptionCollection.cpp b/moses/src/ChartTranslationOptionCollection.cpp index 8f06656b2..8f06656b2 100755..100644 --- a/moses/src/ChartTranslationOptionCollection.cpp +++ b/moses/src/ChartTranslationOptionCollection.cpp diff --git a/moses/src/ChartTranslationOptionCollection.h b/moses/src/ChartTranslationOptionCollection.h index e1f3490a9..e1f3490a9 100755..100644 --- a/moses/src/ChartTranslationOptionCollection.h +++ b/moses/src/ChartTranslationOptionCollection.h diff --git a/moses/src/ChartTranslationOptionList.cpp b/moses/src/ChartTranslationOptionList.cpp index cb1b8e256..cb1b8e256 100755..100644 --- a/moses/src/ChartTranslationOptionList.cpp +++ b/moses/src/ChartTranslationOptionList.cpp diff --git a/moses/src/ChartTranslationOptionList.h b/moses/src/ChartTranslationOptionList.h index 7272a01cf..7272a01cf 100755..100644 --- a/moses/src/ChartTranslationOptionList.h +++ b/moses/src/ChartTranslationOptionList.h diff --git a/moses/src/ChartTrellisDetour.cpp b/moses/src/ChartTrellisDetour.cpp index 550a44a2c..550a44a2c 100755..100644 --- a/moses/src/ChartTrellisDetour.cpp +++ b/moses/src/ChartTrellisDetour.cpp diff --git a/moses/src/ChartTrellisDetour.h b/moses/src/ChartTrellisDetour.h index a3b07ad00..a3b07ad00 100755..100644 --- a/moses/src/ChartTrellisDetour.h +++ b/moses/src/ChartTrellisDetour.h diff --git a/moses/src/ChartTrellisDetourQueue.cpp b/moses/src/ChartTrellisDetourQueue.cpp index 9b359ca43..9b359ca43 100755..100644 --- a/moses/src/ChartTrellisDetourQueue.cpp +++ b/moses/src/ChartTrellisDetourQueue.cpp diff --git a/moses/src/ChartTrellisDetourQueue.h b/moses/src/ChartTrellisDetourQueue.h index f679708e4..f679708e4 100755..100644 --- a/moses/src/ChartTrellisDetourQueue.h +++ b/moses/src/ChartTrellisDetourQueue.h diff --git a/moses/src/ChartTrellisNode.cpp b/moses/src/ChartTrellisNode.cpp index ce3c9eaf7..ce3c9eaf7 100755..100644 --- a/moses/src/ChartTrellisNode.cpp +++ b/moses/src/ChartTrellisNode.cpp diff --git a/moses/src/ChartTrellisNode.h b/moses/src/ChartTrellisNode.h index 7b81ff4b2..7b81ff4b2 100755..100644 --- a/moses/src/ChartTrellisNode.h +++ b/moses/src/ChartTrellisNode.h diff --git a/moses/src/ChartTrellisPath.cpp b/moses/src/ChartTrellisPath.cpp index 231d4237a..231d4237a 100755..100644 --- a/moses/src/ChartTrellisPath.cpp +++ b/moses/src/ChartTrellisPath.cpp diff --git a/moses/src/ChartTrellisPath.h b/moses/src/ChartTrellisPath.h index 4dee018c3..4dee018c3 100755..100644 --- a/moses/src/ChartTrellisPath.h +++ b/moses/src/ChartTrellisPath.h diff --git a/moses/src/ChartTrellisPathList.h b/moses/src/ChartTrellisPathList.h index 5b811d621..5b811d621 100755..100644 --- a/moses/src/ChartTrellisPathList.h +++ b/moses/src/ChartTrellisPathList.h diff --git a/moses/src/ConfusionNet.cpp b/moses/src/ConfusionNet.cpp index 8ff119497..8ff119497 100755..100644 --- a/moses/src/ConfusionNet.cpp +++ b/moses/src/ConfusionNet.cpp diff --git a/moses/src/ConfusionNet.h b/moses/src/ConfusionNet.h index 25b5a021d..25b5a021d 100755..100644 --- a/moses/src/ConfusionNet.h +++ b/moses/src/ConfusionNet.h diff --git a/moses/src/DecodeFeature.cpp b/moses/src/DecodeFeature.cpp index 67999384a..67999384a 100755..100644 --- a/moses/src/DecodeFeature.cpp +++ b/moses/src/DecodeFeature.cpp diff --git a/moses/src/DecodeFeature.h b/moses/src/DecodeFeature.h index afa1771ba..afa1771ba 100755..100644 --- a/moses/src/DecodeFeature.h +++ b/moses/src/DecodeFeature.h diff --git a/moses/src/DecodeGraph.cpp b/moses/src/DecodeGraph.cpp index 107258d56..107258d56 100755..100644 --- a/moses/src/DecodeGraph.cpp +++ b/moses/src/DecodeGraph.cpp diff --git a/moses/src/DecodeGraph.h b/moses/src/DecodeGraph.h index 770cb9958..770cb9958 100755..100644 --- a/moses/src/DecodeGraph.h +++ b/moses/src/DecodeGraph.h diff --git a/moses/src/DecodeStep.cpp b/moses/src/DecodeStep.cpp index 3345f95d0..3345f95d0 100755..100644 --- a/moses/src/DecodeStep.cpp +++ b/moses/src/DecodeStep.cpp diff --git a/moses/src/DecodeStep.h b/moses/src/DecodeStep.h index 4d2f2280b..4d2f2280b 100755..100644 --- a/moses/src/DecodeStep.h +++ b/moses/src/DecodeStep.h diff --git a/moses/src/DecodeStepGeneration.cpp b/moses/src/DecodeStepGeneration.cpp index 0f0c5c2dc..0f0c5c2dc 100755..100644 --- a/moses/src/DecodeStepGeneration.cpp +++ b/moses/src/DecodeStepGeneration.cpp diff --git a/moses/src/DecodeStepGeneration.h b/moses/src/DecodeStepGeneration.h index d22534ccc..d22534ccc 100755..100644 --- a/moses/src/DecodeStepGeneration.h +++ b/moses/src/DecodeStepGeneration.h diff --git a/moses/src/DecodeStepTranslation.cpp b/moses/src/DecodeStepTranslation.cpp index 03713b50c..03713b50c 100755..100644 --- a/moses/src/DecodeStepTranslation.cpp +++ b/moses/src/DecodeStepTranslation.cpp diff --git a/moses/src/DecodeStepTranslation.h b/moses/src/DecodeStepTranslation.h index 651754b1c..651754b1c 100755..100644 --- a/moses/src/DecodeStepTranslation.h +++ b/moses/src/DecodeStepTranslation.h diff --git a/moses/src/Dictionary.cpp b/moses/src/Dictionary.cpp index 05b4f6bb4..05b4f6bb4 100755..100644 --- a/moses/src/Dictionary.cpp +++ b/moses/src/Dictionary.cpp diff --git a/moses/src/Dictionary.h b/moses/src/Dictionary.h index 7bb4d09b1..7bb4d09b1 100755..100644 --- a/moses/src/Dictionary.h +++ b/moses/src/Dictionary.h diff --git a/moses/src/DotChart.cpp b/moses/src/DotChart.cpp index c7ba3f43e..c7ba3f43e 100755..100644 --- a/moses/src/DotChart.cpp +++ b/moses/src/DotChart.cpp diff --git a/moses/src/DotChart.h b/moses/src/DotChart.h index b9da12afd..b9da12afd 100755..100644 --- a/moses/src/DotChart.h +++ b/moses/src/DotChart.h diff --git a/moses/src/DotChartInMemory.cpp b/moses/src/DotChartInMemory.cpp index 095e3418f..095e3418f 100755..100644 --- a/moses/src/DotChartInMemory.cpp +++ b/moses/src/DotChartInMemory.cpp diff --git a/moses/src/DotChartInMemory.h b/moses/src/DotChartInMemory.h index fe7ab64c5..fe7ab64c5 100755..100644 --- a/moses/src/DotChartInMemory.h +++ b/moses/src/DotChartInMemory.h diff --git a/moses/src/DotChartOnDisk.cpp b/moses/src/DotChartOnDisk.cpp index 1674f0e21..1674f0e21 100755..100644 --- a/moses/src/DotChartOnDisk.cpp +++ b/moses/src/DotChartOnDisk.cpp diff --git a/moses/src/DotChartOnDisk.h b/moses/src/DotChartOnDisk.h index 4473b7685..4473b7685 100755..100644 --- a/moses/src/DotChartOnDisk.h +++ b/moses/src/DotChartOnDisk.h diff --git a/moses/src/DummyScoreProducers.cpp b/moses/src/DummyScoreProducers.cpp index 03c635b03..03c635b03 100755..100644 --- a/moses/src/DummyScoreProducers.cpp +++ b/moses/src/DummyScoreProducers.cpp diff --git a/moses/src/DummyScoreProducers.h b/moses/src/DummyScoreProducers.h index d8d9393d2..d8d9393d2 100755..100644 --- a/moses/src/DummyScoreProducers.h +++ b/moses/src/DummyScoreProducers.h diff --git a/moses/src/DynSAInclude/RandLMCache.h b/moses/src/DynSAInclude/RandLMCache.h index 3f38cae02..3f38cae02 100755..100644 --- a/moses/src/DynSAInclude/RandLMCache.h +++ b/moses/src/DynSAInclude/RandLMCache.h diff --git a/moses/src/DynSAInclude/RandLMFilter.h b/moses/src/DynSAInclude/RandLMFilter.h index 556bbe44a..556bbe44a 100755..100644 --- a/moses/src/DynSAInclude/RandLMFilter.h +++ b/moses/src/DynSAInclude/RandLMFilter.h diff --git a/moses/src/DynSAInclude/fdstream.h b/moses/src/DynSAInclude/fdstream.h index 4c95d032c..4c95d032c 100755..100644 --- a/moses/src/DynSAInclude/fdstream.h +++ b/moses/src/DynSAInclude/fdstream.h diff --git a/moses/src/DynSAInclude/file.cpp b/moses/src/DynSAInclude/file.cpp index d2901c1ae..d2901c1ae 100755..100644 --- a/moses/src/DynSAInclude/file.cpp +++ b/moses/src/DynSAInclude/file.cpp diff --git a/moses/src/DynSAInclude/file.h b/moses/src/DynSAInclude/file.h index 3157f918b..3157f918b 100755..100644 --- a/moses/src/DynSAInclude/file.h +++ b/moses/src/DynSAInclude/file.h diff --git a/moses/src/DynSAInclude/hash.h b/moses/src/DynSAInclude/hash.h index 233d0be5d..233d0be5d 100755..100644 --- a/moses/src/DynSAInclude/hash.h +++ b/moses/src/DynSAInclude/hash.h diff --git a/moses/src/DynSAInclude/onlineRLM.h b/moses/src/DynSAInclude/onlineRLM.h index adc7934fc..adc7934fc 100755..100644 --- a/moses/src/DynSAInclude/onlineRLM.h +++ b/moses/src/DynSAInclude/onlineRLM.h diff --git a/moses/src/DynSAInclude/params.cpp b/moses/src/DynSAInclude/params.cpp index 4be3a1676..4be3a1676 100755..100644 --- a/moses/src/DynSAInclude/params.cpp +++ b/moses/src/DynSAInclude/params.cpp diff --git a/moses/src/DynSAInclude/params.h b/moses/src/DynSAInclude/params.h index e0aab6135..e0aab6135 100755..100644 --- a/moses/src/DynSAInclude/params.h +++ b/moses/src/DynSAInclude/params.h diff --git a/moses/src/DynSAInclude/perfectHash.h b/moses/src/DynSAInclude/perfectHash.h index 2e3bfbc8f..2e3bfbc8f 100755..100644 --- a/moses/src/DynSAInclude/perfectHash.h +++ b/moses/src/DynSAInclude/perfectHash.h diff --git a/moses/src/DynSAInclude/quantizer.h b/moses/src/DynSAInclude/quantizer.h index c12189615..c12189615 100755..100644 --- a/moses/src/DynSAInclude/quantizer.h +++ b/moses/src/DynSAInclude/quantizer.h diff --git a/moses/src/DynSAInclude/types.h b/moses/src/DynSAInclude/types.h index 444010889..444010889 100755..100644 --- a/moses/src/DynSAInclude/types.h +++ b/moses/src/DynSAInclude/types.h diff --git a/moses/src/DynSAInclude/utils.h b/moses/src/DynSAInclude/utils.h index 74703863d..74703863d 100755..100644 --- a/moses/src/DynSAInclude/utils.h +++ b/moses/src/DynSAInclude/utils.h diff --git a/moses/src/DynSAInclude/vocab.cpp b/moses/src/DynSAInclude/vocab.cpp index 27e052260..27e052260 100755..100644 --- a/moses/src/DynSAInclude/vocab.cpp +++ b/moses/src/DynSAInclude/vocab.cpp diff --git a/moses/src/DynSAInclude/vocab.h b/moses/src/DynSAInclude/vocab.h index 467d16fdb..467d16fdb 100755..100644 --- a/moses/src/DynSAInclude/vocab.h +++ b/moses/src/DynSAInclude/vocab.h diff --git a/moses/src/DynSuffixArray.cpp b/moses/src/DynSuffixArray.cpp index f043f5c9b..f043f5c9b 100755..100644 --- a/moses/src/DynSuffixArray.cpp +++ b/moses/src/DynSuffixArray.cpp diff --git a/moses/src/DynSuffixArray.h b/moses/src/DynSuffixArray.h index 05ea0596a..05ea0596a 100755..100644 --- a/moses/src/DynSuffixArray.h +++ b/moses/src/DynSuffixArray.h diff --git a/moses/src/FFState.cpp b/moses/src/FFState.cpp index 5bf0992a9..5bf0992a9 100755..100644 --- a/moses/src/FFState.cpp +++ b/moses/src/FFState.cpp diff --git a/moses/src/FFState.h b/moses/src/FFState.h index 49b0e55a8..49b0e55a8 100755..100644 --- a/moses/src/FFState.h +++ b/moses/src/FFState.h diff --git a/moses/src/Factor.cpp b/moses/src/Factor.cpp index 60a93fdc3..60a93fdc3 100755..100644 --- a/moses/src/Factor.cpp +++ b/moses/src/Factor.cpp diff --git a/moses/src/Factor.h b/moses/src/Factor.h index 4ea44f4e1..4ea44f4e1 100755..100644 --- a/moses/src/Factor.h +++ b/moses/src/Factor.h diff --git a/moses/src/FactorCollection.cpp b/moses/src/FactorCollection.cpp index 2ef55b6d1..2ef55b6d1 100755..100644 --- a/moses/src/FactorCollection.cpp +++ b/moses/src/FactorCollection.cpp diff --git a/moses/src/FactorCollection.h b/moses/src/FactorCollection.h index d55be1b17..d55be1b17 100755..100644 --- a/moses/src/FactorCollection.h +++ b/moses/src/FactorCollection.h diff --git a/moses/src/FactorTypeSet.cpp b/moses/src/FactorTypeSet.cpp index de2166ba2..de2166ba2 100755..100644 --- a/moses/src/FactorTypeSet.cpp +++ b/moses/src/FactorTypeSet.cpp diff --git a/moses/src/FactorTypeSet.h b/moses/src/FactorTypeSet.h index b0db0502c..b0db0502c 100755..100644 --- a/moses/src/FactorTypeSet.h +++ b/moses/src/FactorTypeSet.h diff --git a/moses/src/FeatureFunction.cpp b/moses/src/FeatureFunction.cpp index d435df5af..d435df5af 100755..100644 --- a/moses/src/FeatureFunction.cpp +++ b/moses/src/FeatureFunction.cpp diff --git a/moses/src/FeatureFunction.h b/moses/src/FeatureFunction.h index 4ef85ecd9..4ef85ecd9 100755..100644 --- a/moses/src/FeatureFunction.h +++ b/moses/src/FeatureFunction.h diff --git a/moses/src/FeatureVector.cpp b/moses/src/FeatureVector.cpp index 911c29600..ac3deb876 100755..100644 --- a/moses/src/FeatureVector.cpp +++ b/moses/src/FeatureVector.cpp @@ -156,29 +156,29 @@ namespace Moses { } } - void FVector::clear() { - m_coreFeatures.resize(0); - m_features.clear(); - } + void FVector::clear() { + m_coreFeatures.resize(0); + m_features.clear(); + } - bool FVector::load(const std::string& filename) { + bool FVector::load(const std::string& filename) { clear(); - ifstream in (filename.c_str()); - if (!in) { + ifstream in (filename.c_str()); + if (!in) { return false; - } - string line; - while(getline(in,line)) { - if (line[0] == '#') continue; - istringstream linestream(line); - string namestring; - FValue value; - linestream >> namestring; - linestream >> value; - FName fname(namestring); - cerr << "Setting sparse weight " << fname << " to value " << value << "." << endl; - set(fname,value); - } + } + string line; + while(getline(in,line)) { + if (line[0] == '#') continue; + istringstream linestream(line); + string namestring; + FValue value; + linestream >> namestring; + linestream >> value; + FName fname(namestring); + //cerr << "Setting sparse weight " << fname << " to value " << value << "." << endl; + set(fname,value); + } return true; } @@ -416,34 +416,38 @@ namespace Moses { void FVector::updateConfidenceCounts(const FVector& weightUpdate, bool signedCounts) { for (size_t i = 0; i < weightUpdate.m_coreFeatures.size(); ++i) { if (signedCounts) { - int sign = weightUpdate.m_coreFeatures[i] >= 0 ? 1 : -1; - m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]) * sign; + //int sign = weightUpdate.m_coreFeatures[i] >= 0 ? 1 : -1; + //m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]) * sign; + m_coreFeatures[i] += weightUpdate.m_coreFeatures[i]; } else - m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]); + //m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]); + m_coreFeatures[i] += abs(weightUpdate.m_coreFeatures[i]); } for (const_iterator i = weightUpdate.cbegin(); i != weightUpdate.cend(); ++i) { if (weightUpdate[i->first] == 0) continue; - float value = get(i->first); + float value = get(i->first); if (signedCounts) { - int sign = weightUpdate[i->first] >= 0 ? 1 : -1; - value += (weightUpdate[i->first] * weightUpdate[i->first]) * sign; + //int sign = weightUpdate[i->first] >= 0 ? 1 : -1; + //value += (weightUpdate[i->first] * weightUpdate[i->first]) * sign; + value += weightUpdate[i->first]; } else - value += (weightUpdate[i->first] * weightUpdate[i->first]); + //value += (weightUpdate[i->first] * weightUpdate[i->first]); + value += abs(weightUpdate[i->first]); set(i->first, value); } } - void FVector::updateLearningRates(float decay, const FVector &confidenceCounts, float core_r0, float sparse_r0) { + void FVector::updateLearningRates(float decay_core, float decay_sparse, const FVector &confidenceCounts, float core_r0, float sparse_r0) { for (size_t i = 0; i < confidenceCounts.m_coreFeatures.size(); ++i) { - m_coreFeatures[i] = 1.0/(1.0/core_r0 + decay * abs(confidenceCounts.m_coreFeatures[i])); + m_coreFeatures[i] = 1.0/(1.0/core_r0 + decay_core * abs(confidenceCounts.m_coreFeatures[i])); } for (const_iterator i = confidenceCounts.cbegin(); i != confidenceCounts.cend(); ++i) { - float value = 1.0/(1.0/sparse_r0 + decay * abs(i->second)); + float value = 1.0/(1.0/sparse_r0 + decay_sparse * abs(i->second)); set(i->first, value); } } @@ -612,8 +616,8 @@ namespace Moses { return norm; } - void FVector::l1regularize(float lambda) { - for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + size_t FVector::sparseL1regularize(float lambda) { + /*for (size_t i = 0; i < m_coreFeatures.size(); ++i) { float value = m_coreFeatures[i]; if (value > 0) { m_coreFeatures[i] = max(0.0f, value - lambda); @@ -621,23 +625,39 @@ namespace Moses { else { m_coreFeatures[i] = min(0.0f, value + lambda); } - } + }*/ + size_t numberPruned = size(); + vector<FName> toErase; for (iterator i = begin(); i != end(); ++i) { float value = i->second; - if (value > 0) { - i->second = max(0.0f, value - lambda); - } - else { - i->second = min(0.0f, value + lambda); + if (value != 0.0f) { + if (value > 0) + value = max(0.0f, value - lambda); + else + value = min(0.0f, value + lambda); + + if (value != 0.0f) + i->second = value; + else { + toErase.push_back(i->first); + const std::string& fname = (i->first).name(); + FName::eraseId(FName::getId(fname)); + } } } + + // erase features that have become zero + for (size_t i = 0; i < toErase.size(); ++i) + m_features.erase(toErase[i]); + numberPruned -= size(); + return numberPruned; } - void FVector::l2regularize(float lambda) { - for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + void FVector::sparseL2regularize(float lambda) { + /*for (size_t i = 0; i < m_coreFeatures.size(); ++i) { m_coreFeatures[i] *= (1 - lambda); - } + }*/ for (iterator i = begin(); i != end(); ++i) { i->second *= (1 - lambda); diff --git a/moses/src/FeatureVector.h b/moses/src/FeatureVector.h index a86b30001..5dbfa01fe 100755..100644 --- a/moses/src/FeatureVector.h +++ b/moses/src/FeatureVector.h @@ -192,8 +192,8 @@ namespace Moses { FValue l1norm_coreFeatures() const; FValue l2norm() const; FValue linfnorm() const; - void l1regularize(float lambda); - void l2regularize(float lambda); + size_t sparseL1regularize(float lambda); + void sparseL2regularize(float lambda); FValue sum() const; /** pretty printing */ @@ -218,7 +218,7 @@ namespace Moses { size_t pruneSparseFeatures(size_t threshold); size_t pruneZeroWeightFeatures(); void updateConfidenceCounts(const FVector& weightUpdate, bool signedCounts); - void updateLearningRates(float decay, const FVector& confidence_counts, float core_r0, float sparse_r0); + void updateLearningRates(float decay_core, float decay_sparse, const FVector& confidence_counts, float core_r0, float sparse_r0); // vector which, for each element of the original vector, reflects whether an element is zero or non-zero void setToBinaryOf(const FVector& rhs); diff --git a/moses/src/FeatureVectorTest.cpp b/moses/src/FeatureVectorTest.cpp index af1829e62..af1829e62 100755..100644 --- a/moses/src/FeatureVectorTest.cpp +++ b/moses/src/FeatureVectorTest.cpp diff --git a/moses/src/File.cpp b/moses/src/File.cpp index 3d15b071c..3d15b071c 100755..100644 --- a/moses/src/File.cpp +++ b/moses/src/File.cpp diff --git a/moses/src/File.h b/moses/src/File.h index 09d9ddc92..09d9ddc92 100755..100644 --- a/moses/src/File.h +++ b/moses/src/File.h diff --git a/moses/src/FilePtr.h b/moses/src/FilePtr.h index 1613b2f2f..1613b2f2f 100755..100644 --- a/moses/src/FilePtr.h +++ b/moses/src/FilePtr.h diff --git a/moses/src/FloydWarshall.cpp b/moses/src/FloydWarshall.cpp index e63de819b..e63de819b 100755..100644 --- a/moses/src/FloydWarshall.cpp +++ b/moses/src/FloydWarshall.cpp diff --git a/moses/src/FloydWarshall.h b/moses/src/FloydWarshall.h index 96845caf6..96845caf6 100755..100644 --- a/moses/src/FloydWarshall.h +++ b/moses/src/FloydWarshall.h diff --git a/moses/src/GenerationDictionary.cpp b/moses/src/GenerationDictionary.cpp index f7b35fb63..f7b35fb63 100755..100644 --- a/moses/src/GenerationDictionary.cpp +++ b/moses/src/GenerationDictionary.cpp diff --git a/moses/src/GenerationDictionary.h b/moses/src/GenerationDictionary.h index 1c3f75f75..1c3f75f75 100755..100644 --- a/moses/src/GenerationDictionary.h +++ b/moses/src/GenerationDictionary.h diff --git a/moses/src/GlobalLexicalModel.cpp b/moses/src/GlobalLexicalModel.cpp index 65d3afaa1..65d3afaa1 100755..100644 --- a/moses/src/GlobalLexicalModel.cpp +++ b/moses/src/GlobalLexicalModel.cpp diff --git a/moses/src/GlobalLexicalModel.h b/moses/src/GlobalLexicalModel.h index 3105cb466..3105cb466 100755..100644 --- a/moses/src/GlobalLexicalModel.h +++ b/moses/src/GlobalLexicalModel.h diff --git a/moses/src/GlobalLexicalModelUnlimited.cpp b/moses/src/GlobalLexicalModelUnlimited.cpp index f1de65bd0..f1de65bd0 100755..100644 --- a/moses/src/GlobalLexicalModelUnlimited.cpp +++ b/moses/src/GlobalLexicalModelUnlimited.cpp diff --git a/moses/src/GlobalLexicalModelUnlimited.h b/moses/src/GlobalLexicalModelUnlimited.h index c05c11be8..c05c11be8 100755..100644 --- a/moses/src/GlobalLexicalModelUnlimited.h +++ b/moses/src/GlobalLexicalModelUnlimited.h diff --git a/moses/src/Hypothesis.cpp b/moses/src/Hypothesis.cpp index 72105ec7b..72105ec7b 100755..100644 --- a/moses/src/Hypothesis.cpp +++ b/moses/src/Hypothesis.cpp diff --git a/moses/src/Hypothesis.h b/moses/src/Hypothesis.h index 23980326e..23980326e 100755..100644 --- a/moses/src/Hypothesis.h +++ b/moses/src/Hypothesis.h diff --git a/moses/src/HypothesisStack.cpp b/moses/src/HypothesisStack.cpp index dfa03108e..dfa03108e 100755..100644 --- a/moses/src/HypothesisStack.cpp +++ b/moses/src/HypothesisStack.cpp diff --git a/moses/src/HypothesisStack.h b/moses/src/HypothesisStack.h index 784cd7a39..784cd7a39 100755..100644 --- a/moses/src/HypothesisStack.h +++ b/moses/src/HypothesisStack.h diff --git a/moses/src/HypothesisStackCubePruning.cpp b/moses/src/HypothesisStackCubePruning.cpp index ca54bf944..ca54bf944 100755..100644 --- a/moses/src/HypothesisStackCubePruning.cpp +++ b/moses/src/HypothesisStackCubePruning.cpp diff --git a/moses/src/HypothesisStackCubePruning.h b/moses/src/HypothesisStackCubePruning.h index 53cad04a1..53cad04a1 100755..100644 --- a/moses/src/HypothesisStackCubePruning.h +++ b/moses/src/HypothesisStackCubePruning.h diff --git a/moses/src/HypothesisStackNormal.cpp b/moses/src/HypothesisStackNormal.cpp index 7d72f76aa..7d72f76aa 100755..100644 --- a/moses/src/HypothesisStackNormal.cpp +++ b/moses/src/HypothesisStackNormal.cpp diff --git a/moses/src/HypothesisStackNormal.h b/moses/src/HypothesisStackNormal.h index 6996f4003..6996f4003 100755..100644 --- a/moses/src/HypothesisStackNormal.h +++ b/moses/src/HypothesisStackNormal.h diff --git a/moses/src/InputFileStream.cpp b/moses/src/InputFileStream.cpp index d2d774bf1..d2d774bf1 100755..100644 --- a/moses/src/InputFileStream.cpp +++ b/moses/src/InputFileStream.cpp diff --git a/moses/src/InputFileStream.h b/moses/src/InputFileStream.h index d53abfc23..d53abfc23 100755..100644 --- a/moses/src/InputFileStream.h +++ b/moses/src/InputFileStream.h diff --git a/moses/src/InputType.cpp b/moses/src/InputType.cpp index 77857bdaf..77857bdaf 100755..100644 --- a/moses/src/InputType.cpp +++ b/moses/src/InputType.cpp diff --git a/moses/src/InputType.h b/moses/src/InputType.h index ca710c408..ca710c408 100755..100644 --- a/moses/src/InputType.h +++ b/moses/src/InputType.h diff --git a/moses/src/Jamfile b/moses/src/Jamfile index b6e379070..b6e379070 100755..100644 --- a/moses/src/Jamfile +++ b/moses/src/Jamfile diff --git a/moses/src/LM/Base.cpp b/moses/src/LM/Base.cpp index 45edf973e..45edf973e 100755..100644 --- a/moses/src/LM/Base.cpp +++ b/moses/src/LM/Base.cpp diff --git a/moses/src/LM/Base.h b/moses/src/LM/Base.h index 621ab0dc3..621ab0dc3 100755..100644 --- a/moses/src/LM/Base.h +++ b/moses/src/LM/Base.h diff --git a/moses/src/LM/DMap.cpp b/moses/src/LM/DMap.cpp index 3bef47083..3bef47083 100755..100644 --- a/moses/src/LM/DMap.cpp +++ b/moses/src/LM/DMap.cpp diff --git a/moses/src/LM/DMap.h b/moses/src/LM/DMap.h index f9fbf56c2..f9fbf56c2 100755..100644 --- a/moses/src/LM/DMap.h +++ b/moses/src/LM/DMap.h diff --git a/moses/src/LM/Factory.cpp b/moses/src/LM/Factory.cpp index 5d877e9d6..5d877e9d6 100755..100644 --- a/moses/src/LM/Factory.cpp +++ b/moses/src/LM/Factory.cpp diff --git a/moses/src/LM/Factory.h b/moses/src/LM/Factory.h index 1a140846a..1a140846a 100755..100644 --- a/moses/src/LM/Factory.h +++ b/moses/src/LM/Factory.h diff --git a/moses/src/LM/IRST.cpp b/moses/src/LM/IRST.cpp index 692385b4d..692385b4d 100755..100644 --- a/moses/src/LM/IRST.cpp +++ b/moses/src/LM/IRST.cpp diff --git a/moses/src/LM/IRST.h b/moses/src/LM/IRST.h index 2ffe67f98..2ffe67f98 100755..100644 --- a/moses/src/LM/IRST.h +++ b/moses/src/LM/IRST.h diff --git a/moses/src/LM/Implementation.cpp b/moses/src/LM/Implementation.cpp index 589ed375a..589ed375a 100755..100644 --- a/moses/src/LM/Implementation.cpp +++ b/moses/src/LM/Implementation.cpp diff --git a/moses/src/LM/Implementation.h b/moses/src/LM/Implementation.h index a7c637ef8..a7c637ef8 100755..100644 --- a/moses/src/LM/Implementation.h +++ b/moses/src/LM/Implementation.h diff --git a/moses/src/LM/Jamfile b/moses/src/LM/Jamfile index 2f3cde8ef..2f3cde8ef 100755..100644 --- a/moses/src/LM/Jamfile +++ b/moses/src/LM/Jamfile diff --git a/moses/src/LM/Joint.cpp b/moses/src/LM/Joint.cpp index 266f08029..266f08029 100755..100644 --- a/moses/src/LM/Joint.cpp +++ b/moses/src/LM/Joint.cpp diff --git a/moses/src/LM/Joint.h b/moses/src/LM/Joint.h index e000409be..e000409be 100755..100644 --- a/moses/src/LM/Joint.h +++ b/moses/src/LM/Joint.h diff --git a/moses/src/LM/Ken.cpp b/moses/src/LM/Ken.cpp index c5307ffc0..c5307ffc0 100755..100644 --- a/moses/src/LM/Ken.cpp +++ b/moses/src/LM/Ken.cpp diff --git a/moses/src/LM/Ken.h b/moses/src/LM/Ken.h index f55679395..f55679395 100755..100644 --- a/moses/src/LM/Ken.h +++ b/moses/src/LM/Ken.h diff --git a/moses/src/LM/MultiFactor.cpp b/moses/src/LM/MultiFactor.cpp index d141b88b3..d141b88b3 100755..100644 --- a/moses/src/LM/MultiFactor.cpp +++ b/moses/src/LM/MultiFactor.cpp diff --git a/moses/src/LM/MultiFactor.h b/moses/src/LM/MultiFactor.h index 32cef51d9..32cef51d9 100755..100644 --- a/moses/src/LM/MultiFactor.h +++ b/moses/src/LM/MultiFactor.h diff --git a/moses/src/LM/ORLM.cpp b/moses/src/LM/ORLM.cpp index 30f3ada55..30f3ada55 100755..100644 --- a/moses/src/LM/ORLM.cpp +++ b/moses/src/LM/ORLM.cpp diff --git a/moses/src/LM/ORLM.h b/moses/src/LM/ORLM.h index c1ccb4387..c1ccb4387 100755..100644 --- a/moses/src/LM/ORLM.h +++ b/moses/src/LM/ORLM.h diff --git a/moses/src/LM/ParallelBackoff.cpp b/moses/src/LM/ParallelBackoff.cpp index ec2fb2f78..ec2fb2f78 100755..100644 --- a/moses/src/LM/ParallelBackoff.cpp +++ b/moses/src/LM/ParallelBackoff.cpp diff --git a/moses/src/LM/ParallelBackoff.h b/moses/src/LM/ParallelBackoff.h index 8e4241395..8e4241395 100755..100644 --- a/moses/src/LM/ParallelBackoff.h +++ b/moses/src/LM/ParallelBackoff.h diff --git a/moses/src/LM/Rand.cpp b/moses/src/LM/Rand.cpp index 93411b8d3..93411b8d3 100755..100644 --- a/moses/src/LM/Rand.cpp +++ b/moses/src/LM/Rand.cpp diff --git a/moses/src/LM/Rand.h b/moses/src/LM/Rand.h index c4ef99947..c4ef99947 100755..100644 --- a/moses/src/LM/Rand.h +++ b/moses/src/LM/Rand.h diff --git a/moses/src/LM/Remote.cpp b/moses/src/LM/Remote.cpp index 78380c75e..78380c75e 100755..100644 --- a/moses/src/LM/Remote.cpp +++ b/moses/src/LM/Remote.cpp diff --git a/moses/src/LM/Remote.h b/moses/src/LM/Remote.h index 9a70d64c0..9a70d64c0 100755..100644 --- a/moses/src/LM/Remote.h +++ b/moses/src/LM/Remote.h diff --git a/moses/src/LM/SRI.cpp b/moses/src/LM/SRI.cpp index 9c5a9c3d4..9c5a9c3d4 100755..100644 --- a/moses/src/LM/SRI.cpp +++ b/moses/src/LM/SRI.cpp diff --git a/moses/src/LM/SRI.h b/moses/src/LM/SRI.h index 69d55f117..69d55f117 100755..100644 --- a/moses/src/LM/SRI.h +++ b/moses/src/LM/SRI.h diff --git a/moses/src/LM/SingleFactor.cpp b/moses/src/LM/SingleFactor.cpp index 73c9e6e83..73c9e6e83 100755..100644 --- a/moses/src/LM/SingleFactor.cpp +++ b/moses/src/LM/SingleFactor.cpp diff --git a/moses/src/LM/SingleFactor.h b/moses/src/LM/SingleFactor.h index 44fdf9b16..44fdf9b16 100755..100644 --- a/moses/src/LM/SingleFactor.h +++ b/moses/src/LM/SingleFactor.h diff --git a/moses/src/LMList.cpp b/moses/src/LMList.cpp index cac1199e0..cac1199e0 100755..100644 --- a/moses/src/LMList.cpp +++ b/moses/src/LMList.cpp diff --git a/moses/src/LMList.h b/moses/src/LMList.h index a803e16df..a803e16df 100755..100644 --- a/moses/src/LMList.h +++ b/moses/src/LMList.h diff --git a/moses/src/LVoc.cpp b/moses/src/LVoc.cpp index d1ee92237..d1ee92237 100755..100644 --- a/moses/src/LVoc.cpp +++ b/moses/src/LVoc.cpp diff --git a/moses/src/LVoc.h b/moses/src/LVoc.h index ec20fe3cd..ec20fe3cd 100755..100644 --- a/moses/src/LVoc.h +++ b/moses/src/LVoc.h diff --git a/moses/src/LexicalReordering.cpp b/moses/src/LexicalReordering.cpp index b32b96b0d..b32b96b0d 100755..100644 --- a/moses/src/LexicalReordering.cpp +++ b/moses/src/LexicalReordering.cpp diff --git a/moses/src/LexicalReordering.h b/moses/src/LexicalReordering.h index e979b5bf5..e979b5bf5 100755..100644 --- a/moses/src/LexicalReordering.h +++ b/moses/src/LexicalReordering.h diff --git a/moses/src/LexicalReorderingState.cpp b/moses/src/LexicalReorderingState.cpp index 346ac745b..346ac745b 100755..100644 --- a/moses/src/LexicalReorderingState.cpp +++ b/moses/src/LexicalReorderingState.cpp diff --git a/moses/src/LexicalReorderingState.h b/moses/src/LexicalReorderingState.h index 802f1fcb0..802f1fcb0 100755..100644 --- a/moses/src/LexicalReorderingState.h +++ b/moses/src/LexicalReorderingState.h diff --git a/moses/src/LexicalReorderingTable.cpp b/moses/src/LexicalReorderingTable.cpp index 00be0a495..00be0a495 100755..100644 --- a/moses/src/LexicalReorderingTable.cpp +++ b/moses/src/LexicalReorderingTable.cpp diff --git a/moses/src/LexicalReorderingTable.h b/moses/src/LexicalReorderingTable.h index ae10ae386..ae10ae386 100755..100644 --- a/moses/src/LexicalReorderingTable.h +++ b/moses/src/LexicalReorderingTable.h diff --git a/moses/src/Makefile.am b/moses/src/Makefile.am index b43191773..b43191773 100755..100644 --- a/moses/src/Makefile.am +++ b/moses/src/Makefile.am diff --git a/moses/src/Manager.cpp b/moses/src/Manager.cpp index c27d88fee..c27d88fee 100755..100644 --- a/moses/src/Manager.cpp +++ b/moses/src/Manager.cpp diff --git a/moses/src/Manager.h b/moses/src/Manager.h index 4b45414ee..4b45414ee 100755..100644 --- a/moses/src/Manager.h +++ b/moses/src/Manager.h diff --git a/moses/src/MockHypothesis.cpp b/moses/src/MockHypothesis.cpp index 0afd3b071..0afd3b071 100755..100644 --- a/moses/src/MockHypothesis.cpp +++ b/moses/src/MockHypothesis.cpp diff --git a/moses/src/MockHypothesis.h b/moses/src/MockHypothesis.h index 41f118f40..41f118f40 100755..100644 --- a/moses/src/MockHypothesis.h +++ b/moses/src/MockHypothesis.h diff --git a/moses/src/MosesTest.cpp b/moses/src/MosesTest.cpp index 45f5bdba4..45f5bdba4 100755..100644 --- a/moses/src/MosesTest.cpp +++ b/moses/src/MosesTest.cpp diff --git a/moses/src/NonTerminal.cpp b/moses/src/NonTerminal.cpp index 37ebc9b68..37ebc9b68 100755..100644 --- a/moses/src/NonTerminal.cpp +++ b/moses/src/NonTerminal.cpp diff --git a/moses/src/NonTerminal.h b/moses/src/NonTerminal.h index 3fa2ede92..3fa2ede92 100755..100644 --- a/moses/src/NonTerminal.h +++ b/moses/src/NonTerminal.h diff --git a/moses/src/ObjectPool.h b/moses/src/ObjectPool.h index 05c849f79..05c849f79 100755..100644 --- a/moses/src/ObjectPool.h +++ b/moses/src/ObjectPool.h diff --git a/moses/src/OnlineCommand.cpp b/moses/src/OnlineCommand.cpp index 93024b92c..93024b92c 100755..100644 --- a/moses/src/OnlineCommand.cpp +++ b/moses/src/OnlineCommand.cpp diff --git a/moses/src/OnlineCommand.h b/moses/src/OnlineCommand.h index 90f778ea4..90f778ea4 100755..100644 --- a/moses/src/OnlineCommand.h +++ b/moses/src/OnlineCommand.h diff --git a/moses/src/OutputCollector.h b/moses/src/OutputCollector.h index 0b56c6d80..0b56c6d80 100755..100644 --- a/moses/src/OutputCollector.h +++ b/moses/src/OutputCollector.h diff --git a/moses/src/PCNTools.cpp b/moses/src/PCNTools.cpp index fac386444..fac386444 100755..100644 --- a/moses/src/PCNTools.cpp +++ b/moses/src/PCNTools.cpp diff --git a/moses/src/PCNTools.h b/moses/src/PCNTools.h index 1147b73bb..1147b73bb 100755..100644 --- a/moses/src/PCNTools.h +++ b/moses/src/PCNTools.h diff --git a/moses/src/PDTAimp.h b/moses/src/PDTAimp.h index 7bff39c07..0c2a984b0 100755..100644 --- a/moses/src/PDTAimp.h +++ b/moses/src/PDTAimp.h @@ -33,13 +33,11 @@ class PDTAimp protected: PDTAimp(PhraseDictionaryTreeAdaptor *p,unsigned nis) - : m_languageModels(0),m_weightWP(0.0),m_dict(0), + : m_languageModels(0),m_dict(0), m_obj(p),useCache(1),m_numInputScores(nis),totalE(0),distinctE(0) {} -public: - std::vector<float> m_weights; + public: LMList const* m_languageModels; - float m_weightWP; std::vector<FactorType> m_input,m_output; PhraseDictionaryTree *m_dict; typedef std::vector<TargetPhraseCollection const*> vTPC; @@ -139,6 +137,10 @@ public: return 0; } + const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); + std::vector<float> weights = system.GetTranslationWeights(m_obj->GetDictIndex()); + float weightWP = system.GetWeightWordPenalty(); + std::vector<TargetPhrase> tCands; tCands.reserve(cands.size()); std::vector<std::pair<float,size_t> > costs; @@ -165,7 +167,7 @@ public: *(cands[i].fnames[j]), cands[i].fvalues[j]); } } - CreateTargetPhrase(targetPhrase,factorStrings,scoreVector, sparseFeatures, wacands[i],&src); + CreateTargetPhrase(targetPhrase,factorStrings,scoreVector, sparseFeatures, wacands[i], weights, weightWP, &src); costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size())); tCands.push_back(targetPhrase); } @@ -188,9 +190,8 @@ public: void Create(const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath - , const std::vector<float> &weight + , const std::vector<float> &weight , const LMList &languageModels - , float weightWP ) { // set my members @@ -198,8 +199,6 @@ public: m_input=input; m_output=output; m_languageModels=&languageModels; - m_weightWP=weightWP; - m_weights=weight; const StaticData &staticData = StaticData::Instance(); m_dict->UseWordAlignment(staticData.UseAlignmentInfo()); @@ -262,8 +261,10 @@ public: Scores const& scoreVector, const ScoreComponentCollection& sparseFeatures, const std::string& alignmentString, + std::vector<float> &weights, + float weightWP, Phrase const* srcPtr=0) const { - CreateTargetPhrase(targetPhrase, factorStrings, scoreVector, sparseFeatures, srcPtr); + CreateTargetPhrase(targetPhrase, factorStrings, scoreVector, sparseFeatures, weights, weightWP, srcPtr); targetPhrase.SetAlignmentInfo(alignmentString); } @@ -272,6 +273,8 @@ public: StringTgtCand::Tokens const& factorStrings, Scores const& scoreVector, const ScoreComponentCollection& sparseFeatures, + std::vector<float> &weights, + float weightWP, Phrase const* srcPtr=0) const { FactorCollection &factorCollection = FactorCollection::Instance(); @@ -284,7 +287,7 @@ public: } } - targetPhrase.SetScore(m_obj->GetFeature(), scoreVector, sparseFeatures, m_weights, m_weightWP, *m_languageModels); + targetPhrase.SetScore(m_obj->GetFeature(), scoreVector, sparseFeatures, weights, weightWP, *m_languageModels); targetPhrase.SetSourcePhrase(*srcPtr); } @@ -366,6 +369,10 @@ public: for(Position i=0 ; i < srcSize ; ++i) stack.push_back(State(i, i, m_dict->GetRoot(), std::vector<float>(m_numInputScores,0.0))); + const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); + std::vector<float> weightT = system.GetTranslationWeights(m_obj->GetDictIndex()); + float weightWP = system.GetWeightWordPenalty(); + while(!stack.empty()) { State curr(stack.back()); stack.pop_back(); @@ -440,19 +447,13 @@ public: //put in phrase table scores, logging as we insert std::transform(tcands[i].scores.begin(),tcands[i].scores.end(),nscores.begin() + m_numInputScores,TransformScore); - - CHECK(nscores.size()==m_weights.size()); - - const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); + CHECK(nscores.size()==weightT.size()); + //tally up - std::vector<float> weightT = system.GetTranslationWeights(); - //float score=std::inner_product(nscores.begin(), nscores.end(), m_weights.begin(), 0.0f); float score=std::inner_product(nscores.begin(), nscores.end(), weightT.begin(), 0.0f); //count word penalty - float weightWP = system.GetWeightWordPenalty(); - //score-=tcands[i].tokens.size() * m_weightWP; score-=tcands[i].tokens.size() * weightWP; std::pair<E2Costs::iterator,bool> p=e2costs.insert(std::make_pair(tcands[i].tokens,TScores())); @@ -501,7 +502,7 @@ public: for(E2Costs::const_iterator j=i->second.begin(); j!=i->second.end(); ++j) { TScores const & scores=j->second; TargetPhrase targetPhrase(Output); - CreateTargetPhrase(targetPhrase,j->first,scores.trans,ScoreComponentCollection(),scores.src); + CreateTargetPhrase(targetPhrase,j->first,scores.trans,ScoreComponentCollection(),weightT,weightWP,scores.src); costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size())); tCands.push_back(targetPhrase); //std::cerr << i->first.first << "-" << i->first.second << ": " << targetPhrase << std::endl; diff --git a/moses/src/Parameter.cpp b/moses/src/Parameter.cpp index a24c6033c..a24c6033c 100755..100644 --- a/moses/src/Parameter.cpp +++ b/moses/src/Parameter.cpp diff --git a/moses/src/Parameter.h b/moses/src/Parameter.h index de0d3f7e7..de0d3f7e7 100755..100644 --- a/moses/src/Parameter.h +++ b/moses/src/Parameter.h diff --git a/moses/src/PartialTranslOptColl.cpp b/moses/src/PartialTranslOptColl.cpp index 91ad65415..91ad65415 100755..100644 --- a/moses/src/PartialTranslOptColl.cpp +++ b/moses/src/PartialTranslOptColl.cpp diff --git a/moses/src/PartialTranslOptColl.h b/moses/src/PartialTranslOptColl.h index b8b16b901..b8b16b901 100755..100644 --- a/moses/src/PartialTranslOptColl.h +++ b/moses/src/PartialTranslOptColl.h diff --git a/moses/src/Phrase.cpp b/moses/src/Phrase.cpp index db808390c..db808390c 100755..100644 --- a/moses/src/Phrase.cpp +++ b/moses/src/Phrase.cpp diff --git a/moses/src/Phrase.h b/moses/src/Phrase.h index b64a84a25..b64a84a25 100755..100644 --- a/moses/src/Phrase.h +++ b/moses/src/Phrase.h diff --git a/moses/src/PhraseBoundaryFeature.cpp b/moses/src/PhraseBoundaryFeature.cpp index 461fcbe57..461fcbe57 100755..100644 --- a/moses/src/PhraseBoundaryFeature.cpp +++ b/moses/src/PhraseBoundaryFeature.cpp diff --git a/moses/src/PhraseBoundaryFeature.h b/moses/src/PhraseBoundaryFeature.h index 4167ab9b2..4167ab9b2 100755..100644 --- a/moses/src/PhraseBoundaryFeature.h +++ b/moses/src/PhraseBoundaryFeature.h diff --git a/moses/src/PhraseDictionary.cpp b/moses/src/PhraseDictionary.cpp index 0fc39a286..49615456c 100755..100644 --- a/moses/src/PhraseDictionary.cpp +++ b/moses/src/PhraseDictionary.cpp @@ -46,6 +46,11 @@ GetTargetPhraseCollection(InputType const& src,WordsRange const& range) const return GetTargetPhraseCollection(src.GetSubString(range)); } +size_t PhraseDictionary::GetDictIndex() const +{ + return m_feature->GetDictIndex(); +} + PhraseDictionaryFeature::PhraseDictionaryFeature (PhraseTableImplementation implementation , SparsePhraseDictionaryFeature* spdf @@ -55,13 +60,14 @@ PhraseDictionaryFeature::PhraseDictionaryFeature , const std::vector<FactorType> &output , const std::string &filePath , const std::vector<float> &weight + , size_t dictIndex , size_t tableLimit , const std::string &targetFile // default param , const std::string &alignmentsFile) // default param :DecodeFeature("PhraseModel",numScoreComponent,input,output), m_numInputScores(numInputScores), m_filePath(filePath), - m_weight(weight), + m_dictIndex(dictIndex), m_tableLimit(tableLimit), m_implementation(implementation), m_targetFile(targetFile), @@ -70,6 +76,10 @@ PhraseDictionaryFeature::PhraseDictionaryFeature { if (implementation == Memory || implementation == SCFG || implementation == SuffixArray) { m_useThreadSafePhraseDictionary = true; + if (implementation == SuffixArray) { + cerr << "Warning: implementation holds chached weights!" << endl; + exit(1); + } } else { m_useThreadSafePhraseDictionary = false; } @@ -78,7 +88,7 @@ PhraseDictionaryFeature::PhraseDictionaryFeature PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSystem* system) { const StaticData& staticData = StaticData::Instance(); - std::vector<float> weightT = system->GetTranslationWeights(); + std::vector<float> weightT = system->GetTranslationWeights(m_dictIndex); if (m_implementation == Memory) { // memory phrase table @@ -95,7 +105,6 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst PhraseDictionaryMemory* pdm = new PhraseDictionaryMemory(GetNumScoreComponents(),this); bool ret = pdm->Load(GetInput(), GetOutput() , m_filePath - //, m_weight , weightT , m_tableLimit , system->GetLanguageModels() @@ -107,7 +116,6 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdta->Load( GetInput() , GetOutput() , m_filePath - //, m_weight , weightT , m_tableLimit , system->GetLanguageModels() @@ -126,8 +134,7 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdm->Load(GetInput() , GetOutput() , m_filePath - // m_weight - , weightT + , weightT , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); @@ -145,7 +152,6 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdm->Load(GetInput() , GetOutput() , m_filePath - //, m_weight , weightT , m_tableLimit , system->GetLanguageModels() @@ -154,6 +160,8 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst return pdm; } else if (m_implementation == ALSuffixArray) { // memory phrase table + cerr << "Warning: Implementation holds cached weights!" << endl; + exit(1); VERBOSE(2,"using Hiero format phrase tables" << std::endl); if (!FileExists(m_filePath) && FileExists(m_filePath + ".gz")) { m_filePath += ".gz"; @@ -164,9 +172,8 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdm->Load(GetInput() , GetOutput() , m_filePath - //, m_weight - , weightT - , m_tableLimit + , weightT + , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); CHECK(ret); @@ -177,14 +184,15 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdta->Load(GetInput() , GetOutput() , m_filePath - //, m_weight - , weightT - , m_tableLimit + , weightT + , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); CHECK(ret); return pdta; } else if (m_implementation == SuffixArray) { + cerr << "Warning: Implementation holds cached weights!" << endl; + exit(1); #ifndef WIN32 PhraseDictionaryDynSuffixArray *pd = new PhraseDictionaryDynSuffixArray(GetNumScoreComponents(), this); if(!(pd->Load( @@ -192,11 +200,10 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst ,GetOutput() ,m_filePath ,m_targetFile - , m_alignmentsFile - //, m_weight, m_tableLimit - , weightT, m_tableLimit - , system->GetLanguageModels() - , system->GetWeightWordPenalty()))) { + ,m_alignmentsFile + ,weightT, m_tableLimit + ,system->GetLanguageModels() + ,system->GetWeightWordPenalty()))) { std::cerr << "FAILED TO LOAD\n" << endl; delete pd; pd = NULL; @@ -297,5 +304,10 @@ const PhraseDictionaryFeature* PhraseDictionary::GetFeature() const return m_feature; } +size_t PhraseDictionaryFeature::GetDictIndex() const +{ + return m_dictIndex; +} + } diff --git a/moses/src/PhraseDictionary.h b/moses/src/PhraseDictionary.h index e8f478079..18b48436e 100755..100644 --- a/moses/src/PhraseDictionary.h +++ b/moses/src/PhraseDictionary.h @@ -69,7 +69,7 @@ public: return Translate; } const PhraseDictionaryFeature* GetFeature() const; - + size_t GetDictIndex() const; //! find list of translations that can translates src. Only for phrase input virtual const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase& src) const=0; @@ -105,6 +105,7 @@ public: , const std::vector<FactorType> &output , const std::string &filePath , const std::vector<float> &weight + , size_t dictIndex , size_t tableLimit , const std::string &targetFile , const std::string &alignmentsFile); @@ -131,6 +132,10 @@ public: //Get the dictionary. Be sure to initialise it first. const PhraseDictionary* GetDictionary() const; PhraseDictionary* GetDictionary(); + size_t GetDictIndex() const; + +protected: + size_t m_dictIndex; private: /** Load the appropriate phrase table */ @@ -138,7 +143,6 @@ private: unsigned m_numInputScores; std::string m_filePath; - std::vector<float> m_weight; size_t m_tableLimit; //We instantiate either the the thread-safe or non-thread-safe dictionary, //but not both. The thread-safe one can be instantiated in the constructor and shared diff --git a/moses/src/PhraseDictionaryALSuffixArray.cpp b/moses/src/PhraseDictionaryALSuffixArray.cpp index 71b810a91..99fe968f7 100755..100644 --- a/moses/src/PhraseDictionaryALSuffixArray.cpp +++ b/moses/src/PhraseDictionaryALSuffixArray.cpp @@ -56,8 +56,7 @@ void PhraseDictionaryALSuffixArray::InitializeForInput(InputType const& source) std::auto_ptr<RuleTableLoader> loader = RuleTableLoaderFactory::Create(grammarFile); - std::vector<float> weightT = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(); - //bool ret = loader->Load(*m_input, *m_output, inFile, *m_weight, m_tableLimit, + std::vector<float> weightT = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(GetDictIndex()); bool ret = loader->Load(*m_input, *m_output, inFile, weightT, m_tableLimit, *m_languageModels, m_wpProducer, *this); diff --git a/moses/src/PhraseDictionaryALSuffixArray.h b/moses/src/PhraseDictionaryALSuffixArray.h index 79c928c69..79c928c69 100755..100644 --- a/moses/src/PhraseDictionaryALSuffixArray.h +++ b/moses/src/PhraseDictionaryALSuffixArray.h diff --git a/moses/src/PhraseDictionaryDynSuffixArray.cpp b/moses/src/PhraseDictionaryDynSuffixArray.cpp index d4f0161db..d4f0161db 100755..100644 --- a/moses/src/PhraseDictionaryDynSuffixArray.cpp +++ b/moses/src/PhraseDictionaryDynSuffixArray.cpp diff --git a/moses/src/PhraseDictionaryDynSuffixArray.h b/moses/src/PhraseDictionaryDynSuffixArray.h index 0d6be08dd..0d6be08dd 100755..100644 --- a/moses/src/PhraseDictionaryDynSuffixArray.h +++ b/moses/src/PhraseDictionaryDynSuffixArray.h diff --git a/moses/src/PhraseDictionaryHiero.cpp b/moses/src/PhraseDictionaryHiero.cpp index 5899ec8b1..5899ec8b1 100755..100644 --- a/moses/src/PhraseDictionaryHiero.cpp +++ b/moses/src/PhraseDictionaryHiero.cpp diff --git a/moses/src/PhraseDictionaryHiero.h b/moses/src/PhraseDictionaryHiero.h index f845c4e15..f845c4e15 100755..100644 --- a/moses/src/PhraseDictionaryHiero.h +++ b/moses/src/PhraseDictionaryHiero.h diff --git a/moses/src/PhraseDictionaryMemory.cpp b/moses/src/PhraseDictionaryMemory.cpp index 1ee449172..1ee449172 100755..100644 --- a/moses/src/PhraseDictionaryMemory.cpp +++ b/moses/src/PhraseDictionaryMemory.cpp diff --git a/moses/src/PhraseDictionaryMemory.h b/moses/src/PhraseDictionaryMemory.h index dad7f4a9b..dad7f4a9b 100755..100644 --- a/moses/src/PhraseDictionaryMemory.h +++ b/moses/src/PhraseDictionaryMemory.h diff --git a/moses/src/PhraseDictionaryNode.cpp b/moses/src/PhraseDictionaryNode.cpp index 5b552bd53..5b552bd53 100755..100644 --- a/moses/src/PhraseDictionaryNode.cpp +++ b/moses/src/PhraseDictionaryNode.cpp diff --git a/moses/src/PhraseDictionaryNode.h b/moses/src/PhraseDictionaryNode.h index 95ae48b0e..95ae48b0e 100755..100644 --- a/moses/src/PhraseDictionaryNode.h +++ b/moses/src/PhraseDictionaryNode.h diff --git a/moses/src/PhraseDictionaryNodeSCFG.cpp b/moses/src/PhraseDictionaryNodeSCFG.cpp index 91f5f7acd..91f5f7acd 100755..100644 --- a/moses/src/PhraseDictionaryNodeSCFG.cpp +++ b/moses/src/PhraseDictionaryNodeSCFG.cpp diff --git a/moses/src/PhraseDictionaryNodeSCFG.h b/moses/src/PhraseDictionaryNodeSCFG.h index 80e3722b5..80e3722b5 100755..100644 --- a/moses/src/PhraseDictionaryNodeSCFG.h +++ b/moses/src/PhraseDictionaryNodeSCFG.h diff --git a/moses/src/PhraseDictionaryOnDisk.cpp b/moses/src/PhraseDictionaryOnDisk.cpp index 3c89408f5..5bada2aea 100755..100644 --- a/moses/src/PhraseDictionaryOnDisk.cpp +++ b/moses/src/PhraseDictionaryOnDisk.cpp @@ -39,7 +39,7 @@ PhraseDictionaryOnDisk::~PhraseDictionaryOnDisk() bool PhraseDictionaryOnDisk::Load(const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath - , const std::vector<float> &weight + , const std::vector<float> &weight , size_t tableLimit , const LMList& languageModels , const WordPenaltyProducer* wpProducer) @@ -51,8 +51,6 @@ bool PhraseDictionaryOnDisk::Load(const std::vector<FactorType> &input m_inputFactorsVec = input; m_outputFactorsVec = output; - m_weight = weight; - LoadTargetLookup(); if (!m_dbWrapper.BeginLoad(filePath)) @@ -94,13 +92,11 @@ ChartRuleLookupManager *PhraseDictionaryOnDisk::CreateRuleLookupManager( const InputType &sentence, const ChartCellCollection &cellCollection) { - std::vector<float> weightT = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(); + std::vector<float> weightT = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(GetDictIndex()); return new ChartRuleLookupManagerOnDisk(sentence, cellCollection, *this, m_dbWrapper, m_languageModels, m_wpProducer, m_inputFactorsVec, - //m_outputFactorsVec, m_weight, - m_outputFactorsVec, weightT, - m_filePath); + m_outputFactorsVec, m_filePath); } } diff --git a/moses/src/PhraseDictionaryOnDisk.h b/moses/src/PhraseDictionaryOnDisk.h index 0a8dfc808..64c77cbd7 100755..100644 --- a/moses/src/PhraseDictionaryOnDisk.h +++ b/moses/src/PhraseDictionaryOnDisk.h @@ -45,7 +45,6 @@ protected: const LMList* m_languageModels; const WordPenaltyProducer* m_wpProducer; std::vector<FactorType> m_inputFactorsVec, m_outputFactorsVec; - std::vector<float> m_weight; std::string m_filePath; void LoadTargetLookup(); @@ -63,7 +62,7 @@ public: bool Load(const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath - , const std::vector<float> &weight + , const std::vector<float> &weight , size_t tableLimit, const LMList& languageModels, const WordPenaltyProducer* wpProducer); diff --git a/moses/src/PhraseDictionarySCFG.cpp b/moses/src/PhraseDictionarySCFG.cpp index c3a243990..c3a243990 100755..100644 --- a/moses/src/PhraseDictionarySCFG.cpp +++ b/moses/src/PhraseDictionarySCFG.cpp diff --git a/moses/src/PhraseDictionarySCFG.h b/moses/src/PhraseDictionarySCFG.h index e338dac50..e338dac50 100755..100644 --- a/moses/src/PhraseDictionarySCFG.h +++ b/moses/src/PhraseDictionarySCFG.h diff --git a/moses/src/PhraseDictionaryTree.cpp b/moses/src/PhraseDictionaryTree.cpp index 13b404724..13b404724 100755..100644 --- a/moses/src/PhraseDictionaryTree.cpp +++ b/moses/src/PhraseDictionaryTree.cpp diff --git a/moses/src/PhraseDictionaryTree.h b/moses/src/PhraseDictionaryTree.h index ebe43a618..ebe43a618 100755..100644 --- a/moses/src/PhraseDictionaryTree.h +++ b/moses/src/PhraseDictionaryTree.h diff --git a/moses/src/PhraseDictionaryTreeAdaptor.cpp b/moses/src/PhraseDictionaryTreeAdaptor.cpp index cce4a2ee4..154572bce 100755..100644 --- a/moses/src/PhraseDictionaryTreeAdaptor.cpp +++ b/moses/src/PhraseDictionaryTreeAdaptor.cpp @@ -55,8 +55,7 @@ bool PhraseDictionaryTreeAdaptor::Load(const std::vector<FactorType> &input // set PhraseDictionary members m_tableLimit=tableLimit; - imp->Create(input,output,filePath, - weight,languageModels,weightWP); + imp->Create(input,output,filePath,weight,languageModels); return true; } diff --git a/moses/src/PhraseDictionaryTreeAdaptor.h b/moses/src/PhraseDictionaryTreeAdaptor.h index c38e234fb..c38e234fb 100755..100644 --- a/moses/src/PhraseDictionaryTreeAdaptor.h +++ b/moses/src/PhraseDictionaryTreeAdaptor.h diff --git a/moses/src/PhraseLengthFeature.cpp b/moses/src/PhraseLengthFeature.cpp index ece3b0923..ece3b0923 100755..100644 --- a/moses/src/PhraseLengthFeature.cpp +++ b/moses/src/PhraseLengthFeature.cpp diff --git a/moses/src/PhraseLengthFeature.h b/moses/src/PhraseLengthFeature.h index f25142230..f25142230 100755..100644 --- a/moses/src/PhraseLengthFeature.h +++ b/moses/src/PhraseLengthFeature.h diff --git a/moses/src/PhrasePairFeature.cpp b/moses/src/PhrasePairFeature.cpp index c44807205..c44807205 100755..100644 --- a/moses/src/PhrasePairFeature.cpp +++ b/moses/src/PhrasePairFeature.cpp diff --git a/moses/src/PhrasePairFeature.h b/moses/src/PhrasePairFeature.h index 6dab1a998..6dab1a998 100755..100644 --- a/moses/src/PhrasePairFeature.h +++ b/moses/src/PhrasePairFeature.h diff --git a/moses/src/PrefixTree.h b/moses/src/PrefixTree.h index 3215fb834..3215fb834 100755..100644 --- a/moses/src/PrefixTree.h +++ b/moses/src/PrefixTree.h diff --git a/moses/src/PrefixTreeMap.cpp b/moses/src/PrefixTreeMap.cpp index 7e5512a05..7e5512a05 100755..100644 --- a/moses/src/PrefixTreeMap.cpp +++ b/moses/src/PrefixTreeMap.cpp diff --git a/moses/src/PrefixTreeMap.h b/moses/src/PrefixTreeMap.h index 6bb6b769f..6bb6b769f 100755..100644 --- a/moses/src/PrefixTreeMap.h +++ b/moses/src/PrefixTreeMap.h diff --git a/moses/src/ReorderingConstraint.cpp b/moses/src/ReorderingConstraint.cpp index ea0c1b807..ea0c1b807 100755..100644 --- a/moses/src/ReorderingConstraint.cpp +++ b/moses/src/ReorderingConstraint.cpp diff --git a/moses/src/ReorderingConstraint.h b/moses/src/ReorderingConstraint.h index 742b2228b..742b2228b 100755..100644 --- a/moses/src/ReorderingConstraint.h +++ b/moses/src/ReorderingConstraint.h diff --git a/moses/src/ReorderingStack.cpp b/moses/src/ReorderingStack.cpp index 49a723a36..49a723a36 100755..100644 --- a/moses/src/ReorderingStack.cpp +++ b/moses/src/ReorderingStack.cpp diff --git a/moses/src/ReorderingStack.h b/moses/src/ReorderingStack.h index 9a710b348..9a710b348 100755..100644 --- a/moses/src/ReorderingStack.h +++ b/moses/src/ReorderingStack.h diff --git a/moses/src/RuleCube.cpp b/moses/src/RuleCube.cpp index b028abe3e..b028abe3e 100755..100644 --- a/moses/src/RuleCube.cpp +++ b/moses/src/RuleCube.cpp diff --git a/moses/src/RuleCube.h b/moses/src/RuleCube.h index 5d6aabe46..5d6aabe46 100755..100644 --- a/moses/src/RuleCube.h +++ b/moses/src/RuleCube.h diff --git a/moses/src/RuleCubeItem.cpp b/moses/src/RuleCubeItem.cpp index fa4b4c51e..fa4b4c51e 100755..100644 --- a/moses/src/RuleCubeItem.cpp +++ b/moses/src/RuleCubeItem.cpp diff --git a/moses/src/RuleCubeItem.h b/moses/src/RuleCubeItem.h index 1b953b1b4..1b953b1b4 100755..100644 --- a/moses/src/RuleCubeItem.h +++ b/moses/src/RuleCubeItem.h diff --git a/moses/src/RuleCubeQueue.cpp b/moses/src/RuleCubeQueue.cpp index 89020a4e5..89020a4e5 100755..100644 --- a/moses/src/RuleCubeQueue.cpp +++ b/moses/src/RuleCubeQueue.cpp diff --git a/moses/src/RuleCubeQueue.h b/moses/src/RuleCubeQueue.h index 5ef3b3af2..5ef3b3af2 100755..100644 --- a/moses/src/RuleCubeQueue.h +++ b/moses/src/RuleCubeQueue.h diff --git a/moses/src/RuleTableLoader.h b/moses/src/RuleTableLoader.h index b5d4d8e24..b5d4d8e24 100755..100644 --- a/moses/src/RuleTableLoader.h +++ b/moses/src/RuleTableLoader.h diff --git a/moses/src/RuleTableLoaderCompact.cpp b/moses/src/RuleTableLoaderCompact.cpp index 64297948f..64297948f 100755..100644 --- a/moses/src/RuleTableLoaderCompact.cpp +++ b/moses/src/RuleTableLoaderCompact.cpp diff --git a/moses/src/RuleTableLoaderCompact.h b/moses/src/RuleTableLoaderCompact.h index 56918782b..56918782b 100755..100644 --- a/moses/src/RuleTableLoaderCompact.h +++ b/moses/src/RuleTableLoaderCompact.h diff --git a/moses/src/RuleTableLoaderFactory.cpp b/moses/src/RuleTableLoaderFactory.cpp index 7ac97bde2..7ac97bde2 100755..100644 --- a/moses/src/RuleTableLoaderFactory.cpp +++ b/moses/src/RuleTableLoaderFactory.cpp diff --git a/moses/src/RuleTableLoaderFactory.h b/moses/src/RuleTableLoaderFactory.h index c9c4daf1d..c9c4daf1d 100755..100644 --- a/moses/src/RuleTableLoaderFactory.h +++ b/moses/src/RuleTableLoaderFactory.h diff --git a/moses/src/RuleTableLoaderHiero.cpp b/moses/src/RuleTableLoaderHiero.cpp index 6899cfa61..6899cfa61 100755..100644 --- a/moses/src/RuleTableLoaderHiero.cpp +++ b/moses/src/RuleTableLoaderHiero.cpp diff --git a/moses/src/RuleTableLoaderHiero.h b/moses/src/RuleTableLoaderHiero.h index a1f8a9205..a1f8a9205 100755..100644 --- a/moses/src/RuleTableLoaderHiero.h +++ b/moses/src/RuleTableLoaderHiero.h diff --git a/moses/src/RuleTableLoaderStandard.cpp b/moses/src/RuleTableLoaderStandard.cpp index 122e75c8c..122e75c8c 100755..100644 --- a/moses/src/RuleTableLoaderStandard.cpp +++ b/moses/src/RuleTableLoaderStandard.cpp diff --git a/moses/src/RuleTableLoaderStandard.h b/moses/src/RuleTableLoaderStandard.h index 9477db962..9477db962 100755..100644 --- a/moses/src/RuleTableLoaderStandard.h +++ b/moses/src/RuleTableLoaderStandard.h diff --git a/moses/src/ScoreComponentCollection.cpp b/moses/src/ScoreComponentCollection.cpp index d9638f21c..e393c3a74 100755..100644 --- a/moses/src/ScoreComponentCollection.cpp +++ b/moses/src/ScoreComponentCollection.cpp @@ -137,12 +137,12 @@ float ScoreComponentCollection::GetLInfNorm() const { return m_scores.linfnorm(); } -void ScoreComponentCollection::L1Regularize(float lambda) { - m_scores.l1regularize(lambda); +size_t ScoreComponentCollection::SparseL1Regularize(float lambda) { + return m_scores.sparseL1regularize(lambda); } -void ScoreComponentCollection::L2Regularize(float lambda) { - m_scores.l2regularize(lambda); +void ScoreComponentCollection::SparseL2Regularize(float lambda) { + m_scores.sparseL2regularize(lambda); } void ScoreComponentCollection::Save(ostream& out) const { diff --git a/moses/src/ScoreComponentCollection.h b/moses/src/ScoreComponentCollection.h index f80055613..e170d0cd9 100755..100644 --- a/moses/src/ScoreComponentCollection.h +++ b/moses/src/ScoreComponentCollection.h @@ -234,25 +234,29 @@ public: m_scores[fname] += score; } - void Assign(const ScoreProducer* sp, const std::vector<float>& scores) - { + void Assign(const ScoreProducer* sp, const std::vector<float>& scores) + { IndexPair indexes = GetIndexes(sp); CHECK(scores.size() == indexes.second - indexes.first); for (size_t i = 0; i < scores.size(); ++i) { m_scores[i + indexes.first] = scores[i]; } - } - - //! Special version Assign(ScoreProducer, vector<float>) - //! to add the score from a single ScoreProducer that produces - //! a single value - void Assign(const ScoreProducer* sp, float score) - { + } + + //! Special version Assign(ScoreProducer, vector<float>) + //! to add the score from a single ScoreProducer that produces + //! a single value + void Assign(const ScoreProducer* sp, float score) + { IndexPair indexes = GetIndexes(sp); CHECK(1 == indexes.second - indexes.first); m_scores[indexes.first] = score; - } - + } + + // Assign core weight by index + void Assign(size_t index, float score) { + m_scores[index] = score; + } //For features which have an unbounded number of components void Assign(const ScoreProducer*sp, const std::string name, float score) @@ -367,8 +371,8 @@ public: float GetL1Norm() const; float GetL2Norm() const; float GetLInfNorm() const; - void L1Regularize(float lambda); - void L2Regularize(float lambda); + size_t SparseL1Regularize(float lambda); + void SparseL2Regularize(float lambda); void Save(const std::string& filename) const; void Save(std::ostream&) const; @@ -381,7 +385,7 @@ public: size_t PruneSparseFeatures(size_t threshold) { return m_scores.pruneSparseFeatures(threshold); } size_t PruneZeroWeightFeatures() { return m_scores.pruneZeroWeightFeatures(); } void UpdateConfidenceCounts(ScoreComponentCollection &weightUpdate, bool signedCounts) { m_scores.updateConfidenceCounts(weightUpdate.m_scores, signedCounts); } - void UpdateLearningRates(float decay, ScoreComponentCollection &confidenceCounts, float core_r0, float sparse_r0) { m_scores.updateLearningRates(decay, confidenceCounts.m_scores, core_r0, sparse_r0); } + void UpdateLearningRates(float decay_core, float decay_sparse, ScoreComponentCollection &confidenceCounts, float core_r0, float sparse_r0) { m_scores.updateLearningRates(decay_core, decay_sparse, confidenceCounts.m_scores, core_r0, sparse_r0); } #ifdef MPI_ENABLE public: diff --git a/moses/src/ScoreComponentCollectionTest.cpp b/moses/src/ScoreComponentCollectionTest.cpp index 10e96a017..10e96a017 100755..100644 --- a/moses/src/ScoreComponentCollectionTest.cpp +++ b/moses/src/ScoreComponentCollectionTest.cpp diff --git a/moses/src/ScoreProducer.cpp b/moses/src/ScoreProducer.cpp index 07823125c..07823125c 100755..100644 --- a/moses/src/ScoreProducer.cpp +++ b/moses/src/ScoreProducer.cpp diff --git a/moses/src/ScoreProducer.h b/moses/src/ScoreProducer.h index c88e5efca..c88e5efca 100755..100644 --- a/moses/src/ScoreProducer.h +++ b/moses/src/ScoreProducer.h diff --git a/moses/src/Search.cpp b/moses/src/Search.cpp index aa3bfe59f..aa3bfe59f 100755..100644 --- a/moses/src/Search.cpp +++ b/moses/src/Search.cpp diff --git a/moses/src/Search.h b/moses/src/Search.h index 67a6c69ea..67a6c69ea 100755..100644 --- a/moses/src/Search.h +++ b/moses/src/Search.h diff --git a/moses/src/SearchCubePruning.cpp b/moses/src/SearchCubePruning.cpp index 9fe6aec24..9fe6aec24 100755..100644 --- a/moses/src/SearchCubePruning.cpp +++ b/moses/src/SearchCubePruning.cpp diff --git a/moses/src/SearchCubePruning.h b/moses/src/SearchCubePruning.h index 9f8f73ef5..9f8f73ef5 100755..100644 --- a/moses/src/SearchCubePruning.h +++ b/moses/src/SearchCubePruning.h diff --git a/moses/src/SearchNormal.cpp b/moses/src/SearchNormal.cpp index 9f26270e9..9f26270e9 100755..100644 --- a/moses/src/SearchNormal.cpp +++ b/moses/src/SearchNormal.cpp diff --git a/moses/src/SearchNormal.h b/moses/src/SearchNormal.h index 8b4f904d9..8b4f904d9 100755..100644 --- a/moses/src/SearchNormal.h +++ b/moses/src/SearchNormal.h diff --git a/moses/src/Sentence.cpp b/moses/src/Sentence.cpp index 210e5ccb5..210e5ccb5 100755..100644 --- a/moses/src/Sentence.cpp +++ b/moses/src/Sentence.cpp diff --git a/moses/src/Sentence.h b/moses/src/Sentence.h index b1c25f9dd..b1c25f9dd 100755..100644 --- a/moses/src/Sentence.h +++ b/moses/src/Sentence.h diff --git a/moses/src/SentenceStats.cpp b/moses/src/SentenceStats.cpp index 218fe2187..218fe2187 100755..100644 --- a/moses/src/SentenceStats.cpp +++ b/moses/src/SentenceStats.cpp diff --git a/moses/src/SentenceStats.h b/moses/src/SentenceStats.h index 6a72008ba..6a72008ba 100755..100644 --- a/moses/src/SentenceStats.h +++ b/moses/src/SentenceStats.h diff --git a/moses/src/SourceWordDeletionFeature.cpp b/moses/src/SourceWordDeletionFeature.cpp index 4f247c7e7..4f247c7e7 100755..100644 --- a/moses/src/SourceWordDeletionFeature.cpp +++ b/moses/src/SourceWordDeletionFeature.cpp diff --git a/moses/src/SourceWordDeletionFeature.h b/moses/src/SourceWordDeletionFeature.h index 6d8d21a42..6d8d21a42 100755..100644 --- a/moses/src/SourceWordDeletionFeature.h +++ b/moses/src/SourceWordDeletionFeature.h diff --git a/moses/src/SparsePhraseDictionaryFeature.cpp b/moses/src/SparsePhraseDictionaryFeature.cpp index 8b1a35ee3..8b1a35ee3 100755..100644 --- a/moses/src/SparsePhraseDictionaryFeature.cpp +++ b/moses/src/SparsePhraseDictionaryFeature.cpp diff --git a/moses/src/SparsePhraseDictionaryFeature.h b/moses/src/SparsePhraseDictionaryFeature.h index ad38e7078..ad38e7078 100755..100644 --- a/moses/src/SparsePhraseDictionaryFeature.h +++ b/moses/src/SparsePhraseDictionaryFeature.h diff --git a/moses/src/SquareMatrix.cpp b/moses/src/SquareMatrix.cpp index 276d10053..276d10053 100755..100644 --- a/moses/src/SquareMatrix.cpp +++ b/moses/src/SquareMatrix.cpp diff --git a/moses/src/SquareMatrix.h b/moses/src/SquareMatrix.h index 3d36962d4..3d36962d4 100755..100644 --- a/moses/src/SquareMatrix.h +++ b/moses/src/SquareMatrix.h diff --git a/moses/src/StaticData.cpp b/moses/src/StaticData.cpp index 218be7f2b..6ccb8fb34 100755..100644 --- a/moses/src/StaticData.cpp +++ b/moses/src/StaticData.cpp @@ -1397,6 +1397,7 @@ bool StaticData::LoadPhraseTables() , output , filePath , weight + , currDict , maxTargetPhrase[index] , targetPath, alignmentsFile); @@ -1551,6 +1552,7 @@ bool StaticData::LoadDecodeGraphs() DecodeGraph *decodeGraph; if (m_searchAlgorithm == ChartDecoding) { size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN; + cerr << "max-chart-span: " << maxChartSpans[decodeGraphInd] << endl; decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan); } else { decodeGraph = new DecodeGraph(m_decodeGraphs.size()); diff --git a/moses/src/StaticData.h b/moses/src/StaticData.h index 1b8075e25..1b8075e25 100755..100644 --- a/moses/src/StaticData.h +++ b/moses/src/StaticData.h diff --git a/moses/src/SyntacticLanguageModel.cpp b/moses/src/SyntacticLanguageModel.cpp index c9b3f1e22..c9b3f1e22 100755..100644 --- a/moses/src/SyntacticLanguageModel.cpp +++ b/moses/src/SyntacticLanguageModel.cpp diff --git a/moses/src/SyntacticLanguageModel.h b/moses/src/SyntacticLanguageModel.h index 3497d3dc4..3497d3dc4 100755..100644 --- a/moses/src/SyntacticLanguageModel.h +++ b/moses/src/SyntacticLanguageModel.h diff --git a/moses/src/SyntacticLanguageModelFiles.h b/moses/src/SyntacticLanguageModelFiles.h index c6533c424..c6533c424 100755..100644 --- a/moses/src/SyntacticLanguageModelFiles.h +++ b/moses/src/SyntacticLanguageModelFiles.h diff --git a/moses/src/SyntacticLanguageModelState.h b/moses/src/SyntacticLanguageModelState.h index 0877a59b3..0877a59b3 100755..100644 --- a/moses/src/SyntacticLanguageModelState.h +++ b/moses/src/SyntacticLanguageModelState.h diff --git a/moses/src/TargetBigramFeature.cpp b/moses/src/TargetBigramFeature.cpp index a9ad2216b..a9ad2216b 100755..100644 --- a/moses/src/TargetBigramFeature.cpp +++ b/moses/src/TargetBigramFeature.cpp diff --git a/moses/src/TargetBigramFeature.h b/moses/src/TargetBigramFeature.h index 76b4f6ef7..76b4f6ef7 100755..100644 --- a/moses/src/TargetBigramFeature.h +++ b/moses/src/TargetBigramFeature.h diff --git a/moses/src/TargetBigramFeatureTest.cpp b/moses/src/TargetBigramFeatureTest.cpp index 2ee7243bb..2ee7243bb 100755..100644 --- a/moses/src/TargetBigramFeatureTest.cpp +++ b/moses/src/TargetBigramFeatureTest.cpp diff --git a/moses/src/TargetNgramFeature.cpp b/moses/src/TargetNgramFeature.cpp index 3fefdfba2..3fefdfba2 100755..100644 --- a/moses/src/TargetNgramFeature.cpp +++ b/moses/src/TargetNgramFeature.cpp diff --git a/moses/src/TargetNgramFeature.h b/moses/src/TargetNgramFeature.h index 681e7d6aa..681e7d6aa 100755..100644 --- a/moses/src/TargetNgramFeature.h +++ b/moses/src/TargetNgramFeature.h diff --git a/moses/src/TargetPhrase.cpp b/moses/src/TargetPhrase.cpp index c79b844f8..2783a7604 100755..100644 --- a/moses/src/TargetPhrase.cpp +++ b/moses/src/TargetPhrase.cpp @@ -142,12 +142,7 @@ void TargetPhrase::SetScore(const ScoreProducer* translationScoreProducer, CHECK(weightT.size() == scoreVector.size()); // calc average score if non-best - const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); - std::vector<float> weightsT = system.GetTranslationWeights(); - weightWP = system.GetWeightWordPenalty(); - - //m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); - m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightsT.begin(), 0.0f); + m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); m_scoreBreakdown.PlusEquals(translationScoreProducer, scoreVector); m_scoreBreakdown.PlusEquals(sparseScoreVector); @@ -197,15 +192,10 @@ void TargetPhrase::SetScoreChart(const ScoreProducer* translationScoreProducer, ,const LMList &languageModels ,const WordPenaltyProducer* wpProducer) { - CHECK(weightT.size() == scoreVector.size()); - - const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); - std::vector<float> weightsT = system.GetTranslationWeights(); // calc average score if non-best - //m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); - m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightsT.begin(), 0.0f); + m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); m_scoreBreakdown.PlusEquals(translationScoreProducer, scoreVector); // Replicated from TranslationOptions.cpp @@ -269,11 +259,7 @@ void TargetPhrase::SetWeights(const ScoreProducer* translationScoreProducer, con addition to the usual phrase translation scaling factors) the input weight factor as last element */ - - const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); - std::vector<float> weightsT = system.GetTranslationWeights(); - //m_transScore = m_scoreBreakdown.PartialInnerProduct(translationScoreProducer, weightT); - m_transScore = m_scoreBreakdown.PartialInnerProduct(translationScoreProducer, weightsT); + m_transScore = m_scoreBreakdown.PartialInnerProduct(translationScoreProducer, weightT); } void TargetPhrase::ResetScore() diff --git a/moses/src/TargetPhrase.h b/moses/src/TargetPhrase.h index 377985c5e..377985c5e 100755..100644 --- a/moses/src/TargetPhrase.h +++ b/moses/src/TargetPhrase.h diff --git a/moses/src/TargetPhraseCollection.cpp b/moses/src/TargetPhraseCollection.cpp index 38570f8d5..38570f8d5 100755..100644 --- a/moses/src/TargetPhraseCollection.cpp +++ b/moses/src/TargetPhraseCollection.cpp diff --git a/moses/src/TargetPhraseCollection.h b/moses/src/TargetPhraseCollection.h index f4124f458..f4124f458 100755..100644 --- a/moses/src/TargetPhraseCollection.h +++ b/moses/src/TargetPhraseCollection.h diff --git a/moses/src/TargetWordInsertionFeature.cpp b/moses/src/TargetWordInsertionFeature.cpp index f7d5a5c7e..f7d5a5c7e 100755..100644 --- a/moses/src/TargetWordInsertionFeature.cpp +++ b/moses/src/TargetWordInsertionFeature.cpp diff --git a/moses/src/TargetWordInsertionFeature.h b/moses/src/TargetWordInsertionFeature.h index 0d53582d5..0d53582d5 100755..100644 --- a/moses/src/TargetWordInsertionFeature.h +++ b/moses/src/TargetWordInsertionFeature.h diff --git a/moses/src/ThreadPool.cpp b/moses/src/ThreadPool.cpp index ca9bd2f60..ca9bd2f60 100755..100644 --- a/moses/src/ThreadPool.cpp +++ b/moses/src/ThreadPool.cpp diff --git a/moses/src/ThreadPool.h b/moses/src/ThreadPool.h index a76872027..a76872027 100755..100644 --- a/moses/src/ThreadPool.h +++ b/moses/src/ThreadPool.h diff --git a/moses/src/Timer.cpp b/moses/src/Timer.cpp index bbe1bcabd..bbe1bcabd 100755..100644 --- a/moses/src/Timer.cpp +++ b/moses/src/Timer.cpp diff --git a/moses/src/Timer.h b/moses/src/Timer.h index 1d7c4aacc..1d7c4aacc 100755..100644 --- a/moses/src/Timer.h +++ b/moses/src/Timer.h diff --git a/moses/src/TranslationOption.cpp b/moses/src/TranslationOption.cpp index c823dadb8..c823dadb8 100755..100644 --- a/moses/src/TranslationOption.cpp +++ b/moses/src/TranslationOption.cpp diff --git a/moses/src/TranslationOption.h b/moses/src/TranslationOption.h index fa4065b80..fa4065b80 100755..100644 --- a/moses/src/TranslationOption.h +++ b/moses/src/TranslationOption.h diff --git a/moses/src/TranslationOptionCollection.cpp b/moses/src/TranslationOptionCollection.cpp index d9eefc9d6..d9eefc9d6 100755..100644 --- a/moses/src/TranslationOptionCollection.cpp +++ b/moses/src/TranslationOptionCollection.cpp diff --git a/moses/src/TranslationOptionCollection.h b/moses/src/TranslationOptionCollection.h index e1fa26a15..e1fa26a15 100755..100644 --- a/moses/src/TranslationOptionCollection.h +++ b/moses/src/TranslationOptionCollection.h diff --git a/moses/src/TranslationOptionCollectionConfusionNet.cpp b/moses/src/TranslationOptionCollectionConfusionNet.cpp index c2713fb3a..c2713fb3a 100755..100644 --- a/moses/src/TranslationOptionCollectionConfusionNet.cpp +++ b/moses/src/TranslationOptionCollectionConfusionNet.cpp diff --git a/moses/src/TranslationOptionCollectionConfusionNet.h b/moses/src/TranslationOptionCollectionConfusionNet.h index 66ef953a8..66ef953a8 100755..100644 --- a/moses/src/TranslationOptionCollectionConfusionNet.h +++ b/moses/src/TranslationOptionCollectionConfusionNet.h diff --git a/moses/src/TranslationOptionCollectionText.cpp b/moses/src/TranslationOptionCollectionText.cpp index c3676eaef..c3676eaef 100755..100644 --- a/moses/src/TranslationOptionCollectionText.cpp +++ b/moses/src/TranslationOptionCollectionText.cpp diff --git a/moses/src/TranslationOptionCollectionText.h b/moses/src/TranslationOptionCollectionText.h index bb819441e..bb819441e 100755..100644 --- a/moses/src/TranslationOptionCollectionText.h +++ b/moses/src/TranslationOptionCollectionText.h diff --git a/moses/src/TranslationOptionList.cpp b/moses/src/TranslationOptionList.cpp index 1d99729fe..1d99729fe 100755..100644 --- a/moses/src/TranslationOptionList.cpp +++ b/moses/src/TranslationOptionList.cpp diff --git a/moses/src/TranslationOptionList.h b/moses/src/TranslationOptionList.h index d56578029..d56578029 100755..100644 --- a/moses/src/TranslationOptionList.h +++ b/moses/src/TranslationOptionList.h diff --git a/moses/src/TranslationSystem.cpp b/moses/src/TranslationSystem.cpp index 722ea864e..e017ba241 100755..100644 --- a/moses/src/TranslationSystem.cpp +++ b/moses/src/TranslationSystem.cpp @@ -175,8 +175,8 @@ namespace Moses { return StaticData::Instance().GetWeight(m_distortionScoreProducer); } - std::vector<float> TranslationSystem::GetTranslationWeights() const { - std::vector<float> weights = StaticData::Instance().GetWeights(GetTranslationScoreProducer()); + std::vector<float> TranslationSystem::GetTranslationWeights(size_t index) const { + std::vector<float> weights = StaticData::Instance().GetWeights(GetTranslationScoreProducer(index)); //VERBOSE(1, "Read weightT from translation sytem.. "); for (size_t i = 0; i < weights.size(); ++i) //VERBOSE(1, weights[i] << " "); diff --git a/moses/src/TranslationSystem.h b/moses/src/TranslationSystem.h index 615fc7d8a..6f66fd25a 100755..100644 --- a/moses/src/TranslationSystem.h +++ b/moses/src/TranslationSystem.h @@ -83,12 +83,12 @@ class TranslationSystem { const UnknownWordPenaltyProducer *GetUnknownWordPenaltyProducer() const { return m_unknownWpProducer; } const DistortionScoreProducer* GetDistortionProducer() const {return m_distortionScoreProducer;} - const PhraseDictionaryFeature *GetTranslationScoreProducer() const { return GetPhraseDictionaries()[0]; } + const PhraseDictionaryFeature *GetTranslationScoreProducer(size_t index) const { return GetPhraseDictionaries()[index]; } float GetWeightWordPenalty() const; float GetWeightUnknownWordPenalty() const; float GetWeightDistortion() const; - std::vector<float> GetTranslationWeights() const; + std::vector<float> GetTranslationWeights(size_t index) const; //sentence (and thread) specific initialisationn and cleanup void InitializeBeforeSentenceProcessing(const InputType& source) const; diff --git a/moses/src/TreeInput.cpp b/moses/src/TreeInput.cpp index d6e0f1e78..d6e0f1e78 100755..100644 --- a/moses/src/TreeInput.cpp +++ b/moses/src/TreeInput.cpp diff --git a/moses/src/TreeInput.h b/moses/src/TreeInput.h index a6242d97e..a6242d97e 100755..100644 --- a/moses/src/TreeInput.h +++ b/moses/src/TreeInput.h diff --git a/moses/src/TrellisPath.cpp b/moses/src/TrellisPath.cpp index c73575b2c..c73575b2c 100755..100644 --- a/moses/src/TrellisPath.cpp +++ b/moses/src/TrellisPath.cpp diff --git a/moses/src/TrellisPath.h b/moses/src/TrellisPath.h index c1b347de9..c1b347de9 100755..100644 --- a/moses/src/TrellisPath.h +++ b/moses/src/TrellisPath.h diff --git a/moses/src/TrellisPathCollection.cpp b/moses/src/TrellisPathCollection.cpp index 89d429f4b..89d429f4b 100755..100644 --- a/moses/src/TrellisPathCollection.cpp +++ b/moses/src/TrellisPathCollection.cpp diff --git a/moses/src/TrellisPathCollection.h b/moses/src/TrellisPathCollection.h index 82e771d78..82e771d78 100755..100644 --- a/moses/src/TrellisPathCollection.h +++ b/moses/src/TrellisPathCollection.h diff --git a/moses/src/TrellisPathList.h b/moses/src/TrellisPathList.h index 073efb9b8..073efb9b8 100755..100644 --- a/moses/src/TrellisPathList.h +++ b/moses/src/TrellisPathList.h diff --git a/moses/src/TypeDef.h b/moses/src/TypeDef.h index bb439f5c9..bb439f5c9 100755..100644 --- a/moses/src/TypeDef.h +++ b/moses/src/TypeDef.h diff --git a/moses/src/UniqueObject.h b/moses/src/UniqueObject.h index 789c5e1b3..789c5e1b3 100755..100644 --- a/moses/src/UniqueObject.h +++ b/moses/src/UniqueObject.h diff --git a/moses/src/UserMessage.cpp b/moses/src/UserMessage.cpp index ad89d80a7..ad89d80a7 100755..100644 --- a/moses/src/UserMessage.cpp +++ b/moses/src/UserMessage.cpp diff --git a/moses/src/UserMessage.h b/moses/src/UserMessage.h index aac38cae4..aac38cae4 100755..100644 --- a/moses/src/UserMessage.h +++ b/moses/src/UserMessage.h diff --git a/moses/src/Util.cpp b/moses/src/Util.cpp index be02fb27b..be02fb27b 100755..100644 --- a/moses/src/Util.cpp +++ b/moses/src/Util.cpp diff --git a/moses/src/Util.h b/moses/src/Util.h index 1b0638ec2..1b0638ec2 100755..100644 --- a/moses/src/Util.h +++ b/moses/src/Util.h diff --git a/moses/src/Word.cpp b/moses/src/Word.cpp index 1ff5df496..1ff5df496 100755..100644 --- a/moses/src/Word.cpp +++ b/moses/src/Word.cpp diff --git a/moses/src/Word.h b/moses/src/Word.h index 7dd395030..7dd395030 100755..100644 --- a/moses/src/Word.h +++ b/moses/src/Word.h diff --git a/moses/src/WordLattice.cpp b/moses/src/WordLattice.cpp index f3344dc67..f3344dc67 100755..100644 --- a/moses/src/WordLattice.cpp +++ b/moses/src/WordLattice.cpp diff --git a/moses/src/WordLattice.h b/moses/src/WordLattice.h index 21df48054..21df48054 100755..100644 --- a/moses/src/WordLattice.h +++ b/moses/src/WordLattice.h diff --git a/moses/src/WordTranslationFeature.cpp b/moses/src/WordTranslationFeature.cpp index c0b3e0025..c0b3e0025 100755..100644 --- a/moses/src/WordTranslationFeature.cpp +++ b/moses/src/WordTranslationFeature.cpp diff --git a/moses/src/WordTranslationFeature.h b/moses/src/WordTranslationFeature.h index 39be24d41..39be24d41 100755..100644 --- a/moses/src/WordTranslationFeature.h +++ b/moses/src/WordTranslationFeature.h diff --git a/moses/src/WordsBitmap.cpp b/moses/src/WordsBitmap.cpp index b3203145f..b3203145f 100755..100644 --- a/moses/src/WordsBitmap.cpp +++ b/moses/src/WordsBitmap.cpp diff --git a/moses/src/WordsBitmap.h b/moses/src/WordsBitmap.h index 370120571..370120571 100755..100644 --- a/moses/src/WordsBitmap.h +++ b/moses/src/WordsBitmap.h diff --git a/moses/src/WordsRange.cpp b/moses/src/WordsRange.cpp index f37a738c0..f37a738c0 100755..100644 --- a/moses/src/WordsRange.cpp +++ b/moses/src/WordsRange.cpp diff --git a/moses/src/WordsRange.h b/moses/src/WordsRange.h index 7191d259e..7191d259e 100755..100644 --- a/moses/src/WordsRange.h +++ b/moses/src/WordsRange.h diff --git a/moses/src/XmlOption.cpp b/moses/src/XmlOption.cpp index f5eee83f9..f5eee83f9 100755..100644 --- a/moses/src/XmlOption.cpp +++ b/moses/src/XmlOption.cpp diff --git a/moses/src/XmlOption.h b/moses/src/XmlOption.h index 45989c841..45989c841 100755..100644 --- a/moses/src/XmlOption.h +++ b/moses/src/XmlOption.h diff --git a/moses/src/gzfilebuf.h b/moses/src/gzfilebuf.h index d59d3e1b7..d59d3e1b7 100755..100644 --- a/moses/src/gzfilebuf.h +++ b/moses/src/gzfilebuf.h diff --git a/moses/src/hash.cpp b/moses/src/hash.cpp index 9afac463d..9afac463d 100755..100644 --- a/moses/src/hash.cpp +++ b/moses/src/hash.cpp diff --git a/moses/src/hash.h b/moses/src/hash.h index 227f3b90a..227f3b90a 100755..100644 --- a/moses/src/hash.h +++ b/moses/src/hash.h diff --git a/moses/src/hypergraph.proto b/moses/src/hypergraph.proto index e292eacca..e292eacca 100755..100644 --- a/moses/src/hypergraph.proto +++ b/moses/src/hypergraph.proto diff --git a/moses/src/rule.proto b/moses/src/rule.proto index 5f8084636..5f8084636 100755..100644 --- a/moses/src/rule.proto +++ b/moses/src/rule.proto |