diff options
author | Eva Hasler <ehasler@saxnot.inf.ed.ac.uk> | 2012-06-01 04:49:42 +0400 |
---|---|---|
committer | Eva Hasler <ehasler@saxnot.inf.ed.ac.uk> | 2012-06-01 04:49:42 +0400 |
commit | 6a6a35c65eaeb42b3f1aa44237332dd6f1bc652c (patch) | |
tree | 8a02d798832fe36e9efb60f8a2e30d9368728a13 /moses | |
parent | 62d10a2af3f9509bf7680b8549ff1ffd9774dd83 (diff) |
fix start weights in experiment.perl, add hypothesis queue for picking hope and fear translations, add variations to 1slack formulation
Diffstat (limited to 'moses')
314 files changed, 179 insertions, 144 deletions
diff --git a/moses/src/AlignmentInfo.cpp b/moses/src/AlignmentInfo.cpp index 959e7591f..959e7591f 100755..100644 --- a/moses/src/AlignmentInfo.cpp +++ b/moses/src/AlignmentInfo.cpp diff --git a/moses/src/AlignmentInfo.h b/moses/src/AlignmentInfo.h index 12c5ee2e9..12c5ee2e9 100755..100644 --- a/moses/src/AlignmentInfo.h +++ b/moses/src/AlignmentInfo.h diff --git a/moses/src/AlignmentInfoCollection.cpp b/moses/src/AlignmentInfoCollection.cpp index aa411fc63..aa411fc63 100755..100644 --- a/moses/src/AlignmentInfoCollection.cpp +++ b/moses/src/AlignmentInfoCollection.cpp diff --git a/moses/src/AlignmentInfoCollection.h b/moses/src/AlignmentInfoCollection.h index a6c90f135..a6c90f135 100755..100644 --- a/moses/src/AlignmentInfoCollection.h +++ b/moses/src/AlignmentInfoCollection.h diff --git a/moses/src/BilingualDynSuffixArray.cpp b/moses/src/BilingualDynSuffixArray.cpp index a2d5e3e90..a2d5e3e90 100755..100644 --- a/moses/src/BilingualDynSuffixArray.cpp +++ b/moses/src/BilingualDynSuffixArray.cpp diff --git a/moses/src/BilingualDynSuffixArray.h b/moses/src/BilingualDynSuffixArray.h index 1543c3709..1543c3709 100755..100644 --- a/moses/src/BilingualDynSuffixArray.h +++ b/moses/src/BilingualDynSuffixArray.h diff --git a/moses/src/BitmapContainer.cpp b/moses/src/BitmapContainer.cpp index c80f3b542..c80f3b542 100755..100644 --- a/moses/src/BitmapContainer.cpp +++ b/moses/src/BitmapContainer.cpp diff --git a/moses/src/BitmapContainer.h b/moses/src/BitmapContainer.h index 987440750..987440750 100755..100644 --- a/moses/src/BitmapContainer.h +++ b/moses/src/BitmapContainer.h diff --git a/moses/src/BleuScoreFeature.cpp b/moses/src/BleuScoreFeature.cpp index a0f4df200..8ba979742 100755..100644 --- a/moses/src/BleuScoreFeature.cpp +++ b/moses/src/BleuScoreFeature.cpp @@ -22,11 +22,28 @@ int BleuScoreState::Compare(const FFState& o) const if (&o == this) return 0; + const StaticData &staticData = StaticData::Instance(); + SearchAlgorithm searchAlgorithm = staticData.GetSearchAlgorithm(); + bool chartDecoding = (searchAlgorithm == ChartDecoding); + if (chartDecoding) + return 0; + const BleuScoreState& other = dynamic_cast<const BleuScoreState&>(o); int c = m_words.Compare(other.m_words); if (c != 0) return c; + /*for(size_t i = 0; i < m_ngram_counts.size(); i++) { + if (m_ngram_counts[i] < other.m_ngram_counts[i]) + return -1; + if (m_ngram_counts[i] > other.m_ngram_counts[i]) + return 1; + if (m_ngram_matches[i] < other.m_ngram_matches[i]) + return -1; + if (m_ngram_matches[i] > other.m_ngram_matches[i]) + return 1; + }*/ + return 0; } std::ostream& operator<<(std::ostream& out, const BleuScoreState& state) { diff --git a/moses/src/BleuScoreFeature.h b/moses/src/BleuScoreFeature.h index fc7aa2e39..5bae3a1fb 100755..100644 --- a/moses/src/BleuScoreFeature.h +++ b/moses/src/BleuScoreFeature.h @@ -123,7 +123,7 @@ public: FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection* accumulator) const; - + bool Enabled() const { return m_enabled; } float CalculateBleu(BleuScoreState*) const; float CalculateBleu(Phrase translation) const; const FFState* EmptyHypothesisState(const InputType&) const; diff --git a/moses/src/CellCollection.h b/moses/src/CellCollection.h index 5b99fe0fe..5b99fe0fe 100755..100644 --- a/moses/src/CellCollection.h +++ b/moses/src/CellCollection.h diff --git a/moses/src/ChartCell.cpp b/moses/src/ChartCell.cpp index 21c5dd68a..21c5dd68a 100755..100644 --- a/moses/src/ChartCell.cpp +++ b/moses/src/ChartCell.cpp diff --git a/moses/src/ChartCell.h b/moses/src/ChartCell.h index ab9e5b1ea..ab9e5b1ea 100755..100644 --- a/moses/src/ChartCell.h +++ b/moses/src/ChartCell.h diff --git a/moses/src/ChartCellCollection.cpp b/moses/src/ChartCellCollection.cpp index 894bb352c..894bb352c 100755..100644 --- a/moses/src/ChartCellCollection.cpp +++ b/moses/src/ChartCellCollection.cpp diff --git a/moses/src/ChartCellCollection.h b/moses/src/ChartCellCollection.h index 6627b6b67..6627b6b67 100755..100644 --- a/moses/src/ChartCellCollection.h +++ b/moses/src/ChartCellCollection.h diff --git a/moses/src/ChartCellLabel.h b/moses/src/ChartCellLabel.h index 4feeac322..4feeac322 100755..100644 --- a/moses/src/ChartCellLabel.h +++ b/moses/src/ChartCellLabel.h diff --git a/moses/src/ChartCellLabelSet.h b/moses/src/ChartCellLabelSet.h index 2a359758d..2a359758d 100755..100644 --- a/moses/src/ChartCellLabelSet.h +++ b/moses/src/ChartCellLabelSet.h diff --git a/moses/src/ChartHypothesis.cpp b/moses/src/ChartHypothesis.cpp index 88b930243..88b930243 100755..100644 --- a/moses/src/ChartHypothesis.cpp +++ b/moses/src/ChartHypothesis.cpp diff --git a/moses/src/ChartHypothesis.h b/moses/src/ChartHypothesis.h index aa663a140..aa663a140 100755..100644 --- a/moses/src/ChartHypothesis.h +++ b/moses/src/ChartHypothesis.h diff --git a/moses/src/ChartHypothesisCollection.cpp b/moses/src/ChartHypothesisCollection.cpp index 48a108aab..48a108aab 100755..100644 --- a/moses/src/ChartHypothesisCollection.cpp +++ b/moses/src/ChartHypothesisCollection.cpp diff --git a/moses/src/ChartHypothesisCollection.h b/moses/src/ChartHypothesisCollection.h index ebf1c6002..ebf1c6002 100755..100644 --- a/moses/src/ChartHypothesisCollection.h +++ b/moses/src/ChartHypothesisCollection.h diff --git a/moses/src/ChartManager.cpp b/moses/src/ChartManager.cpp index ad8f0dc84..77b703007 100755..100644 --- a/moses/src/ChartManager.cpp +++ b/moses/src/ChartManager.cpp @@ -185,7 +185,8 @@ void ChartManager::CalcNBest(size_t count, ChartTrellisPathList &ret, bool onlyD // Get all complete translations Word *w = new Word(); - w->CreateFromString(Output, staticData.GetOutputFactorOrder(), "TOP", true); + //w->CreateFromString(Output, staticData.GetOutputFactorOrder(), "TOP", true); + w->CreateFromString(Output, staticData.GetOutputFactorOrder(), "S", true); const HypoList topHypos = lastCell.GetSortedHypotheses(*w); // Create a ChartTrellisDetour for each complete translation and add it to the queue diff --git a/moses/src/ChartManager.h b/moses/src/ChartManager.h index dbcd42b6c..dbcd42b6c 100755..100644 --- a/moses/src/ChartManager.h +++ b/moses/src/ChartManager.h diff --git a/moses/src/ChartRuleLookupManager.cpp b/moses/src/ChartRuleLookupManager.cpp index 63e941429..63e941429 100755..100644 --- a/moses/src/ChartRuleLookupManager.cpp +++ b/moses/src/ChartRuleLookupManager.cpp diff --git a/moses/src/ChartRuleLookupManager.h b/moses/src/ChartRuleLookupManager.h index 2166f28f5..2166f28f5 100755..100644 --- a/moses/src/ChartRuleLookupManager.h +++ b/moses/src/ChartRuleLookupManager.h diff --git a/moses/src/ChartRuleLookupManagerMemory.cpp b/moses/src/ChartRuleLookupManagerMemory.cpp index aab3dfcce..aab3dfcce 100755..100644 --- a/moses/src/ChartRuleLookupManagerMemory.cpp +++ b/moses/src/ChartRuleLookupManagerMemory.cpp diff --git a/moses/src/ChartRuleLookupManagerMemory.h b/moses/src/ChartRuleLookupManagerMemory.h index a9273499c..a9273499c 100755..100644 --- a/moses/src/ChartRuleLookupManagerMemory.h +++ b/moses/src/ChartRuleLookupManagerMemory.h diff --git a/moses/src/ChartRuleLookupManagerOnDisk.cpp b/moses/src/ChartRuleLookupManagerOnDisk.cpp index 17e57f73c..010973900 100755..100644 --- a/moses/src/ChartRuleLookupManagerOnDisk.cpp +++ b/moses/src/ChartRuleLookupManagerOnDisk.cpp @@ -41,7 +41,6 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk( const WordPenaltyProducer *wpProducer, const std::vector<FactorType> &inputFactorsVec, const std::vector<FactorType> &outputFactorsVec, - const std::vector<float> &weight, const std::string &filePath) : ChartRuleLookupManager(sentence, cellColl) , m_dictionary(dictionary) @@ -50,7 +49,6 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk( , m_wpProducer(wpProducer) , m_inputFactorsVec(inputFactorsVec) , m_outputFactorsVec(outputFactorsVec) - , m_weight(weight) , m_filePath(filePath) { CHECK(m_expandableDottedRuleListVec.size() == 0); @@ -241,12 +239,11 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(m_dictionary.GetTableLimit(), m_dbWrapper); - std::vector<float> weightT = staticData.GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(); + std::vector<float> weightT = staticData.GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(m_dictionary.GetDictIndex()); targetPhraseCollection = tpcollBerkeleyDb->ConvertToMoses(m_inputFactorsVec ,m_outputFactorsVec ,m_dictionary - //, m_weight ,weightT ,m_wpProducer ,*m_languageModels diff --git a/moses/src/ChartRuleLookupManagerOnDisk.h b/moses/src/ChartRuleLookupManagerOnDisk.h index ba87dbbf8..31f44bdfe 100755..100644 --- a/moses/src/ChartRuleLookupManagerOnDisk.h +++ b/moses/src/ChartRuleLookupManagerOnDisk.h @@ -44,7 +44,6 @@ public: const WordPenaltyProducer *wpProducer, const std::vector<FactorType> &inputFactorsVec, const std::vector<FactorType> &outputFactorsVec, - const std::vector<float> &weight, const std::string &filePath); ~ChartRuleLookupManagerOnDisk(); @@ -60,7 +59,6 @@ private: const WordPenaltyProducer *m_wpProducer; const std::vector<FactorType> &m_inputFactorsVec; const std::vector<FactorType> &m_outputFactorsVec; - const std::vector<float> &m_weight; const std::string &m_filePath; std::vector<DottedRuleStackOnDisk*> m_expandableDottedRuleListVec; std::map<UINT64, const TargetPhraseCollection*> m_cache; diff --git a/moses/src/ChartTranslationOption.cpp b/moses/src/ChartTranslationOption.cpp index 6aa04d189..6aa04d189 100755..100644 --- a/moses/src/ChartTranslationOption.cpp +++ b/moses/src/ChartTranslationOption.cpp diff --git a/moses/src/ChartTranslationOption.h b/moses/src/ChartTranslationOption.h index 116eff037..116eff037 100755..100644 --- a/moses/src/ChartTranslationOption.h +++ b/moses/src/ChartTranslationOption.h diff --git a/moses/src/ChartTranslationOptionCollection.cpp b/moses/src/ChartTranslationOptionCollection.cpp index 8f06656b2..8f06656b2 100755..100644 --- a/moses/src/ChartTranslationOptionCollection.cpp +++ b/moses/src/ChartTranslationOptionCollection.cpp diff --git a/moses/src/ChartTranslationOptionCollection.h b/moses/src/ChartTranslationOptionCollection.h index e1f3490a9..e1f3490a9 100755..100644 --- a/moses/src/ChartTranslationOptionCollection.h +++ b/moses/src/ChartTranslationOptionCollection.h diff --git a/moses/src/ChartTranslationOptionList.cpp b/moses/src/ChartTranslationOptionList.cpp index cb1b8e256..cb1b8e256 100755..100644 --- a/moses/src/ChartTranslationOptionList.cpp +++ b/moses/src/ChartTranslationOptionList.cpp diff --git a/moses/src/ChartTranslationOptionList.h b/moses/src/ChartTranslationOptionList.h index 7272a01cf..7272a01cf 100755..100644 --- a/moses/src/ChartTranslationOptionList.h +++ b/moses/src/ChartTranslationOptionList.h diff --git a/moses/src/ChartTrellisDetour.cpp b/moses/src/ChartTrellisDetour.cpp index 550a44a2c..550a44a2c 100755..100644 --- a/moses/src/ChartTrellisDetour.cpp +++ b/moses/src/ChartTrellisDetour.cpp diff --git a/moses/src/ChartTrellisDetour.h b/moses/src/ChartTrellisDetour.h index a3b07ad00..a3b07ad00 100755..100644 --- a/moses/src/ChartTrellisDetour.h +++ b/moses/src/ChartTrellisDetour.h diff --git a/moses/src/ChartTrellisDetourQueue.cpp b/moses/src/ChartTrellisDetourQueue.cpp index 9b359ca43..9b359ca43 100755..100644 --- a/moses/src/ChartTrellisDetourQueue.cpp +++ b/moses/src/ChartTrellisDetourQueue.cpp diff --git a/moses/src/ChartTrellisDetourQueue.h b/moses/src/ChartTrellisDetourQueue.h index f679708e4..f679708e4 100755..100644 --- a/moses/src/ChartTrellisDetourQueue.h +++ b/moses/src/ChartTrellisDetourQueue.h diff --git a/moses/src/ChartTrellisNode.cpp b/moses/src/ChartTrellisNode.cpp index ce3c9eaf7..ce3c9eaf7 100755..100644 --- a/moses/src/ChartTrellisNode.cpp +++ b/moses/src/ChartTrellisNode.cpp diff --git a/moses/src/ChartTrellisNode.h b/moses/src/ChartTrellisNode.h index 7b81ff4b2..7b81ff4b2 100755..100644 --- a/moses/src/ChartTrellisNode.h +++ b/moses/src/ChartTrellisNode.h diff --git a/moses/src/ChartTrellisPath.cpp b/moses/src/ChartTrellisPath.cpp index 231d4237a..231d4237a 100755..100644 --- a/moses/src/ChartTrellisPath.cpp +++ b/moses/src/ChartTrellisPath.cpp diff --git a/moses/src/ChartTrellisPath.h b/moses/src/ChartTrellisPath.h index 4dee018c3..4dee018c3 100755..100644 --- a/moses/src/ChartTrellisPath.h +++ b/moses/src/ChartTrellisPath.h diff --git a/moses/src/ChartTrellisPathList.h b/moses/src/ChartTrellisPathList.h index 5b811d621..5b811d621 100755..100644 --- a/moses/src/ChartTrellisPathList.h +++ b/moses/src/ChartTrellisPathList.h diff --git a/moses/src/ConfusionNet.cpp b/moses/src/ConfusionNet.cpp index 8ff119497..8ff119497 100755..100644 --- a/moses/src/ConfusionNet.cpp +++ b/moses/src/ConfusionNet.cpp diff --git a/moses/src/ConfusionNet.h b/moses/src/ConfusionNet.h index 25b5a021d..25b5a021d 100755..100644 --- a/moses/src/ConfusionNet.h +++ b/moses/src/ConfusionNet.h diff --git a/moses/src/DecodeFeature.cpp b/moses/src/DecodeFeature.cpp index 67999384a..67999384a 100755..100644 --- a/moses/src/DecodeFeature.cpp +++ b/moses/src/DecodeFeature.cpp diff --git a/moses/src/DecodeFeature.h b/moses/src/DecodeFeature.h index afa1771ba..afa1771ba 100755..100644 --- a/moses/src/DecodeFeature.h +++ b/moses/src/DecodeFeature.h diff --git a/moses/src/DecodeGraph.cpp b/moses/src/DecodeGraph.cpp index 107258d56..107258d56 100755..100644 --- a/moses/src/DecodeGraph.cpp +++ b/moses/src/DecodeGraph.cpp diff --git a/moses/src/DecodeGraph.h b/moses/src/DecodeGraph.h index 770cb9958..770cb9958 100755..100644 --- a/moses/src/DecodeGraph.h +++ b/moses/src/DecodeGraph.h diff --git a/moses/src/DecodeStep.cpp b/moses/src/DecodeStep.cpp index 3345f95d0..3345f95d0 100755..100644 --- a/moses/src/DecodeStep.cpp +++ b/moses/src/DecodeStep.cpp diff --git a/moses/src/DecodeStep.h b/moses/src/DecodeStep.h index 4d2f2280b..4d2f2280b 100755..100644 --- a/moses/src/DecodeStep.h +++ b/moses/src/DecodeStep.h diff --git a/moses/src/DecodeStepGeneration.cpp b/moses/src/DecodeStepGeneration.cpp index 0f0c5c2dc..0f0c5c2dc 100755..100644 --- a/moses/src/DecodeStepGeneration.cpp +++ b/moses/src/DecodeStepGeneration.cpp diff --git a/moses/src/DecodeStepGeneration.h b/moses/src/DecodeStepGeneration.h index d22534ccc..d22534ccc 100755..100644 --- a/moses/src/DecodeStepGeneration.h +++ b/moses/src/DecodeStepGeneration.h diff --git a/moses/src/DecodeStepTranslation.cpp b/moses/src/DecodeStepTranslation.cpp index 03713b50c..03713b50c 100755..100644 --- a/moses/src/DecodeStepTranslation.cpp +++ b/moses/src/DecodeStepTranslation.cpp diff --git a/moses/src/DecodeStepTranslation.h b/moses/src/DecodeStepTranslation.h index 651754b1c..651754b1c 100755..100644 --- a/moses/src/DecodeStepTranslation.h +++ b/moses/src/DecodeStepTranslation.h diff --git a/moses/src/Dictionary.cpp b/moses/src/Dictionary.cpp index 05b4f6bb4..05b4f6bb4 100755..100644 --- a/moses/src/Dictionary.cpp +++ b/moses/src/Dictionary.cpp diff --git a/moses/src/Dictionary.h b/moses/src/Dictionary.h index 7bb4d09b1..7bb4d09b1 100755..100644 --- a/moses/src/Dictionary.h +++ b/moses/src/Dictionary.h diff --git a/moses/src/DotChart.cpp b/moses/src/DotChart.cpp index c7ba3f43e..c7ba3f43e 100755..100644 --- a/moses/src/DotChart.cpp +++ b/moses/src/DotChart.cpp diff --git a/moses/src/DotChart.h b/moses/src/DotChart.h index b9da12afd..b9da12afd 100755..100644 --- a/moses/src/DotChart.h +++ b/moses/src/DotChart.h diff --git a/moses/src/DotChartInMemory.cpp b/moses/src/DotChartInMemory.cpp index 095e3418f..095e3418f 100755..100644 --- a/moses/src/DotChartInMemory.cpp +++ b/moses/src/DotChartInMemory.cpp diff --git a/moses/src/DotChartInMemory.h b/moses/src/DotChartInMemory.h index fe7ab64c5..fe7ab64c5 100755..100644 --- a/moses/src/DotChartInMemory.h +++ b/moses/src/DotChartInMemory.h diff --git a/moses/src/DotChartOnDisk.cpp b/moses/src/DotChartOnDisk.cpp index 1674f0e21..1674f0e21 100755..100644 --- a/moses/src/DotChartOnDisk.cpp +++ b/moses/src/DotChartOnDisk.cpp diff --git a/moses/src/DotChartOnDisk.h b/moses/src/DotChartOnDisk.h index 4473b7685..4473b7685 100755..100644 --- a/moses/src/DotChartOnDisk.h +++ b/moses/src/DotChartOnDisk.h diff --git a/moses/src/DummyScoreProducers.cpp b/moses/src/DummyScoreProducers.cpp index 03c635b03..03c635b03 100755..100644 --- a/moses/src/DummyScoreProducers.cpp +++ b/moses/src/DummyScoreProducers.cpp diff --git a/moses/src/DummyScoreProducers.h b/moses/src/DummyScoreProducers.h index d8d9393d2..d8d9393d2 100755..100644 --- a/moses/src/DummyScoreProducers.h +++ b/moses/src/DummyScoreProducers.h diff --git a/moses/src/DynSAInclude/RandLMCache.h b/moses/src/DynSAInclude/RandLMCache.h index 3f38cae02..3f38cae02 100755..100644 --- a/moses/src/DynSAInclude/RandLMCache.h +++ b/moses/src/DynSAInclude/RandLMCache.h diff --git a/moses/src/DynSAInclude/RandLMFilter.h b/moses/src/DynSAInclude/RandLMFilter.h index 556bbe44a..556bbe44a 100755..100644 --- a/moses/src/DynSAInclude/RandLMFilter.h +++ b/moses/src/DynSAInclude/RandLMFilter.h diff --git a/moses/src/DynSAInclude/fdstream.h b/moses/src/DynSAInclude/fdstream.h index 4c95d032c..4c95d032c 100755..100644 --- a/moses/src/DynSAInclude/fdstream.h +++ b/moses/src/DynSAInclude/fdstream.h diff --git a/moses/src/DynSAInclude/file.cpp b/moses/src/DynSAInclude/file.cpp index d2901c1ae..d2901c1ae 100755..100644 --- a/moses/src/DynSAInclude/file.cpp +++ b/moses/src/DynSAInclude/file.cpp diff --git a/moses/src/DynSAInclude/file.h b/moses/src/DynSAInclude/file.h index 3157f918b..3157f918b 100755..100644 --- a/moses/src/DynSAInclude/file.h +++ b/moses/src/DynSAInclude/file.h diff --git a/moses/src/DynSAInclude/hash.h b/moses/src/DynSAInclude/hash.h index 233d0be5d..233d0be5d 100755..100644 --- a/moses/src/DynSAInclude/hash.h +++ b/moses/src/DynSAInclude/hash.h diff --git a/moses/src/DynSAInclude/onlineRLM.h b/moses/src/DynSAInclude/onlineRLM.h index adc7934fc..adc7934fc 100755..100644 --- a/moses/src/DynSAInclude/onlineRLM.h +++ b/moses/src/DynSAInclude/onlineRLM.h diff --git a/moses/src/DynSAInclude/params.cpp b/moses/src/DynSAInclude/params.cpp index 4be3a1676..4be3a1676 100755..100644 --- a/moses/src/DynSAInclude/params.cpp +++ b/moses/src/DynSAInclude/params.cpp diff --git a/moses/src/DynSAInclude/params.h b/moses/src/DynSAInclude/params.h index e0aab6135..e0aab6135 100755..100644 --- a/moses/src/DynSAInclude/params.h +++ b/moses/src/DynSAInclude/params.h diff --git a/moses/src/DynSAInclude/perfectHash.h b/moses/src/DynSAInclude/perfectHash.h index 2e3bfbc8f..2e3bfbc8f 100755..100644 --- a/moses/src/DynSAInclude/perfectHash.h +++ b/moses/src/DynSAInclude/perfectHash.h diff --git a/moses/src/DynSAInclude/quantizer.h b/moses/src/DynSAInclude/quantizer.h index c12189615..c12189615 100755..100644 --- a/moses/src/DynSAInclude/quantizer.h +++ b/moses/src/DynSAInclude/quantizer.h diff --git a/moses/src/DynSAInclude/types.h b/moses/src/DynSAInclude/types.h index 444010889..444010889 100755..100644 --- a/moses/src/DynSAInclude/types.h +++ b/moses/src/DynSAInclude/types.h diff --git a/moses/src/DynSAInclude/utils.h b/moses/src/DynSAInclude/utils.h index 74703863d..74703863d 100755..100644 --- a/moses/src/DynSAInclude/utils.h +++ b/moses/src/DynSAInclude/utils.h diff --git a/moses/src/DynSAInclude/vocab.cpp b/moses/src/DynSAInclude/vocab.cpp index 27e052260..27e052260 100755..100644 --- a/moses/src/DynSAInclude/vocab.cpp +++ b/moses/src/DynSAInclude/vocab.cpp diff --git a/moses/src/DynSAInclude/vocab.h b/moses/src/DynSAInclude/vocab.h index 467d16fdb..467d16fdb 100755..100644 --- a/moses/src/DynSAInclude/vocab.h +++ b/moses/src/DynSAInclude/vocab.h diff --git a/moses/src/DynSuffixArray.cpp b/moses/src/DynSuffixArray.cpp index f043f5c9b..f043f5c9b 100755..100644 --- a/moses/src/DynSuffixArray.cpp +++ b/moses/src/DynSuffixArray.cpp diff --git a/moses/src/DynSuffixArray.h b/moses/src/DynSuffixArray.h index 05ea0596a..05ea0596a 100755..100644 --- a/moses/src/DynSuffixArray.h +++ b/moses/src/DynSuffixArray.h diff --git a/moses/src/FFState.cpp b/moses/src/FFState.cpp index 5bf0992a9..5bf0992a9 100755..100644 --- a/moses/src/FFState.cpp +++ b/moses/src/FFState.cpp diff --git a/moses/src/FFState.h b/moses/src/FFState.h index 49b0e55a8..49b0e55a8 100755..100644 --- a/moses/src/FFState.h +++ b/moses/src/FFState.h diff --git a/moses/src/Factor.cpp b/moses/src/Factor.cpp index 60a93fdc3..60a93fdc3 100755..100644 --- a/moses/src/Factor.cpp +++ b/moses/src/Factor.cpp diff --git a/moses/src/Factor.h b/moses/src/Factor.h index 4ea44f4e1..4ea44f4e1 100755..100644 --- a/moses/src/Factor.h +++ b/moses/src/Factor.h diff --git a/moses/src/FactorCollection.cpp b/moses/src/FactorCollection.cpp index 2ef55b6d1..2ef55b6d1 100755..100644 --- a/moses/src/FactorCollection.cpp +++ b/moses/src/FactorCollection.cpp diff --git a/moses/src/FactorCollection.h b/moses/src/FactorCollection.h index d55be1b17..d55be1b17 100755..100644 --- a/moses/src/FactorCollection.h +++ b/moses/src/FactorCollection.h diff --git a/moses/src/FactorTypeSet.cpp b/moses/src/FactorTypeSet.cpp index de2166ba2..de2166ba2 100755..100644 --- a/moses/src/FactorTypeSet.cpp +++ b/moses/src/FactorTypeSet.cpp diff --git a/moses/src/FactorTypeSet.h b/moses/src/FactorTypeSet.h index b0db0502c..b0db0502c 100755..100644 --- a/moses/src/FactorTypeSet.h +++ b/moses/src/FactorTypeSet.h diff --git a/moses/src/FeatureFunction.cpp b/moses/src/FeatureFunction.cpp index d435df5af..d435df5af 100755..100644 --- a/moses/src/FeatureFunction.cpp +++ b/moses/src/FeatureFunction.cpp diff --git a/moses/src/FeatureFunction.h b/moses/src/FeatureFunction.h index 4ef85ecd9..4ef85ecd9 100755..100644 --- a/moses/src/FeatureFunction.h +++ b/moses/src/FeatureFunction.h diff --git a/moses/src/FeatureVector.cpp b/moses/src/FeatureVector.cpp index 911c29600..ac3deb876 100755..100644 --- a/moses/src/FeatureVector.cpp +++ b/moses/src/FeatureVector.cpp @@ -156,29 +156,29 @@ namespace Moses { } } - void FVector::clear() { - m_coreFeatures.resize(0); - m_features.clear(); - } + void FVector::clear() { + m_coreFeatures.resize(0); + m_features.clear(); + } - bool FVector::load(const std::string& filename) { + bool FVector::load(const std::string& filename) { clear(); - ifstream in (filename.c_str()); - if (!in) { + ifstream in (filename.c_str()); + if (!in) { return false; - } - string line; - while(getline(in,line)) { - if (line[0] == '#') continue; - istringstream linestream(line); - string namestring; - FValue value; - linestream >> namestring; - linestream >> value; - FName fname(namestring); - cerr << "Setting sparse weight " << fname << " to value " << value << "." << endl; - set(fname,value); - } + } + string line; + while(getline(in,line)) { + if (line[0] == '#') continue; + istringstream linestream(line); + string namestring; + FValue value; + linestream >> namestring; + linestream >> value; + FName fname(namestring); + //cerr << "Setting sparse weight " << fname << " to value " << value << "." << endl; + set(fname,value); + } return true; } @@ -416,34 +416,38 @@ namespace Moses { void FVector::updateConfidenceCounts(const FVector& weightUpdate, bool signedCounts) { for (size_t i = 0; i < weightUpdate.m_coreFeatures.size(); ++i) { if (signedCounts) { - int sign = weightUpdate.m_coreFeatures[i] >= 0 ? 1 : -1; - m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]) * sign; + //int sign = weightUpdate.m_coreFeatures[i] >= 0 ? 1 : -1; + //m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]) * sign; + m_coreFeatures[i] += weightUpdate.m_coreFeatures[i]; } else - m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]); + //m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]); + m_coreFeatures[i] += abs(weightUpdate.m_coreFeatures[i]); } for (const_iterator i = weightUpdate.cbegin(); i != weightUpdate.cend(); ++i) { if (weightUpdate[i->first] == 0) continue; - float value = get(i->first); + float value = get(i->first); if (signedCounts) { - int sign = weightUpdate[i->first] >= 0 ? 1 : -1; - value += (weightUpdate[i->first] * weightUpdate[i->first]) * sign; + //int sign = weightUpdate[i->first] >= 0 ? 1 : -1; + //value += (weightUpdate[i->first] * weightUpdate[i->first]) * sign; + value += weightUpdate[i->first]; } else - value += (weightUpdate[i->first] * weightUpdate[i->first]); + //value += (weightUpdate[i->first] * weightUpdate[i->first]); + value += abs(weightUpdate[i->first]); set(i->first, value); } } - void FVector::updateLearningRates(float decay, const FVector &confidenceCounts, float core_r0, float sparse_r0) { + void FVector::updateLearningRates(float decay_core, float decay_sparse, const FVector &confidenceCounts, float core_r0, float sparse_r0) { for (size_t i = 0; i < confidenceCounts.m_coreFeatures.size(); ++i) { - m_coreFeatures[i] = 1.0/(1.0/core_r0 + decay * abs(confidenceCounts.m_coreFeatures[i])); + m_coreFeatures[i] = 1.0/(1.0/core_r0 + decay_core * abs(confidenceCounts.m_coreFeatures[i])); } for (const_iterator i = confidenceCounts.cbegin(); i != confidenceCounts.cend(); ++i) { - float value = 1.0/(1.0/sparse_r0 + decay * abs(i->second)); + float value = 1.0/(1.0/sparse_r0 + decay_sparse * abs(i->second)); set(i->first, value); } } @@ -612,8 +616,8 @@ namespace Moses { return norm; } - void FVector::l1regularize(float lambda) { - for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + size_t FVector::sparseL1regularize(float lambda) { + /*for (size_t i = 0; i < m_coreFeatures.size(); ++i) { float value = m_coreFeatures[i]; if (value > 0) { m_coreFeatures[i] = max(0.0f, value - lambda); @@ -621,23 +625,39 @@ namespace Moses { else { m_coreFeatures[i] = min(0.0f, value + lambda); } - } + }*/ + size_t numberPruned = size(); + vector<FName> toErase; for (iterator i = begin(); i != end(); ++i) { float value = i->second; - if (value > 0) { - i->second = max(0.0f, value - lambda); - } - else { - i->second = min(0.0f, value + lambda); + if (value != 0.0f) { + if (value > 0) + value = max(0.0f, value - lambda); + else + value = min(0.0f, value + lambda); + + if (value != 0.0f) + i->second = value; + else { + toErase.push_back(i->first); + const std::string& fname = (i->first).name(); + FName::eraseId(FName::getId(fname)); + } } } + + // erase features that have become zero + for (size_t i = 0; i < toErase.size(); ++i) + m_features.erase(toErase[i]); + numberPruned -= size(); + return numberPruned; } - void FVector::l2regularize(float lambda) { - for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + void FVector::sparseL2regularize(float lambda) { + /*for (size_t i = 0; i < m_coreFeatures.size(); ++i) { m_coreFeatures[i] *= (1 - lambda); - } + }*/ for (iterator i = begin(); i != end(); ++i) { i->second *= (1 - lambda); diff --git a/moses/src/FeatureVector.h b/moses/src/FeatureVector.h index a86b30001..5dbfa01fe 100755..100644 --- a/moses/src/FeatureVector.h +++ b/moses/src/FeatureVector.h @@ -192,8 +192,8 @@ namespace Moses { FValue l1norm_coreFeatures() const; FValue l2norm() const; FValue linfnorm() const; - void l1regularize(float lambda); - void l2regularize(float lambda); + size_t sparseL1regularize(float lambda); + void sparseL2regularize(float lambda); FValue sum() const; /** pretty printing */ @@ -218,7 +218,7 @@ namespace Moses { size_t pruneSparseFeatures(size_t threshold); size_t pruneZeroWeightFeatures(); void updateConfidenceCounts(const FVector& weightUpdate, bool signedCounts); - void updateLearningRates(float decay, const FVector& confidence_counts, float core_r0, float sparse_r0); + void updateLearningRates(float decay_core, float decay_sparse, const FVector& confidence_counts, float core_r0, float sparse_r0); // vector which, for each element of the original vector, reflects whether an element is zero or non-zero void setToBinaryOf(const FVector& rhs); diff --git a/moses/src/FeatureVectorTest.cpp b/moses/src/FeatureVectorTest.cpp index af1829e62..af1829e62 100755..100644 --- a/moses/src/FeatureVectorTest.cpp +++ b/moses/src/FeatureVectorTest.cpp diff --git a/moses/src/File.cpp b/moses/src/File.cpp index 3d15b071c..3d15b071c 100755..100644 --- a/moses/src/File.cpp +++ b/moses/src/File.cpp diff --git a/moses/src/File.h b/moses/src/File.h index 09d9ddc92..09d9ddc92 100755..100644 --- a/moses/src/File.h +++ b/moses/src/File.h diff --git a/moses/src/FilePtr.h b/moses/src/FilePtr.h index 1613b2f2f..1613b2f2f 100755..100644 --- a/moses/src/FilePtr.h +++ b/moses/src/FilePtr.h diff --git a/moses/src/FloydWarshall.cpp b/moses/src/FloydWarshall.cpp index e63de819b..e63de819b 100755..100644 --- a/moses/src/FloydWarshall.cpp +++ b/moses/src/FloydWarshall.cpp diff --git a/moses/src/FloydWarshall.h b/moses/src/FloydWarshall.h index 96845caf6..96845caf6 100755..100644 --- a/moses/src/FloydWarshall.h +++ b/moses/src/FloydWarshall.h diff --git a/moses/src/GenerationDictionary.cpp b/moses/src/GenerationDictionary.cpp index f7b35fb63..f7b35fb63 100755..100644 --- a/moses/src/GenerationDictionary.cpp +++ b/moses/src/GenerationDictionary.cpp diff --git a/moses/src/GenerationDictionary.h b/moses/src/GenerationDictionary.h index 1c3f75f75..1c3f75f75 100755..100644 --- a/moses/src/GenerationDictionary.h +++ b/moses/src/GenerationDictionary.h diff --git a/moses/src/GlobalLexicalModel.cpp b/moses/src/GlobalLexicalModel.cpp index 65d3afaa1..65d3afaa1 100755..100644 --- a/moses/src/GlobalLexicalModel.cpp +++ b/moses/src/GlobalLexicalModel.cpp diff --git a/moses/src/GlobalLexicalModel.h b/moses/src/GlobalLexicalModel.h index 3105cb466..3105cb466 100755..100644 --- a/moses/src/GlobalLexicalModel.h +++ b/moses/src/GlobalLexicalModel.h diff --git a/moses/src/GlobalLexicalModelUnlimited.cpp b/moses/src/GlobalLexicalModelUnlimited.cpp index f1de65bd0..f1de65bd0 100755..100644 --- a/moses/src/GlobalLexicalModelUnlimited.cpp +++ b/moses/src/GlobalLexicalModelUnlimited.cpp diff --git a/moses/src/GlobalLexicalModelUnlimited.h b/moses/src/GlobalLexicalModelUnlimited.h index c05c11be8..c05c11be8 100755..100644 --- a/moses/src/GlobalLexicalModelUnlimited.h +++ b/moses/src/GlobalLexicalModelUnlimited.h diff --git a/moses/src/Hypothesis.cpp b/moses/src/Hypothesis.cpp index 72105ec7b..72105ec7b 100755..100644 --- a/moses/src/Hypothesis.cpp +++ b/moses/src/Hypothesis.cpp diff --git a/moses/src/Hypothesis.h b/moses/src/Hypothesis.h index 23980326e..23980326e 100755..100644 --- a/moses/src/Hypothesis.h +++ b/moses/src/Hypothesis.h diff --git a/moses/src/HypothesisStack.cpp b/moses/src/HypothesisStack.cpp index dfa03108e..dfa03108e 100755..100644 --- a/moses/src/HypothesisStack.cpp +++ b/moses/src/HypothesisStack.cpp diff --git a/moses/src/HypothesisStack.h b/moses/src/HypothesisStack.h index 784cd7a39..784cd7a39 100755..100644 --- a/moses/src/HypothesisStack.h +++ b/moses/src/HypothesisStack.h diff --git a/moses/src/HypothesisStackCubePruning.cpp b/moses/src/HypothesisStackCubePruning.cpp index ca54bf944..ca54bf944 100755..100644 --- a/moses/src/HypothesisStackCubePruning.cpp +++ b/moses/src/HypothesisStackCubePruning.cpp diff --git a/moses/src/HypothesisStackCubePruning.h b/moses/src/HypothesisStackCubePruning.h index 53cad04a1..53cad04a1 100755..100644 --- a/moses/src/HypothesisStackCubePruning.h +++ b/moses/src/HypothesisStackCubePruning.h diff --git a/moses/src/HypothesisStackNormal.cpp b/moses/src/HypothesisStackNormal.cpp index 7d72f76aa..7d72f76aa 100755..100644 --- a/moses/src/HypothesisStackNormal.cpp +++ b/moses/src/HypothesisStackNormal.cpp diff --git a/moses/src/HypothesisStackNormal.h b/moses/src/HypothesisStackNormal.h index 6996f4003..6996f4003 100755..100644 --- a/moses/src/HypothesisStackNormal.h +++ b/moses/src/HypothesisStackNormal.h diff --git a/moses/src/InputFileStream.cpp b/moses/src/InputFileStream.cpp index d2d774bf1..d2d774bf1 100755..100644 --- a/moses/src/InputFileStream.cpp +++ b/moses/src/InputFileStream.cpp diff --git a/moses/src/InputFileStream.h b/moses/src/InputFileStream.h index d53abfc23..d53abfc23 100755..100644 --- a/moses/src/InputFileStream.h +++ b/moses/src/InputFileStream.h diff --git a/moses/src/InputType.cpp b/moses/src/InputType.cpp index 77857bdaf..77857bdaf 100755..100644 --- a/moses/src/InputType.cpp +++ b/moses/src/InputType.cpp diff --git a/moses/src/InputType.h b/moses/src/InputType.h index ca710c408..ca710c408 100755..100644 --- a/moses/src/InputType.h +++ b/moses/src/InputType.h diff --git a/moses/src/Jamfile b/moses/src/Jamfile index b6e379070..b6e379070 100755..100644 --- a/moses/src/Jamfile +++ b/moses/src/Jamfile diff --git a/moses/src/LM/Base.cpp b/moses/src/LM/Base.cpp index 45edf973e..45edf973e 100755..100644 --- a/moses/src/LM/Base.cpp +++ b/moses/src/LM/Base.cpp diff --git a/moses/src/LM/Base.h b/moses/src/LM/Base.h index 621ab0dc3..621ab0dc3 100755..100644 --- a/moses/src/LM/Base.h +++ b/moses/src/LM/Base.h diff --git a/moses/src/LM/DMap.cpp b/moses/src/LM/DMap.cpp index 3bef47083..3bef47083 100755..100644 --- a/moses/src/LM/DMap.cpp +++ b/moses/src/LM/DMap.cpp diff --git a/moses/src/LM/DMap.h b/moses/src/LM/DMap.h index f9fbf56c2..f9fbf56c2 100755..100644 --- a/moses/src/LM/DMap.h +++ b/moses/src/LM/DMap.h diff --git a/moses/src/LM/Factory.cpp b/moses/src/LM/Factory.cpp index 5d877e9d6..5d877e9d6 100755..100644 --- a/moses/src/LM/Factory.cpp +++ b/moses/src/LM/Factory.cpp diff --git a/moses/src/LM/Factory.h b/moses/src/LM/Factory.h index 1a140846a..1a140846a 100755..100644 --- a/moses/src/LM/Factory.h +++ b/moses/src/LM/Factory.h diff --git a/moses/src/LM/IRST.cpp b/moses/src/LM/IRST.cpp index 692385b4d..692385b4d 100755..100644 --- a/moses/src/LM/IRST.cpp +++ b/moses/src/LM/IRST.cpp diff --git a/moses/src/LM/IRST.h b/moses/src/LM/IRST.h index 2ffe67f98..2ffe67f98 100755..100644 --- a/moses/src/LM/IRST.h +++ b/moses/src/LM/IRST.h diff --git a/moses/src/LM/Implementation.cpp b/moses/src/LM/Implementation.cpp index 589ed375a..589ed375a 100755..100644 --- a/moses/src/LM/Implementation.cpp +++ b/moses/src/LM/Implementation.cpp diff --git a/moses/src/LM/Implementation.h b/moses/src/LM/Implementation.h index a7c637ef8..a7c637ef8 100755..100644 --- a/moses/src/LM/Implementation.h +++ b/moses/src/LM/Implementation.h diff --git a/moses/src/LM/Jamfile b/moses/src/LM/Jamfile index 2f3cde8ef..2f3cde8ef 100755..100644 --- a/moses/src/LM/Jamfile +++ b/moses/src/LM/Jamfile diff --git a/moses/src/LM/Joint.cpp b/moses/src/LM/Joint.cpp index 266f08029..266f08029 100755..100644 --- a/moses/src/LM/Joint.cpp +++ b/moses/src/LM/Joint.cpp diff --git a/moses/src/LM/Joint.h b/moses/src/LM/Joint.h index e000409be..e000409be 100755..100644 --- a/moses/src/LM/Joint.h +++ b/moses/src/LM/Joint.h diff --git a/moses/src/LM/Ken.cpp b/moses/src/LM/Ken.cpp index c5307ffc0..c5307ffc0 100755..100644 --- a/moses/src/LM/Ken.cpp +++ b/moses/src/LM/Ken.cpp diff --git a/moses/src/LM/Ken.h b/moses/src/LM/Ken.h index f55679395..f55679395 100755..100644 --- a/moses/src/LM/Ken.h +++ b/moses/src/LM/Ken.h diff --git a/moses/src/LM/MultiFactor.cpp b/moses/src/LM/MultiFactor.cpp index d141b88b3..d141b88b3 100755..100644 --- a/moses/src/LM/MultiFactor.cpp +++ b/moses/src/LM/MultiFactor.cpp diff --git a/moses/src/LM/MultiFactor.h b/moses/src/LM/MultiFactor.h index 32cef51d9..32cef51d9 100755..100644 --- a/moses/src/LM/MultiFactor.h +++ b/moses/src/LM/MultiFactor.h diff --git a/moses/src/LM/ORLM.cpp b/moses/src/LM/ORLM.cpp index 30f3ada55..30f3ada55 100755..100644 --- a/moses/src/LM/ORLM.cpp +++ b/moses/src/LM/ORLM.cpp diff --git a/moses/src/LM/ORLM.h b/moses/src/LM/ORLM.h index c1ccb4387..c1ccb4387 100755..100644 --- a/moses/src/LM/ORLM.h +++ b/moses/src/LM/ORLM.h diff --git a/moses/src/LM/ParallelBackoff.cpp b/moses/src/LM/ParallelBackoff.cpp index ec2fb2f78..ec2fb2f78 100755..100644 --- a/moses/src/LM/ParallelBackoff.cpp +++ b/moses/src/LM/ParallelBackoff.cpp diff --git a/moses/src/LM/ParallelBackoff.h b/moses/src/LM/ParallelBackoff.h index 8e4241395..8e4241395 100755..100644 --- a/moses/src/LM/ParallelBackoff.h +++ b/moses/src/LM/ParallelBackoff.h diff --git a/moses/src/LM/Rand.cpp b/moses/src/LM/Rand.cpp index 93411b8d3..93411b8d3 100755..100644 --- a/moses/src/LM/Rand.cpp +++ b/moses/src/LM/Rand.cpp diff --git a/moses/src/LM/Rand.h b/moses/src/LM/Rand.h index c4ef99947..c4ef99947 100755..100644 --- a/moses/src/LM/Rand.h +++ b/moses/src/LM/Rand.h diff --git a/moses/src/LM/Remote.cpp b/moses/src/LM/Remote.cpp index 78380c75e..78380c75e 100755..100644 --- a/moses/src/LM/Remote.cpp +++ b/moses/src/LM/Remote.cpp diff --git a/moses/src/LM/Remote.h b/moses/src/LM/Remote.h index 9a70d64c0..9a70d64c0 100755..100644 --- a/moses/src/LM/Remote.h +++ b/moses/src/LM/Remote.h diff --git a/moses/src/LM/SRI.cpp b/moses/src/LM/SRI.cpp index 9c5a9c3d4..9c5a9c3d4 100755..100644 --- a/moses/src/LM/SRI.cpp +++ b/moses/src/LM/SRI.cpp diff --git a/moses/src/LM/SRI.h b/moses/src/LM/SRI.h index 69d55f117..69d55f117 100755..100644 --- a/moses/src/LM/SRI.h +++ b/moses/src/LM/SRI.h diff --git a/moses/src/LM/SingleFactor.cpp b/moses/src/LM/SingleFactor.cpp index 73c9e6e83..73c9e6e83 100755..100644 --- a/moses/src/LM/SingleFactor.cpp +++ b/moses/src/LM/SingleFactor.cpp diff --git a/moses/src/LM/SingleFactor.h b/moses/src/LM/SingleFactor.h index 44fdf9b16..44fdf9b16 100755..100644 --- a/moses/src/LM/SingleFactor.h +++ b/moses/src/LM/SingleFactor.h diff --git a/moses/src/LMList.cpp b/moses/src/LMList.cpp index cac1199e0..cac1199e0 100755..100644 --- a/moses/src/LMList.cpp +++ b/moses/src/LMList.cpp diff --git a/moses/src/LMList.h b/moses/src/LMList.h index a803e16df..a803e16df 100755..100644 --- a/moses/src/LMList.h +++ b/moses/src/LMList.h diff --git a/moses/src/LVoc.cpp b/moses/src/LVoc.cpp index d1ee92237..d1ee92237 100755..100644 --- a/moses/src/LVoc.cpp +++ b/moses/src/LVoc.cpp diff --git a/moses/src/LVoc.h b/moses/src/LVoc.h index ec20fe3cd..ec20fe3cd 100755..100644 --- a/moses/src/LVoc.h +++ b/moses/src/LVoc.h diff --git a/moses/src/LexicalReordering.cpp b/moses/src/LexicalReordering.cpp index b32b96b0d..b32b96b0d 100755..100644 --- a/moses/src/LexicalReordering.cpp +++ b/moses/src/LexicalReordering.cpp diff --git a/moses/src/LexicalReordering.h b/moses/src/LexicalReordering.h index e979b5bf5..e979b5bf5 100755..100644 --- a/moses/src/LexicalReordering.h +++ b/moses/src/LexicalReordering.h diff --git a/moses/src/LexicalReorderingState.cpp b/moses/src/LexicalReorderingState.cpp index 346ac745b..346ac745b 100755..100644 --- a/moses/src/LexicalReorderingState.cpp +++ b/moses/src/LexicalReorderingState.cpp diff --git a/moses/src/LexicalReorderingState.h b/moses/src/LexicalReorderingState.h index 802f1fcb0..802f1fcb0 100755..100644 --- a/moses/src/LexicalReorderingState.h +++ b/moses/src/LexicalReorderingState.h diff --git a/moses/src/LexicalReorderingTable.cpp b/moses/src/LexicalReorderingTable.cpp index 00be0a495..00be0a495 100755..100644 --- a/moses/src/LexicalReorderingTable.cpp +++ b/moses/src/LexicalReorderingTable.cpp diff --git a/moses/src/LexicalReorderingTable.h b/moses/src/LexicalReorderingTable.h index ae10ae386..ae10ae386 100755..100644 --- a/moses/src/LexicalReorderingTable.h +++ b/moses/src/LexicalReorderingTable.h diff --git a/moses/src/Makefile.am b/moses/src/Makefile.am index b43191773..b43191773 100755..100644 --- a/moses/src/Makefile.am +++ b/moses/src/Makefile.am diff --git a/moses/src/Manager.cpp b/moses/src/Manager.cpp index c27d88fee..c27d88fee 100755..100644 --- a/moses/src/Manager.cpp +++ b/moses/src/Manager.cpp diff --git a/moses/src/Manager.h b/moses/src/Manager.h index 4b45414ee..4b45414ee 100755..100644 --- a/moses/src/Manager.h +++ b/moses/src/Manager.h diff --git a/moses/src/MockHypothesis.cpp b/moses/src/MockHypothesis.cpp index 0afd3b071..0afd3b071 100755..100644 --- a/moses/src/MockHypothesis.cpp +++ b/moses/src/MockHypothesis.cpp diff --git a/moses/src/MockHypothesis.h b/moses/src/MockHypothesis.h index 41f118f40..41f118f40 100755..100644 --- a/moses/src/MockHypothesis.h +++ b/moses/src/MockHypothesis.h diff --git a/moses/src/MosesTest.cpp b/moses/src/MosesTest.cpp index 45f5bdba4..45f5bdba4 100755..100644 --- a/moses/src/MosesTest.cpp +++ b/moses/src/MosesTest.cpp diff --git a/moses/src/NonTerminal.cpp b/moses/src/NonTerminal.cpp index 37ebc9b68..37ebc9b68 100755..100644 --- a/moses/src/NonTerminal.cpp +++ b/moses/src/NonTerminal.cpp diff --git a/moses/src/NonTerminal.h b/moses/src/NonTerminal.h index 3fa2ede92..3fa2ede92 100755..100644 --- a/moses/src/NonTerminal.h +++ b/moses/src/NonTerminal.h diff --git a/moses/src/ObjectPool.h b/moses/src/ObjectPool.h index 05c849f79..05c849f79 100755..100644 --- a/moses/src/ObjectPool.h +++ b/moses/src/ObjectPool.h diff --git a/moses/src/OnlineCommand.cpp b/moses/src/OnlineCommand.cpp index 93024b92c..93024b92c 100755..100644 --- a/moses/src/OnlineCommand.cpp +++ b/moses/src/OnlineCommand.cpp diff --git a/moses/src/OnlineCommand.h b/moses/src/OnlineCommand.h index 90f778ea4..90f778ea4 100755..100644 --- a/moses/src/OnlineCommand.h +++ b/moses/src/OnlineCommand.h diff --git a/moses/src/OutputCollector.h b/moses/src/OutputCollector.h index 0b56c6d80..0b56c6d80 100755..100644 --- a/moses/src/OutputCollector.h +++ b/moses/src/OutputCollector.h diff --git a/moses/src/PCNTools.cpp b/moses/src/PCNTools.cpp index fac386444..fac386444 100755..100644 --- a/moses/src/PCNTools.cpp +++ b/moses/src/PCNTools.cpp diff --git a/moses/src/PCNTools.h b/moses/src/PCNTools.h index 1147b73bb..1147b73bb 100755..100644 --- a/moses/src/PCNTools.h +++ b/moses/src/PCNTools.h diff --git a/moses/src/PDTAimp.h b/moses/src/PDTAimp.h index 7bff39c07..0c2a984b0 100755..100644 --- a/moses/src/PDTAimp.h +++ b/moses/src/PDTAimp.h @@ -33,13 +33,11 @@ class PDTAimp protected: PDTAimp(PhraseDictionaryTreeAdaptor *p,unsigned nis) - : m_languageModels(0),m_weightWP(0.0),m_dict(0), + : m_languageModels(0),m_dict(0), m_obj(p),useCache(1),m_numInputScores(nis),totalE(0),distinctE(0) {} -public: - std::vector<float> m_weights; + public: LMList const* m_languageModels; - float m_weightWP; std::vector<FactorType> m_input,m_output; PhraseDictionaryTree *m_dict; typedef std::vector<TargetPhraseCollection const*> vTPC; @@ -139,6 +137,10 @@ public: return 0; } + const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); + std::vector<float> weights = system.GetTranslationWeights(m_obj->GetDictIndex()); + float weightWP = system.GetWeightWordPenalty(); + std::vector<TargetPhrase> tCands; tCands.reserve(cands.size()); std::vector<std::pair<float,size_t> > costs; @@ -165,7 +167,7 @@ public: *(cands[i].fnames[j]), cands[i].fvalues[j]); } } - CreateTargetPhrase(targetPhrase,factorStrings,scoreVector, sparseFeatures, wacands[i],&src); + CreateTargetPhrase(targetPhrase,factorStrings,scoreVector, sparseFeatures, wacands[i], weights, weightWP, &src); costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size())); tCands.push_back(targetPhrase); } @@ -188,9 +190,8 @@ public: void Create(const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath - , const std::vector<float> &weight + , const std::vector<float> &weight , const LMList &languageModels - , float weightWP ) { // set my members @@ -198,8 +199,6 @@ public: m_input=input; m_output=output; m_languageModels=&languageModels; - m_weightWP=weightWP; - m_weights=weight; const StaticData &staticData = StaticData::Instance(); m_dict->UseWordAlignment(staticData.UseAlignmentInfo()); @@ -262,8 +261,10 @@ public: Scores const& scoreVector, const ScoreComponentCollection& sparseFeatures, const std::string& alignmentString, + std::vector<float> &weights, + float weightWP, Phrase const* srcPtr=0) const { - CreateTargetPhrase(targetPhrase, factorStrings, scoreVector, sparseFeatures, srcPtr); + CreateTargetPhrase(targetPhrase, factorStrings, scoreVector, sparseFeatures, weights, weightWP, srcPtr); targetPhrase.SetAlignmentInfo(alignmentString); } @@ -272,6 +273,8 @@ public: StringTgtCand::Tokens const& factorStrings, Scores const& scoreVector, const ScoreComponentCollection& sparseFeatures, + std::vector<float> &weights, + float weightWP, Phrase const* srcPtr=0) const { FactorCollection &factorCollection = FactorCollection::Instance(); @@ -284,7 +287,7 @@ public: } } - targetPhrase.SetScore(m_obj->GetFeature(), scoreVector, sparseFeatures, m_weights, m_weightWP, *m_languageModels); + targetPhrase.SetScore(m_obj->GetFeature(), scoreVector, sparseFeatures, weights, weightWP, *m_languageModels); targetPhrase.SetSourcePhrase(*srcPtr); } @@ -366,6 +369,10 @@ public: for(Position i=0 ; i < srcSize ; ++i) stack.push_back(State(i, i, m_dict->GetRoot(), std::vector<float>(m_numInputScores,0.0))); + const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); + std::vector<float> weightT = system.GetTranslationWeights(m_obj->GetDictIndex()); + float weightWP = system.GetWeightWordPenalty(); + while(!stack.empty()) { State curr(stack.back()); stack.pop_back(); @@ -440,19 +447,13 @@ public: //put in phrase table scores, logging as we insert std::transform(tcands[i].scores.begin(),tcands[i].scores.end(),nscores.begin() + m_numInputScores,TransformScore); - - CHECK(nscores.size()==m_weights.size()); - - const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); + CHECK(nscores.size()==weightT.size()); + //tally up - std::vector<float> weightT = system.GetTranslationWeights(); - //float score=std::inner_product(nscores.begin(), nscores.end(), m_weights.begin(), 0.0f); float score=std::inner_product(nscores.begin(), nscores.end(), weightT.begin(), 0.0f); //count word penalty - float weightWP = system.GetWeightWordPenalty(); - //score-=tcands[i].tokens.size() * m_weightWP; score-=tcands[i].tokens.size() * weightWP; std::pair<E2Costs::iterator,bool> p=e2costs.insert(std::make_pair(tcands[i].tokens,TScores())); @@ -501,7 +502,7 @@ public: for(E2Costs::const_iterator j=i->second.begin(); j!=i->second.end(); ++j) { TScores const & scores=j->second; TargetPhrase targetPhrase(Output); - CreateTargetPhrase(targetPhrase,j->first,scores.trans,ScoreComponentCollection(),scores.src); + CreateTargetPhrase(targetPhrase,j->first,scores.trans,ScoreComponentCollection(),weightT,weightWP,scores.src); costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size())); tCands.push_back(targetPhrase); //std::cerr << i->first.first << "-" << i->first.second << ": " << targetPhrase << std::endl; diff --git a/moses/src/Parameter.cpp b/moses/src/Parameter.cpp index a24c6033c..a24c6033c 100755..100644 --- a/moses/src/Parameter.cpp +++ b/moses/src/Parameter.cpp diff --git a/moses/src/Parameter.h b/moses/src/Parameter.h index de0d3f7e7..de0d3f7e7 100755..100644 --- a/moses/src/Parameter.h +++ b/moses/src/Parameter.h diff --git a/moses/src/PartialTranslOptColl.cpp b/moses/src/PartialTranslOptColl.cpp index 91ad65415..91ad65415 100755..100644 --- a/moses/src/PartialTranslOptColl.cpp +++ b/moses/src/PartialTranslOptColl.cpp diff --git a/moses/src/PartialTranslOptColl.h b/moses/src/PartialTranslOptColl.h index b8b16b901..b8b16b901 100755..100644 --- a/moses/src/PartialTranslOptColl.h +++ b/moses/src/PartialTranslOptColl.h diff --git a/moses/src/Phrase.cpp b/moses/src/Phrase.cpp index db808390c..db808390c 100755..100644 --- a/moses/src/Phrase.cpp +++ b/moses/src/Phrase.cpp diff --git a/moses/src/Phrase.h b/moses/src/Phrase.h index b64a84a25..b64a84a25 100755..100644 --- a/moses/src/Phrase.h +++ b/moses/src/Phrase.h diff --git a/moses/src/PhraseBoundaryFeature.cpp b/moses/src/PhraseBoundaryFeature.cpp index 461fcbe57..461fcbe57 100755..100644 --- a/moses/src/PhraseBoundaryFeature.cpp +++ b/moses/src/PhraseBoundaryFeature.cpp diff --git a/moses/src/PhraseBoundaryFeature.h b/moses/src/PhraseBoundaryFeature.h index 4167ab9b2..4167ab9b2 100755..100644 --- a/moses/src/PhraseBoundaryFeature.h +++ b/moses/src/PhraseBoundaryFeature.h diff --git a/moses/src/PhraseDictionary.cpp b/moses/src/PhraseDictionary.cpp index 0fc39a286..49615456c 100755..100644 --- a/moses/src/PhraseDictionary.cpp +++ b/moses/src/PhraseDictionary.cpp @@ -46,6 +46,11 @@ GetTargetPhraseCollection(InputType const& src,WordsRange const& range) const return GetTargetPhraseCollection(src.GetSubString(range)); } +size_t PhraseDictionary::GetDictIndex() const +{ + return m_feature->GetDictIndex(); +} + PhraseDictionaryFeature::PhraseDictionaryFeature (PhraseTableImplementation implementation , SparsePhraseDictionaryFeature* spdf @@ -55,13 +60,14 @@ PhraseDictionaryFeature::PhraseDictionaryFeature , const std::vector<FactorType> &output , const std::string &filePath , const std::vector<float> &weight + , size_t dictIndex , size_t tableLimit , const std::string &targetFile // default param , const std::string &alignmentsFile) // default param :DecodeFeature("PhraseModel",numScoreComponent,input,output), m_numInputScores(numInputScores), m_filePath(filePath), - m_weight(weight), + m_dictIndex(dictIndex), m_tableLimit(tableLimit), m_implementation(implementation), m_targetFile(targetFile), @@ -70,6 +76,10 @@ PhraseDictionaryFeature::PhraseDictionaryFeature { if (implementation == Memory || implementation == SCFG || implementation == SuffixArray) { m_useThreadSafePhraseDictionary = true; + if (implementation == SuffixArray) { + cerr << "Warning: implementation holds chached weights!" << endl; + exit(1); + } } else { m_useThreadSafePhraseDictionary = false; } @@ -78,7 +88,7 @@ PhraseDictionaryFeature::PhraseDictionaryFeature PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSystem* system) { const StaticData& staticData = StaticData::Instance(); - std::vector<float> weightT = system->GetTranslationWeights(); + std::vector<float> weightT = system->GetTranslationWeights(m_dictIndex); if (m_implementation == Memory) { // memory phrase table @@ -95,7 +105,6 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst PhraseDictionaryMemory* pdm = new PhraseDictionaryMemory(GetNumScoreComponents(),this); bool ret = pdm->Load(GetInput(), GetOutput() , m_filePath - //, m_weight , weightT , m_tableLimit , system->GetLanguageModels() @@ -107,7 +116,6 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdta->Load( GetInput() , GetOutput() , m_filePath - //, m_weight , weightT , m_tableLimit , system->GetLanguageModels() @@ -126,8 +134,7 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdm->Load(GetInput() , GetOutput() , m_filePath - // m_weight - , weightT + , weightT , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); @@ -145,7 +152,6 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdm->Load(GetInput() , GetOutput() , m_filePath - //, m_weight , weightT , m_tableLimit , system->GetLanguageModels() @@ -154,6 +160,8 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst return pdm; } else if (m_implementation == ALSuffixArray) { // memory phrase table + cerr << "Warning: Implementation holds cached weights!" << endl; + exit(1); VERBOSE(2,"using Hiero format phrase tables" << std::endl); if (!FileExists(m_filePath) && FileExists(m_filePath + ".gz")) { m_filePath += ".gz"; @@ -164,9 +172,8 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdm->Load(GetInput() , GetOutput() , m_filePath - //, m_weight - , weightT - , m_tableLimit + , weightT + , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); CHECK(ret); @@ -177,14 +184,15 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst bool ret = pdta->Load(GetInput() , GetOutput() , m_filePath - //, m_weight - , weightT - , m_tableLimit + , weightT + , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); CHECK(ret); return pdta; } else if (m_implementation == SuffixArray) { + cerr << "Warning: Implementation holds cached weights!" << endl; + exit(1); #ifndef WIN32 PhraseDictionaryDynSuffixArray *pd = new PhraseDictionaryDynSuffixArray(GetNumScoreComponents(), this); if(!(pd->Load( @@ -192,11 +200,10 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst ,GetOutput() ,m_filePath ,m_targetFile - , m_alignmentsFile - //, m_weight, m_tableLimit - , weightT, m_tableLimit - , system->GetLanguageModels() - , system->GetWeightWordPenalty()))) { + ,m_alignmentsFile + ,weightT, m_tableLimit + ,system->GetLanguageModels() + ,system->GetWeightWordPenalty()))) { std::cerr << "FAILED TO LOAD\n" << endl; delete pd; pd = NULL; @@ -297,5 +304,10 @@ const PhraseDictionaryFeature* PhraseDictionary::GetFeature() const return m_feature; } +size_t PhraseDictionaryFeature::GetDictIndex() const +{ + return m_dictIndex; +} + } diff --git a/moses/src/PhraseDictionary.h b/moses/src/PhraseDictionary.h index e8f478079..18b48436e 100755..100644 --- a/moses/src/PhraseDictionary.h +++ b/moses/src/PhraseDictionary.h @@ -69,7 +69,7 @@ public: return Translate; } const PhraseDictionaryFeature* GetFeature() const; - + size_t GetDictIndex() const; //! find list of translations that can translates src. Only for phrase input virtual const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase& src) const=0; @@ -105,6 +105,7 @@ public: , const std::vector<FactorType> &output , const std::string &filePath , const std::vector<float> &weight + , size_t dictIndex , size_t tableLimit , const std::string &targetFile , const std::string &alignmentsFile); @@ -131,6 +132,10 @@ public: //Get the dictionary. Be sure to initialise it first. const PhraseDictionary* GetDictionary() const; PhraseDictionary* GetDictionary(); + size_t GetDictIndex() const; + +protected: + size_t m_dictIndex; private: /** Load the appropriate phrase table */ @@ -138,7 +143,6 @@ private: unsigned m_numInputScores; std::string m_filePath; - std::vector<float> m_weight; size_t m_tableLimit; //We instantiate either the the thread-safe or non-thread-safe dictionary, //but not both. The thread-safe one can be instantiated in the constructor and shared diff --git a/moses/src/PhraseDictionaryALSuffixArray.cpp b/moses/src/PhraseDictionaryALSuffixArray.cpp index 71b810a91..99fe968f7 100755..100644 --- a/moses/src/PhraseDictionaryALSuffixArray.cpp +++ b/moses/src/PhraseDictionaryALSuffixArray.cpp @@ -56,8 +56,7 @@ void PhraseDictionaryALSuffixArray::InitializeForInput(InputType const& source) std::auto_ptr<RuleTableLoader> loader = RuleTableLoaderFactory::Create(grammarFile); - std::vector<float> weightT = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(); - //bool ret = loader->Load(*m_input, *m_output, inFile, *m_weight, m_tableLimit, + std::vector<float> weightT = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(GetDictIndex()); bool ret = loader->Load(*m_input, *m_output, inFile, weightT, m_tableLimit, *m_languageModels, m_wpProducer, *this); diff --git a/moses/src/PhraseDictionaryALSuffixArray.h b/moses/src/PhraseDictionaryALSuffixArray.h index 79c928c69..79c928c69 100755..100644 --- a/moses/src/PhraseDictionaryALSuffixArray.h +++ b/moses/src/PhraseDictionaryALSuffixArray.h diff --git a/moses/src/PhraseDictionaryDynSuffixArray.cpp b/moses/src/PhraseDictionaryDynSuffixArray.cpp index d4f0161db..d4f0161db 100755..100644 --- a/moses/src/PhraseDictionaryDynSuffixArray.cpp +++ b/moses/src/PhraseDictionaryDynSuffixArray.cpp diff --git a/moses/src/PhraseDictionaryDynSuffixArray.h b/moses/src/PhraseDictionaryDynSuffixArray.h index 0d6be08dd..0d6be08dd 100755..100644 --- a/moses/src/PhraseDictionaryDynSuffixArray.h +++ b/moses/src/PhraseDictionaryDynSuffixArray.h diff --git a/moses/src/PhraseDictionaryHiero.cpp b/moses/src/PhraseDictionaryHiero.cpp index 5899ec8b1..5899ec8b1 100755..100644 --- a/moses/src/PhraseDictionaryHiero.cpp +++ b/moses/src/PhraseDictionaryHiero.cpp diff --git a/moses/src/PhraseDictionaryHiero.h b/moses/src/PhraseDictionaryHiero.h index f845c4e15..f845c4e15 100755..100644 --- a/moses/src/PhraseDictionaryHiero.h +++ b/moses/src/PhraseDictionaryHiero.h diff --git a/moses/src/PhraseDictionaryMemory.cpp b/moses/src/PhraseDictionaryMemory.cpp index 1ee449172..1ee449172 100755..100644 --- a/moses/src/PhraseDictionaryMemory.cpp +++ b/moses/src/PhraseDictionaryMemory.cpp diff --git a/moses/src/PhraseDictionaryMemory.h b/moses/src/PhraseDictionaryMemory.h index dad7f4a9b..dad7f4a9b 100755..100644 --- a/moses/src/PhraseDictionaryMemory.h +++ b/moses/src/PhraseDictionaryMemory.h diff --git a/moses/src/PhraseDictionaryNode.cpp b/moses/src/PhraseDictionaryNode.cpp index 5b552bd53..5b552bd53 100755..100644 --- a/moses/src/PhraseDictionaryNode.cpp +++ b/moses/src/PhraseDictionaryNode.cpp diff --git a/moses/src/PhraseDictionaryNode.h b/moses/src/PhraseDictionaryNode.h index 95ae48b0e..95ae48b0e 100755..100644 --- a/moses/src/PhraseDictionaryNode.h +++ b/moses/src/PhraseDictionaryNode.h diff --git a/moses/src/PhraseDictionaryNodeSCFG.cpp b/moses/src/PhraseDictionaryNodeSCFG.cpp index 91f5f7acd..91f5f7acd 100755..100644 --- a/moses/src/PhraseDictionaryNodeSCFG.cpp +++ b/moses/src/PhraseDictionaryNodeSCFG.cpp diff --git a/moses/src/PhraseDictionaryNodeSCFG.h b/moses/src/PhraseDictionaryNodeSCFG.h index 80e3722b5..80e3722b5 100755..100644 --- a/moses/src/PhraseDictionaryNodeSCFG.h +++ b/moses/src/PhraseDictionaryNodeSCFG.h diff --git a/moses/src/PhraseDictionaryOnDisk.cpp b/moses/src/PhraseDictionaryOnDisk.cpp index 3c89408f5..5bada2aea 100755..100644 --- a/moses/src/PhraseDictionaryOnDisk.cpp +++ b/moses/src/PhraseDictionaryOnDisk.cpp @@ -39,7 +39,7 @@ PhraseDictionaryOnDisk::~PhraseDictionaryOnDisk() bool PhraseDictionaryOnDisk::Load(const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath - , const std::vector<float> &weight + , const std::vector<float> &weight , size_t tableLimit , const LMList& languageModels , const WordPenaltyProducer* wpProducer) @@ -51,8 +51,6 @@ bool PhraseDictionaryOnDisk::Load(const std::vector<FactorType> &input m_inputFactorsVec = input; m_outputFactorsVec = output; - m_weight = weight; - LoadTargetLookup(); if (!m_dbWrapper.BeginLoad(filePath)) @@ -94,13 +92,11 @@ ChartRuleLookupManager *PhraseDictionaryOnDisk::CreateRuleLookupManager( const InputType &sentence, const ChartCellCollection &cellCollection) { - std::vector<float> weightT = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(); + std::vector<float> weightT = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT).GetTranslationWeights(GetDictIndex()); return new ChartRuleLookupManagerOnDisk(sentence, cellCollection, *this, m_dbWrapper, m_languageModels, m_wpProducer, m_inputFactorsVec, - //m_outputFactorsVec, m_weight, - m_outputFactorsVec, weightT, - m_filePath); + m_outputFactorsVec, m_filePath); } } diff --git a/moses/src/PhraseDictionaryOnDisk.h b/moses/src/PhraseDictionaryOnDisk.h index 0a8dfc808..64c77cbd7 100755..100644 --- a/moses/src/PhraseDictionaryOnDisk.h +++ b/moses/src/PhraseDictionaryOnDisk.h @@ -45,7 +45,6 @@ protected: const LMList* m_languageModels; const WordPenaltyProducer* m_wpProducer; std::vector<FactorType> m_inputFactorsVec, m_outputFactorsVec; - std::vector<float> m_weight; std::string m_filePath; void LoadTargetLookup(); @@ -63,7 +62,7 @@ public: bool Load(const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath - , const std::vector<float> &weight + , const std::vector<float> &weight , size_t tableLimit, const LMList& languageModels, const WordPenaltyProducer* wpProducer); diff --git a/moses/src/PhraseDictionarySCFG.cpp b/moses/src/PhraseDictionarySCFG.cpp index c3a243990..c3a243990 100755..100644 --- a/moses/src/PhraseDictionarySCFG.cpp +++ b/moses/src/PhraseDictionarySCFG.cpp diff --git a/moses/src/PhraseDictionarySCFG.h b/moses/src/PhraseDictionarySCFG.h index e338dac50..e338dac50 100755..100644 --- a/moses/src/PhraseDictionarySCFG.h +++ b/moses/src/PhraseDictionarySCFG.h diff --git a/moses/src/PhraseDictionaryTree.cpp b/moses/src/PhraseDictionaryTree.cpp index 13b404724..13b404724 100755..100644 --- a/moses/src/PhraseDictionaryTree.cpp +++ b/moses/src/PhraseDictionaryTree.cpp diff --git a/moses/src/PhraseDictionaryTree.h b/moses/src/PhraseDictionaryTree.h index ebe43a618..ebe43a618 100755..100644 --- a/moses/src/PhraseDictionaryTree.h +++ b/moses/src/PhraseDictionaryTree.h diff --git a/moses/src/PhraseDictionaryTreeAdaptor.cpp b/moses/src/PhraseDictionaryTreeAdaptor.cpp index cce4a2ee4..154572bce 100755..100644 --- a/moses/src/PhraseDictionaryTreeAdaptor.cpp +++ b/moses/src/PhraseDictionaryTreeAdaptor.cpp @@ -55,8 +55,7 @@ bool PhraseDictionaryTreeAdaptor::Load(const std::vector<FactorType> &input // set PhraseDictionary members m_tableLimit=tableLimit; - imp->Create(input,output,filePath, - weight,languageModels,weightWP); + imp->Create(input,output,filePath,weight,languageModels); return true; } diff --git a/moses/src/PhraseDictionaryTreeAdaptor.h b/moses/src/PhraseDictionaryTreeAdaptor.h index c38e234fb..c38e234fb 100755..100644 --- a/moses/src/PhraseDictionaryTreeAdaptor.h +++ b/moses/src/PhraseDictionaryTreeAdaptor.h diff --git a/moses/src/PhraseLengthFeature.cpp b/moses/src/PhraseLengthFeature.cpp index ece3b0923..ece3b0923 100755..100644 --- a/moses/src/PhraseLengthFeature.cpp +++ b/moses/src/PhraseLengthFeature.cpp diff --git a/moses/src/PhraseLengthFeature.h b/moses/src/PhraseLengthFeature.h index f25142230..f25142230 100755..100644 --- a/moses/src/PhraseLengthFeature.h +++ b/moses/src/PhraseLengthFeature.h diff --git a/moses/src/PhrasePairFeature.cpp b/moses/src/PhrasePairFeature.cpp index c44807205..c44807205 100755..100644 --- a/moses/src/PhrasePairFeature.cpp +++ b/moses/src/PhrasePairFeature.cpp diff --git a/moses/src/PhrasePairFeature.h b/moses/src/PhrasePairFeature.h index 6dab1a998..6dab1a998 100755..100644 --- a/moses/src/PhrasePairFeature.h +++ b/moses/src/PhrasePairFeature.h diff --git a/moses/src/PrefixTree.h b/moses/src/PrefixTree.h index 3215fb834..3215fb834 100755..100644 --- a/moses/src/PrefixTree.h +++ b/moses/src/PrefixTree.h diff --git a/moses/src/PrefixTreeMap.cpp b/moses/src/PrefixTreeMap.cpp index 7e5512a05..7e5512a05 100755..100644 --- a/moses/src/PrefixTreeMap.cpp +++ b/moses/src/PrefixTreeMap.cpp diff --git a/moses/src/PrefixTreeMap.h b/moses/src/PrefixTreeMap.h index 6bb6b769f..6bb6b769f 100755..100644 --- a/moses/src/PrefixTreeMap.h +++ b/moses/src/PrefixTreeMap.h diff --git a/moses/src/ReorderingConstraint.cpp b/moses/src/ReorderingConstraint.cpp index ea0c1b807..ea0c1b807 100755..100644 --- a/moses/src/ReorderingConstraint.cpp +++ b/moses/src/ReorderingConstraint.cpp diff --git a/moses/src/ReorderingConstraint.h b/moses/src/ReorderingConstraint.h index 742b2228b..742b2228b 100755..100644 --- a/moses/src/ReorderingConstraint.h +++ b/moses/src/ReorderingConstraint.h diff --git a/moses/src/ReorderingStack.cpp b/moses/src/ReorderingStack.cpp index 49a723a36..49a723a36 100755..100644 --- a/moses/src/ReorderingStack.cpp +++ b/moses/src/ReorderingStack.cpp diff --git a/moses/src/ReorderingStack.h b/moses/src/ReorderingStack.h index 9a710b348..9a710b348 100755..100644 --- a/moses/src/ReorderingStack.h +++ b/moses/src/ReorderingStack.h diff --git a/moses/src/RuleCube.cpp b/moses/src/RuleCube.cpp index b028abe3e..b028abe3e 100755..100644 --- a/moses/src/RuleCube.cpp +++ b/moses/src/RuleCube.cpp diff --git a/moses/src/RuleCube.h b/moses/src/RuleCube.h index 5d6aabe46..5d6aabe46 100755..100644 --- a/moses/src/RuleCube.h +++ b/moses/src/RuleCube.h diff --git a/moses/src/RuleCubeItem.cpp b/moses/src/RuleCubeItem.cpp index fa4b4c51e..fa4b4c51e 100755..100644 --- a/moses/src/RuleCubeItem.cpp +++ b/moses/src/RuleCubeItem.cpp diff --git a/moses/src/RuleCubeItem.h b/moses/src/RuleCubeItem.h index 1b953b1b4..1b953b1b4 100755..100644 --- a/moses/src/RuleCubeItem.h +++ b/moses/src/RuleCubeItem.h diff --git a/moses/src/RuleCubeQueue.cpp b/moses/src/RuleCubeQueue.cpp index 89020a4e5..89020a4e5 100755..100644 --- a/moses/src/RuleCubeQueue.cpp +++ b/moses/src/RuleCubeQueue.cpp diff --git a/moses/src/RuleCubeQueue.h b/moses/src/RuleCubeQueue.h index 5ef3b3af2..5ef3b3af2 100755..100644 --- a/moses/src/RuleCubeQueue.h +++ b/moses/src/RuleCubeQueue.h diff --git a/moses/src/RuleTableLoader.h b/moses/src/RuleTableLoader.h index b5d4d8e24..b5d4d8e24 100755..100644 --- a/moses/src/RuleTableLoader.h +++ b/moses/src/RuleTableLoader.h diff --git a/moses/src/RuleTableLoaderCompact.cpp b/moses/src/RuleTableLoaderCompact.cpp index 64297948f..64297948f 100755..100644 --- a/moses/src/RuleTableLoaderCompact.cpp +++ b/moses/src/RuleTableLoaderCompact.cpp diff --git a/moses/src/RuleTableLoaderCompact.h b/moses/src/RuleTableLoaderCompact.h index 56918782b..56918782b 100755..100644 --- a/moses/src/RuleTableLoaderCompact.h +++ b/moses/src/RuleTableLoaderCompact.h diff --git a/moses/src/RuleTableLoaderFactory.cpp b/moses/src/RuleTableLoaderFactory.cpp index 7ac97bde2..7ac97bde2 100755..100644 --- a/moses/src/RuleTableLoaderFactory.cpp +++ b/moses/src/RuleTableLoaderFactory.cpp diff --git a/moses/src/RuleTableLoaderFactory.h b/moses/src/RuleTableLoaderFactory.h index c9c4daf1d..c9c4daf1d 100755..100644 --- a/moses/src/RuleTableLoaderFactory.h +++ b/moses/src/RuleTableLoaderFactory.h diff --git a/moses/src/RuleTableLoaderHiero.cpp b/moses/src/RuleTableLoaderHiero.cpp index 6899cfa61..6899cfa61 100755..100644 --- a/moses/src/RuleTableLoaderHiero.cpp +++ b/moses/src/RuleTableLoaderHiero.cpp diff --git a/moses/src/RuleTableLoaderHiero.h b/moses/src/RuleTableLoaderHiero.h index a1f8a9205..a1f8a9205 100755..100644 --- a/moses/src/RuleTableLoaderHiero.h +++ b/moses/src/RuleTableLoaderHiero.h diff --git a/moses/src/RuleTableLoaderStandard.cpp b/moses/src/RuleTableLoaderStandard.cpp index 122e75c8c..122e75c8c 100755..100644 --- a/moses/src/RuleTableLoaderStandard.cpp +++ b/moses/src/RuleTableLoaderStandard.cpp diff --git a/moses/src/RuleTableLoaderStandard.h b/moses/src/RuleTableLoaderStandard.h index 9477db962..9477db962 100755..100644 --- a/moses/src/RuleTableLoaderStandard.h +++ b/moses/src/RuleTableLoaderStandard.h diff --git a/moses/src/ScoreComponentCollection.cpp b/moses/src/ScoreComponentCollection.cpp index d9638f21c..e393c3a74 100755..100644 --- a/moses/src/ScoreComponentCollection.cpp +++ b/moses/src/ScoreComponentCollection.cpp @@ -137,12 +137,12 @@ float ScoreComponentCollection::GetLInfNorm() const { return m_scores.linfnorm(); } -void ScoreComponentCollection::L1Regularize(float lambda) { - m_scores.l1regularize(lambda); +size_t ScoreComponentCollection::SparseL1Regularize(float lambda) { + return m_scores.sparseL1regularize(lambda); } -void ScoreComponentCollection::L2Regularize(float lambda) { - m_scores.l2regularize(lambda); +void ScoreComponentCollection::SparseL2Regularize(float lambda) { + m_scores.sparseL2regularize(lambda); } void ScoreComponentCollection::Save(ostream& out) const { diff --git a/moses/src/ScoreComponentCollection.h b/moses/src/ScoreComponentCollection.h index f80055613..e170d0cd9 100755..100644 --- a/moses/src/ScoreComponentCollection.h +++ b/moses/src/ScoreComponentCollection.h @@ -234,25 +234,29 @@ public: m_scores[fname] += score; } - void Assign(const ScoreProducer* sp, const std::vector<float>& scores) - { + void Assign(const ScoreProducer* sp, const std::vector<float>& scores) + { IndexPair indexes = GetIndexes(sp); CHECK(scores.size() == indexes.second - indexes.first); for (size_t i = 0; i < scores.size(); ++i) { m_scores[i + indexes.first] = scores[i]; } - } - - //! Special version Assign(ScoreProducer, vector<float>) - //! to add the score from a single ScoreProducer that produces - //! a single value - void Assign(const ScoreProducer* sp, float score) - { + } + + //! Special version Assign(ScoreProducer, vector<float>) + //! to add the score from a single ScoreProducer that produces + //! a single value + void Assign(const ScoreProducer* sp, float score) + { IndexPair indexes = GetIndexes(sp); CHECK(1 == indexes.second - indexes.first); m_scores[indexes.first] = score; - } - + } + + // Assign core weight by index + void Assign(size_t index, float score) { + m_scores[index] = score; + } //For features which have an unbounded number of components void Assign(const ScoreProducer*sp, const std::string name, float score) @@ -367,8 +371,8 @@ public: float GetL1Norm() const; float GetL2Norm() const; float GetLInfNorm() const; - void L1Regularize(float lambda); - void L2Regularize(float lambda); + size_t SparseL1Regularize(float lambda); + void SparseL2Regularize(float lambda); void Save(const std::string& filename) const; void Save(std::ostream&) const; @@ -381,7 +385,7 @@ public: size_t PruneSparseFeatures(size_t threshold) { return m_scores.pruneSparseFeatures(threshold); } size_t PruneZeroWeightFeatures() { return m_scores.pruneZeroWeightFeatures(); } void UpdateConfidenceCounts(ScoreComponentCollection &weightUpdate, bool signedCounts) { m_scores.updateConfidenceCounts(weightUpdate.m_scores, signedCounts); } - void UpdateLearningRates(float decay, ScoreComponentCollection &confidenceCounts, float core_r0, float sparse_r0) { m_scores.updateLearningRates(decay, confidenceCounts.m_scores, core_r0, sparse_r0); } + void UpdateLearningRates(float decay_core, float decay_sparse, ScoreComponentCollection &confidenceCounts, float core_r0, float sparse_r0) { m_scores.updateLearningRates(decay_core, decay_sparse, confidenceCounts.m_scores, core_r0, sparse_r0); } #ifdef MPI_ENABLE public: diff --git a/moses/src/ScoreComponentCollectionTest.cpp b/moses/src/ScoreComponentCollectionTest.cpp index 10e96a017..10e96a017 100755..100644 --- a/moses/src/ScoreComponentCollectionTest.cpp +++ b/moses/src/ScoreComponentCollectionTest.cpp diff --git a/moses/src/ScoreProducer.cpp b/moses/src/ScoreProducer.cpp index 07823125c..07823125c 100755..100644 --- a/moses/src/ScoreProducer.cpp +++ b/moses/src/ScoreProducer.cpp diff --git a/moses/src/ScoreProducer.h b/moses/src/ScoreProducer.h index c88e5efca..c88e5efca 100755..100644 --- a/moses/src/ScoreProducer.h +++ b/moses/src/ScoreProducer.h diff --git a/moses/src/Search.cpp b/moses/src/Search.cpp index aa3bfe59f..aa3bfe59f 100755..100644 --- a/moses/src/Search.cpp +++ b/moses/src/Search.cpp diff --git a/moses/src/Search.h b/moses/src/Search.h index 67a6c69ea..67a6c69ea 100755..100644 --- a/moses/src/Search.h +++ b/moses/src/Search.h diff --git a/moses/src/SearchCubePruning.cpp b/moses/src/SearchCubePruning.cpp index 9fe6aec24..9fe6aec24 100755..100644 --- a/moses/src/SearchCubePruning.cpp +++ b/moses/src/SearchCubePruning.cpp diff --git a/moses/src/SearchCubePruning.h b/moses/src/SearchCubePruning.h index 9f8f73ef5..9f8f73ef5 100755..100644 --- a/moses/src/SearchCubePruning.h +++ b/moses/src/SearchCubePruning.h diff --git a/moses/src/SearchNormal.cpp b/moses/src/SearchNormal.cpp index 9f26270e9..9f26270e9 100755..100644 --- a/moses/src/SearchNormal.cpp +++ b/moses/src/SearchNormal.cpp diff --git a/moses/src/SearchNormal.h b/moses/src/SearchNormal.h index 8b4f904d9..8b4f904d9 100755..100644 --- a/moses/src/SearchNormal.h +++ b/moses/src/SearchNormal.h diff --git a/moses/src/Sentence.cpp b/moses/src/Sentence.cpp index 210e5ccb5..210e5ccb5 100755..100644 --- a/moses/src/Sentence.cpp +++ b/moses/src/Sentence.cpp diff --git a/moses/src/Sentence.h b/moses/src/Sentence.h index b1c25f9dd..b1c25f9dd 100755..100644 --- a/moses/src/Sentence.h +++ b/moses/src/Sentence.h diff --git a/moses/src/SentenceStats.cpp b/moses/src/SentenceStats.cpp index 218fe2187..218fe2187 100755..100644 --- a/moses/src/SentenceStats.cpp +++ b/moses/src/SentenceStats.cpp diff --git a/moses/src/SentenceStats.h b/moses/src/SentenceStats.h index 6a72008ba..6a72008ba 100755..100644 --- a/moses/src/SentenceStats.h +++ b/moses/src/SentenceStats.h diff --git a/moses/src/SourceWordDeletionFeature.cpp b/moses/src/SourceWordDeletionFeature.cpp index 4f247c7e7..4f247c7e7 100755..100644 --- a/moses/src/SourceWordDeletionFeature.cpp +++ b/moses/src/SourceWordDeletionFeature.cpp diff --git a/moses/src/SourceWordDeletionFeature.h b/moses/src/SourceWordDeletionFeature.h index 6d8d21a42..6d8d21a42 100755..100644 --- a/moses/src/SourceWordDeletionFeature.h +++ b/moses/src/SourceWordDeletionFeature.h diff --git a/moses/src/SparsePhraseDictionaryFeature.cpp b/moses/src/SparsePhraseDictionaryFeature.cpp index 8b1a35ee3..8b1a35ee3 100755..100644 --- a/moses/src/SparsePhraseDictionaryFeature.cpp +++ b/moses/src/SparsePhraseDictionaryFeature.cpp diff --git a/moses/src/SparsePhraseDictionaryFeature.h b/moses/src/SparsePhraseDictionaryFeature.h index ad38e7078..ad38e7078 100755..100644 --- a/moses/src/SparsePhraseDictionaryFeature.h +++ b/moses/src/SparsePhraseDictionaryFeature.h diff --git a/moses/src/SquareMatrix.cpp b/moses/src/SquareMatrix.cpp index 276d10053..276d10053 100755..100644 --- a/moses/src/SquareMatrix.cpp +++ b/moses/src/SquareMatrix.cpp diff --git a/moses/src/SquareMatrix.h b/moses/src/SquareMatrix.h index 3d36962d4..3d36962d4 100755..100644 --- a/moses/src/SquareMatrix.h +++ b/moses/src/SquareMatrix.h diff --git a/moses/src/StaticData.cpp b/moses/src/StaticData.cpp index 218be7f2b..6ccb8fb34 100755..100644 --- a/moses/src/StaticData.cpp +++ b/moses/src/StaticData.cpp @@ -1397,6 +1397,7 @@ bool StaticData::LoadPhraseTables() , output , filePath , weight + , currDict , maxTargetPhrase[index] , targetPath, alignmentsFile); @@ -1551,6 +1552,7 @@ bool StaticData::LoadDecodeGraphs() DecodeGraph *decodeGraph; if (m_searchAlgorithm == ChartDecoding) { size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN; + cerr << "max-chart-span: " << maxChartSpans[decodeGraphInd] << endl; decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan); } else { decodeGraph = new DecodeGraph(m_decodeGraphs.size()); diff --git a/moses/src/StaticData.h b/moses/src/StaticData.h index 1b8075e25..1b8075e25 100755..100644 --- a/moses/src/StaticData.h +++ b/moses/src/StaticData.h diff --git a/moses/src/SyntacticLanguageModel.cpp b/moses/src/SyntacticLanguageModel.cpp index c9b3f1e22..c9b3f1e22 100755..100644 --- a/moses/src/SyntacticLanguageModel.cpp +++ b/moses/src/SyntacticLanguageModel.cpp diff --git a/moses/src/SyntacticLanguageModel.h b/moses/src/SyntacticLanguageModel.h index 3497d3dc4..3497d3dc4 100755..100644 --- a/moses/src/SyntacticLanguageModel.h +++ b/moses/src/SyntacticLanguageModel.h diff --git a/moses/src/SyntacticLanguageModelFiles.h b/moses/src/SyntacticLanguageModelFiles.h index c6533c424..c6533c424 100755..100644 --- a/moses/src/SyntacticLanguageModelFiles.h +++ b/moses/src/SyntacticLanguageModelFiles.h diff --git a/moses/src/SyntacticLanguageModelState.h b/moses/src/SyntacticLanguageModelState.h index 0877a59b3..0877a59b3 100755..100644 --- a/moses/src/SyntacticLanguageModelState.h +++ b/moses/src/SyntacticLanguageModelState.h diff --git a/moses/src/TargetBigramFeature.cpp b/moses/src/TargetBigramFeature.cpp index a9ad2216b..a9ad2216b 100755..100644 --- a/moses/src/TargetBigramFeature.cpp +++ b/moses/src/TargetBigramFeature.cpp diff --git a/moses/src/TargetBigramFeature.h b/moses/src/TargetBigramFeature.h index 76b4f6ef7..76b4f6ef7 100755..100644 --- a/moses/src/TargetBigramFeature.h +++ b/moses/src/TargetBigramFeature.h diff --git a/moses/src/TargetBigramFeatureTest.cpp b/moses/src/TargetBigramFeatureTest.cpp index 2ee7243bb..2ee7243bb 100755..100644 --- a/moses/src/TargetBigramFeatureTest.cpp +++ b/moses/src/TargetBigramFeatureTest.cpp diff --git a/moses/src/TargetNgramFeature.cpp b/moses/src/TargetNgramFeature.cpp index 3fefdfba2..3fefdfba2 100755..100644 --- a/moses/src/TargetNgramFeature.cpp +++ b/moses/src/TargetNgramFeature.cpp diff --git a/moses/src/TargetNgramFeature.h b/moses/src/TargetNgramFeature.h index 681e7d6aa..681e7d6aa 100755..100644 --- a/moses/src/TargetNgramFeature.h +++ b/moses/src/TargetNgramFeature.h diff --git a/moses/src/TargetPhrase.cpp b/moses/src/TargetPhrase.cpp index c79b844f8..2783a7604 100755..100644 --- a/moses/src/TargetPhrase.cpp +++ b/moses/src/TargetPhrase.cpp @@ -142,12 +142,7 @@ void TargetPhrase::SetScore(const ScoreProducer* translationScoreProducer, CHECK(weightT.size() == scoreVector.size()); // calc average score if non-best - const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); - std::vector<float> weightsT = system.GetTranslationWeights(); - weightWP = system.GetWeightWordPenalty(); - - //m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); - m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightsT.begin(), 0.0f); + m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); m_scoreBreakdown.PlusEquals(translationScoreProducer, scoreVector); m_scoreBreakdown.PlusEquals(sparseScoreVector); @@ -197,15 +192,10 @@ void TargetPhrase::SetScoreChart(const ScoreProducer* translationScoreProducer, ,const LMList &languageModels ,const WordPenaltyProducer* wpProducer) { - CHECK(weightT.size() == scoreVector.size()); - - const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); - std::vector<float> weightsT = system.GetTranslationWeights(); // calc average score if non-best - //m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); - m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightsT.begin(), 0.0f); + m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); m_scoreBreakdown.PlusEquals(translationScoreProducer, scoreVector); // Replicated from TranslationOptions.cpp @@ -269,11 +259,7 @@ void TargetPhrase::SetWeights(const ScoreProducer* translationScoreProducer, con addition to the usual phrase translation scaling factors) the input weight factor as last element */ - - const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); - std::vector<float> weightsT = system.GetTranslationWeights(); - //m_transScore = m_scoreBreakdown.PartialInnerProduct(translationScoreProducer, weightT); - m_transScore = m_scoreBreakdown.PartialInnerProduct(translationScoreProducer, weightsT); + m_transScore = m_scoreBreakdown.PartialInnerProduct(translationScoreProducer, weightT); } void TargetPhrase::ResetScore() diff --git a/moses/src/TargetPhrase.h b/moses/src/TargetPhrase.h index 377985c5e..377985c5e 100755..100644 --- a/moses/src/TargetPhrase.h +++ b/moses/src/TargetPhrase.h diff --git a/moses/src/TargetPhraseCollection.cpp b/moses/src/TargetPhraseCollection.cpp index 38570f8d5..38570f8d5 100755..100644 --- a/moses/src/TargetPhraseCollection.cpp +++ b/moses/src/TargetPhraseCollection.cpp diff --git a/moses/src/TargetPhraseCollection.h b/moses/src/TargetPhraseCollection.h index f4124f458..f4124f458 100755..100644 --- a/moses/src/TargetPhraseCollection.h +++ b/moses/src/TargetPhraseCollection.h diff --git a/moses/src/TargetWordInsertionFeature.cpp b/moses/src/TargetWordInsertionFeature.cpp index f7d5a5c7e..f7d5a5c7e 100755..100644 --- a/moses/src/TargetWordInsertionFeature.cpp +++ b/moses/src/TargetWordInsertionFeature.cpp diff --git a/moses/src/TargetWordInsertionFeature.h b/moses/src/TargetWordInsertionFeature.h index 0d53582d5..0d53582d5 100755..100644 --- a/moses/src/TargetWordInsertionFeature.h +++ b/moses/src/TargetWordInsertionFeature.h diff --git a/moses/src/ThreadPool.cpp b/moses/src/ThreadPool.cpp index ca9bd2f60..ca9bd2f60 100755..100644 --- a/moses/src/ThreadPool.cpp +++ b/moses/src/ThreadPool.cpp diff --git a/moses/src/ThreadPool.h b/moses/src/ThreadPool.h index a76872027..a76872027 100755..100644 --- a/moses/src/ThreadPool.h +++ b/moses/src/ThreadPool.h diff --git a/moses/src/Timer.cpp b/moses/src/Timer.cpp index bbe1bcabd..bbe1bcabd 100755..100644 --- a/moses/src/Timer.cpp +++ b/moses/src/Timer.cpp diff --git a/moses/src/Timer.h b/moses/src/Timer.h index 1d7c4aacc..1d7c4aacc 100755..100644 --- a/moses/src/Timer.h +++ b/moses/src/Timer.h diff --git a/moses/src/TranslationOption.cpp b/moses/src/TranslationOption.cpp index c823dadb8..c823dadb8 100755..100644 --- a/moses/src/TranslationOption.cpp +++ b/moses/src/TranslationOption.cpp diff --git a/moses/src/TranslationOption.h b/moses/src/TranslationOption.h index fa4065b80..fa4065b80 100755..100644 --- a/moses/src/TranslationOption.h +++ b/moses/src/TranslationOption.h diff --git a/moses/src/TranslationOptionCollection.cpp b/moses/src/TranslationOptionCollection.cpp index d9eefc9d6..d9eefc9d6 100755..100644 --- a/moses/src/TranslationOptionCollection.cpp +++ b/moses/src/TranslationOptionCollection.cpp diff --git a/moses/src/TranslationOptionCollection.h b/moses/src/TranslationOptionCollection.h index e1fa26a15..e1fa26a15 100755..100644 --- a/moses/src/TranslationOptionCollection.h +++ b/moses/src/TranslationOptionCollection.h diff --git a/moses/src/TranslationOptionCollectionConfusionNet.cpp b/moses/src/TranslationOptionCollectionConfusionNet.cpp index c2713fb3a..c2713fb3a 100755..100644 --- a/moses/src/TranslationOptionCollectionConfusionNet.cpp +++ b/moses/src/TranslationOptionCollectionConfusionNet.cpp diff --git a/moses/src/TranslationOptionCollectionConfusionNet.h b/moses/src/TranslationOptionCollectionConfusionNet.h index 66ef953a8..66ef953a8 100755..100644 --- a/moses/src/TranslationOptionCollectionConfusionNet.h +++ b/moses/src/TranslationOptionCollectionConfusionNet.h diff --git a/moses/src/TranslationOptionCollectionText.cpp b/moses/src/TranslationOptionCollectionText.cpp index c3676eaef..c3676eaef 100755..100644 --- a/moses/src/TranslationOptionCollectionText.cpp +++ b/moses/src/TranslationOptionCollectionText.cpp diff --git a/moses/src/TranslationOptionCollectionText.h b/moses/src/TranslationOptionCollectionText.h index bb819441e..bb819441e 100755..100644 --- a/moses/src/TranslationOptionCollectionText.h +++ b/moses/src/TranslationOptionCollectionText.h diff --git a/moses/src/TranslationOptionList.cpp b/moses/src/TranslationOptionList.cpp index 1d99729fe..1d99729fe 100755..100644 --- a/moses/src/TranslationOptionList.cpp +++ b/moses/src/TranslationOptionList.cpp diff --git a/moses/src/TranslationOptionList.h b/moses/src/TranslationOptionList.h index d56578029..d56578029 100755..100644 --- a/moses/src/TranslationOptionList.h +++ b/moses/src/TranslationOptionList.h diff --git a/moses/src/TranslationSystem.cpp b/moses/src/TranslationSystem.cpp index 722ea864e..e017ba241 100755..100644 --- a/moses/src/TranslationSystem.cpp +++ b/moses/src/TranslationSystem.cpp @@ -175,8 +175,8 @@ namespace Moses { return StaticData::Instance().GetWeight(m_distortionScoreProducer); } - std::vector<float> TranslationSystem::GetTranslationWeights() const { - std::vector<float> weights = StaticData::Instance().GetWeights(GetTranslationScoreProducer()); + std::vector<float> TranslationSystem::GetTranslationWeights(size_t index) const { + std::vector<float> weights = StaticData::Instance().GetWeights(GetTranslationScoreProducer(index)); //VERBOSE(1, "Read weightT from translation sytem.. "); for (size_t i = 0; i < weights.size(); ++i) //VERBOSE(1, weights[i] << " "); diff --git a/moses/src/TranslationSystem.h b/moses/src/TranslationSystem.h index 615fc7d8a..6f66fd25a 100755..100644 --- a/moses/src/TranslationSystem.h +++ b/moses/src/TranslationSystem.h @@ -83,12 +83,12 @@ class TranslationSystem { const UnknownWordPenaltyProducer *GetUnknownWordPenaltyProducer() const { return m_unknownWpProducer; } const DistortionScoreProducer* GetDistortionProducer() const {return m_distortionScoreProducer;} - const PhraseDictionaryFeature *GetTranslationScoreProducer() const { return GetPhraseDictionaries()[0]; } + const PhraseDictionaryFeature *GetTranslationScoreProducer(size_t index) const { return GetPhraseDictionaries()[index]; } float GetWeightWordPenalty() const; float GetWeightUnknownWordPenalty() const; float GetWeightDistortion() const; - std::vector<float> GetTranslationWeights() const; + std::vector<float> GetTranslationWeights(size_t index) const; //sentence (and thread) specific initialisationn and cleanup void InitializeBeforeSentenceProcessing(const InputType& source) const; diff --git a/moses/src/TreeInput.cpp b/moses/src/TreeInput.cpp index d6e0f1e78..d6e0f1e78 100755..100644 --- a/moses/src/TreeInput.cpp +++ b/moses/src/TreeInput.cpp diff --git a/moses/src/TreeInput.h b/moses/src/TreeInput.h index a6242d97e..a6242d97e 100755..100644 --- a/moses/src/TreeInput.h +++ b/moses/src/TreeInput.h diff --git a/moses/src/TrellisPath.cpp b/moses/src/TrellisPath.cpp index c73575b2c..c73575b2c 100755..100644 --- a/moses/src/TrellisPath.cpp +++ b/moses/src/TrellisPath.cpp diff --git a/moses/src/TrellisPath.h b/moses/src/TrellisPath.h index c1b347de9..c1b347de9 100755..100644 --- a/moses/src/TrellisPath.h +++ b/moses/src/TrellisPath.h diff --git a/moses/src/TrellisPathCollection.cpp b/moses/src/TrellisPathCollection.cpp index 89d429f4b..89d429f4b 100755..100644 --- a/moses/src/TrellisPathCollection.cpp +++ b/moses/src/TrellisPathCollection.cpp diff --git a/moses/src/TrellisPathCollection.h b/moses/src/TrellisPathCollection.h index 82e771d78..82e771d78 100755..100644 --- a/moses/src/TrellisPathCollection.h +++ b/moses/src/TrellisPathCollection.h diff --git a/moses/src/TrellisPathList.h b/moses/src/TrellisPathList.h index 073efb9b8..073efb9b8 100755..100644 --- a/moses/src/TrellisPathList.h +++ b/moses/src/TrellisPathList.h diff --git a/moses/src/TypeDef.h b/moses/src/TypeDef.h index bb439f5c9..bb439f5c9 100755..100644 --- a/moses/src/TypeDef.h +++ b/moses/src/TypeDef.h diff --git a/moses/src/UniqueObject.h b/moses/src/UniqueObject.h index 789c5e1b3..789c5e1b3 100755..100644 --- a/moses/src/UniqueObject.h +++ b/moses/src/UniqueObject.h diff --git a/moses/src/UserMessage.cpp b/moses/src/UserMessage.cpp index ad89d80a7..ad89d80a7 100755..100644 --- a/moses/src/UserMessage.cpp +++ b/moses/src/UserMessage.cpp diff --git a/moses/src/UserMessage.h b/moses/src/UserMessage.h index aac38cae4..aac38cae4 100755..100644 --- a/moses/src/UserMessage.h +++ b/moses/src/UserMessage.h diff --git a/moses/src/Util.cpp b/moses/src/Util.cpp index be02fb27b..be02fb27b 100755..100644 --- a/moses/src/Util.cpp +++ b/moses/src/Util.cpp diff --git a/moses/src/Util.h b/moses/src/Util.h index 1b0638ec2..1b0638ec2 100755..100644 --- a/moses/src/Util.h +++ b/moses/src/Util.h diff --git a/moses/src/Word.cpp b/moses/src/Word.cpp index 1ff5df496..1ff5df496 100755..100644 --- a/moses/src/Word.cpp +++ b/moses/src/Word.cpp diff --git a/moses/src/Word.h b/moses/src/Word.h index 7dd395030..7dd395030 100755..100644 --- a/moses/src/Word.h +++ b/moses/src/Word.h diff --git a/moses/src/WordLattice.cpp b/moses/src/WordLattice.cpp index f3344dc67..f3344dc67 100755..100644 --- a/moses/src/WordLattice.cpp +++ b/moses/src/WordLattice.cpp diff --git a/moses/src/WordLattice.h b/moses/src/WordLattice.h index 21df48054..21df48054 100755..100644 --- a/moses/src/WordLattice.h +++ b/moses/src/WordLattice.h diff --git a/moses/src/WordTranslationFeature.cpp b/moses/src/WordTranslationFeature.cpp index c0b3e0025..c0b3e0025 100755..100644 --- a/moses/src/WordTranslationFeature.cpp +++ b/moses/src/WordTranslationFeature.cpp diff --git a/moses/src/WordTranslationFeature.h b/moses/src/WordTranslationFeature.h index 39be24d41..39be24d41 100755..100644 --- a/moses/src/WordTranslationFeature.h +++ b/moses/src/WordTranslationFeature.h diff --git a/moses/src/WordsBitmap.cpp b/moses/src/WordsBitmap.cpp index b3203145f..b3203145f 100755..100644 --- a/moses/src/WordsBitmap.cpp +++ b/moses/src/WordsBitmap.cpp diff --git a/moses/src/WordsBitmap.h b/moses/src/WordsBitmap.h index 370120571..370120571 100755..100644 --- a/moses/src/WordsBitmap.h +++ b/moses/src/WordsBitmap.h diff --git a/moses/src/WordsRange.cpp b/moses/src/WordsRange.cpp index f37a738c0..f37a738c0 100755..100644 --- a/moses/src/WordsRange.cpp +++ b/moses/src/WordsRange.cpp diff --git a/moses/src/WordsRange.h b/moses/src/WordsRange.h index 7191d259e..7191d259e 100755..100644 --- a/moses/src/WordsRange.h +++ b/moses/src/WordsRange.h diff --git a/moses/src/XmlOption.cpp b/moses/src/XmlOption.cpp index f5eee83f9..f5eee83f9 100755..100644 --- a/moses/src/XmlOption.cpp +++ b/moses/src/XmlOption.cpp diff --git a/moses/src/XmlOption.h b/moses/src/XmlOption.h index 45989c841..45989c841 100755..100644 --- a/moses/src/XmlOption.h +++ b/moses/src/XmlOption.h diff --git a/moses/src/gzfilebuf.h b/moses/src/gzfilebuf.h index d59d3e1b7..d59d3e1b7 100755..100644 --- a/moses/src/gzfilebuf.h +++ b/moses/src/gzfilebuf.h diff --git a/moses/src/hash.cpp b/moses/src/hash.cpp index 9afac463d..9afac463d 100755..100644 --- a/moses/src/hash.cpp +++ b/moses/src/hash.cpp diff --git a/moses/src/hash.h b/moses/src/hash.h index 227f3b90a..227f3b90a 100755..100644 --- a/moses/src/hash.h +++ b/moses/src/hash.h diff --git a/moses/src/hypergraph.proto b/moses/src/hypergraph.proto index e292eacca..e292eacca 100755..100644 --- a/moses/src/hypergraph.proto +++ b/moses/src/hypergraph.proto diff --git a/moses/src/rule.proto b/moses/src/rule.proto index 5f8084636..5f8084636 100755..100644 --- a/moses/src/rule.proto +++ b/moses/src/rule.proto |