diff options
33 files changed, 310 insertions, 172 deletions
diff --git a/moses/ChartParser.cpp b/moses/ChartParser.cpp index e0c84e199..60899c02d 100644 --- a/moses/ChartParser.cpp +++ b/moses/ChartParser.cpp @@ -46,21 +46,21 @@ ChartParserUnknown::~ChartParserUnknown() RemoveAllInColl(m_unksrcs); } -AllOptions::ptr const& +AllOptions::ptr const& ChartParserUnknown:: -options() const -{ - return m_ttask.lock()->options(); -} +options() const +{ + return m_ttask.lock()->options(); +} -void +void ChartParserUnknown:: Process(const Word &sourceWord, const Range &range, ChartParserCallback &to) { // unknown word, add as trans opt const StaticData &staticData = StaticData::Instance(); - const UnknownWordPenaltyProducer &unknownWordPenaltyProducer - = UnknownWordPenaltyProducer::Instance(); + const UnknownWordPenaltyProducer &unknownWordPenaltyProducer + = UnknownWordPenaltyProducer::Instance(); size_t isDigit = 0; if (options()->unk.drop) { @@ -100,8 +100,8 @@ Process(const Word &sourceWord, const Range &range, ChartParserCallback &to) //const Word &sourceLHS = staticData.GetInputDefaultNonTerminal(); Word *targetLHS = new Word(true); - targetLHS->CreateFromString(Output, options()->output.factor_order, - targetLHSStr, true); + targetLHS->CreateFromString(Output, options()->output.factor_order, + targetLHSStr, true); UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS"); // add to dictionary @@ -141,8 +141,8 @@ Process(const Word &sourceWord, const Range &range, ChartParserCallback &to) //float prob = iterLHS->second; Word *targetLHS = new Word(true); - targetLHS->CreateFromString(Output, staticData.options()->output.factor_order, - targetLHSStr, true); + targetLHS->CreateFromString(Output, staticData.options()->output.factor_order, + targetLHSStr, true); UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS"); targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore); @@ -299,12 +299,12 @@ long ChartParser::GetTranslationId() const } -AllOptions::ptr const& +AllOptions::ptr const& ChartParser:: -options() const -{ - return m_ttask.lock()->options(); -} +options() const +{ + return m_ttask.lock()->options(); +} } // namespace Moses diff --git a/moses/ChartParser.h b/moses/ChartParser.h index 3229e296b..b9d756abb 100644 --- a/moses/ChartParser.h +++ b/moses/ChartParser.h @@ -57,7 +57,7 @@ public: private: std::vector<Phrase*> m_unksrcs; std::list<TargetPhraseCollection::shared_ptr> m_cacheTargetPhraseCollection; - AllOptions::ptr const& options() const; + AllOptions::ptr const& options() const; }; class ChartParser @@ -79,7 +79,7 @@ public: return m_unknown.GetUnknownSources(); } - AllOptions::ptr const& options() const; + AllOptions::ptr const& options() const; private: ChartParserUnknown m_unknown; diff --git a/moses/ConfusionNet.h b/moses/ConfusionNet.h index f16493b23..4d15f37b6 100644 --- a/moses/ConfusionNet.h +++ b/moses/ConfusionNet.h @@ -46,8 +46,8 @@ public: const Column& GetColumn(size_t i) const { UTIL_THROW_IF2(i >= data.size(), - "Out of bounds. Trying to access " << i - << " when vector only contains " << data.size()); + "Out of bounds. Trying to access " << i + << " when vector only contains " << data.size()); return data[i]; } const Column& operator[](size_t i) const { diff --git a/moses/FF/CountNonTerms.h b/moses/FF/CountNonTerms.h index dbe8158a2..d2e697173 100644 --- a/moses/FF/CountNonTerms.h +++ b/moses/FF/CountNonTerms.h @@ -7,8 +7,6 @@ namespace Moses class CountNonTerms : public StatelessFeatureFunction { - Word m_input_default_nonterminal; - Word m_output_default_nonterminal; public: CountNonTerms(const std::string &line); bool IsUseable(const FactorMask &mask) const { diff --git a/moses/FF/PhraseOrientationFeature.cpp b/moses/FF/PhraseOrientationFeature.cpp index 9cac75ccb..332be1d2a 100644 --- a/moses/FF/PhraseOrientationFeature.cpp +++ b/moses/FF/PhraseOrientationFeature.cpp @@ -15,7 +15,6 @@ #include "moses/Hypothesis.h" #include "moses/ChartHypothesis.h" #include "moses/ChartManager.h" -#include "phrase-extract/extract-ghkm/Alignment.h" #include <boost/shared_ptr.hpp> @@ -23,15 +22,59 @@ namespace Moses { size_t PhraseOrientationFeatureState::hash() const { - UTIL_THROW2("TODO:Haven't figure this out yet"); + if (!m_distinguishStates) { + return 0; + } + + size_t ret = 0; + + if (m_leftBoundaryIsSet) { + HashCombineLeftBoundaryRecursive(ret, *this, m_useSparseNT); + } + if (m_rightBoundaryIsSet) { + boost::hash_combine(ret, 42); + HashCombineRightBoundaryRecursive(ret, *this, m_useSparseNT); + } + + return 0; } bool PhraseOrientationFeatureState::operator==(const FFState& other) const { - UTIL_THROW2("TODO:Haven't figure this out yet"); + if (!m_distinguishStates) { + return true; + } + + const PhraseOrientationFeatureState &otherState = static_cast<const PhraseOrientationFeatureState&>(other); + + if (!m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet && + !m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) { + return true; + } + if (m_leftBoundaryIsSet != otherState.m_leftBoundaryIsSet) { + return false; + } + if (m_rightBoundaryIsSet != otherState.m_rightBoundaryIsSet) { + return false; + } + + if (m_leftBoundaryIsSet) { + int compareLeft = CompareLeftBoundaryRecursive(*this, otherState, m_useSparseNT); + if (compareLeft != 0) { + return false; + } + } + if (m_rightBoundaryIsSet) { + int compareRight = CompareRightBoundaryRecursive(*this, otherState, m_useSparseNT); + if (compareRight != 0) { + return false; + } + } + + return true; } -//////////////////////////////////////////////////////////////////////////////// + const std::string PhraseOrientationFeature::MORIENT("M"); const std::string PhraseOrientationFeature::SORIENT("S"); const std::string PhraseOrientationFeature::DORIENT("D"); @@ -144,7 +187,7 @@ void PhraseOrientationFeature::EvaluateInIsolation(const Phrase &source, if (targetPhrase.GetAlignNonTerm().GetSize() != 0) { // Initialize phrase orientation scoring object - MosesTraining::Syntax::GHKM::PhraseOrientation phraseOrientation(source.GetSize(), targetPhrase.GetSize(), + MosesTraining::PhraseOrientation phraseOrientation(source.GetSize(), targetPhrase.GetSize(), targetPhrase.GetAlignTerm(), targetPhrase.GetAlignNonTerm()); PhraseOrientationFeature::ReoClassData* reoClassData = new PhraseOrientationFeature::ReoClassData(); @@ -160,7 +203,7 @@ void PhraseOrientationFeature::EvaluateInIsolation(const Phrase &source, // LEFT-TO-RIGHT DIRECTION - MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS l2rOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,MosesTraining::Syntax::GHKM::PhraseOrientation::REO_DIR_L2R); + MosesTraining::PhraseOrientation::REO_CLASS l2rOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,MosesTraining::PhraseOrientation::REO_DIR_L2R); if ( ((targetIndex == 0) || !phraseOrientation.TargetSpanIsAligned(0,targetIndex)) // boundary non-terminal in rule-initial position (left boundary) && (targetPhraseLHS != m_glueTargetLHS) ) { // and not glue rule @@ -180,7 +223,7 @@ void PhraseOrientationFeature::EvaluateInIsolation(const Phrase &source, if (reoClassData->firstNonTerminalPreviousSourceSpanIsAligned && reoClassData->firstNonTerminalFollowingSourceSpanIsAligned) { // discontinuous - l2rOrientation = MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DLEFT; + l2rOrientation = MosesTraining::PhraseOrientation::REO_CLASS_DLEFT; } else { reoClassData->firstNonTerminalIsBoundary = true; } @@ -190,7 +233,7 @@ void PhraseOrientationFeature::EvaluateInIsolation(const Phrase &source, // RIGHT-TO-LEFT DIRECTION - MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS r2lOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,MosesTraining::Syntax::GHKM::PhraseOrientation::REO_DIR_R2L); + MosesTraining::PhraseOrientation::REO_CLASS r2lOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,MosesTraining::PhraseOrientation::REO_DIR_R2L); if ( ((targetIndex == targetPhrase.GetSize()-1) || !phraseOrientation.TargetSpanIsAligned(targetIndex,targetPhrase.GetSize()-1)) // boundary non-terminal in rule-final position (right boundary) && (targetPhraseLHS != m_glueTargetLHS) ) { // and not glue rule @@ -210,7 +253,7 @@ void PhraseOrientationFeature::EvaluateInIsolation(const Phrase &source, if (reoClassData->lastNonTerminalPreviousSourceSpanIsAligned && reoClassData->lastNonTerminalFollowingSourceSpanIsAligned) { // discontinuous - r2lOrientation = MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DLEFT; + r2lOrientation = MosesTraining::PhraseOrientation::REO_CLASS_DLEFT; } else { reoClassData->lastNonTerminalIsBoundary = true; } @@ -345,25 +388,25 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied( // LEFT-TO-RIGHT DIRECTION - MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS l2rOrientation = reoClassData->nonTerminalReoClassL2R[nNT]; + MosesTraining::PhraseOrientation::REO_CLASS l2rOrientation = reoClassData->nonTerminalReoClassL2R[nNT]; IFFEATUREVERBOSE(2) { FEATUREVERBOSE(2, "l2rOrientation "); switch (l2rOrientation) { - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_LEFT: + case MosesTraining::PhraseOrientation::REO_CLASS_LEFT: FEATUREVERBOSE2(2, "mono" << std::endl); break; - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_RIGHT: + case MosesTraining::PhraseOrientation::REO_CLASS_RIGHT: FEATUREVERBOSE2(2, "swap" << std::endl); break; - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DLEFT: + case MosesTraining::PhraseOrientation::REO_CLASS_DLEFT: FEATUREVERBOSE2(2, "dleft" << std::endl); break; - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DRIGHT: + case MosesTraining::PhraseOrientation::REO_CLASS_DRIGHT: FEATUREVERBOSE2(2, "dright" << std::endl); break; - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_UNKNOWN: - // modelType == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_MSLR + case MosesTraining::PhraseOrientation::REO_CLASS_UNKNOWN: + // modelType == MosesTraining::PhraseOrientation::REO_MSLR FEATUREVERBOSE2(2, "unknown->dleft" << std::endl); break; default: @@ -406,23 +449,23 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied( } else { - if ( l2rOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_LEFT ) { + if ( l2rOrientation == MosesTraining::PhraseOrientation::REO_CLASS_LEFT ) { newScores[0] += TransformScore(orientationPhraseProperty->GetLeftToRightProbabilityMono()); // if sub-derivation has left-boundary non-terminal: // add recursive actual score of boundary non-terminal from subderivation LeftBoundaryL2RScoreRecursive(featureID, prevState, 0x1, newScores, accumulator); - } else if ( l2rOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_RIGHT ) { + } else if ( l2rOrientation == MosesTraining::PhraseOrientation::REO_CLASS_RIGHT ) { newScores[1] += TransformScore(orientationPhraseProperty->GetLeftToRightProbabilitySwap()); // if sub-derivation has left-boundary non-terminal: // add recursive actual score of boundary non-terminal from subderivation LeftBoundaryL2RScoreRecursive(featureID, prevState, 0x2, newScores, accumulator); - } else if ( ( l2rOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DLEFT ) || - ( l2rOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DRIGHT ) || - ( l2rOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_UNKNOWN ) ) { + } else if ( ( l2rOrientation == MosesTraining::PhraseOrientation::REO_CLASS_DLEFT ) || + ( l2rOrientation == MosesTraining::PhraseOrientation::REO_CLASS_DRIGHT ) || + ( l2rOrientation == MosesTraining::PhraseOrientation::REO_CLASS_UNKNOWN ) ) { newScores[2] += TransformScore(orientationPhraseProperty->GetLeftToRightProbabilityDiscontinuous()); // if sub-derivation has left-boundary non-terminal: @@ -447,25 +490,25 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied( // RIGHT-TO-LEFT DIRECTION - MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS r2lOrientation = reoClassData->nonTerminalReoClassR2L[nNT]; + MosesTraining::PhraseOrientation::REO_CLASS r2lOrientation = reoClassData->nonTerminalReoClassR2L[nNT]; IFFEATUREVERBOSE(2) { FEATUREVERBOSE(2, "r2lOrientation "); switch (r2lOrientation) { - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_LEFT: + case MosesTraining::PhraseOrientation::REO_CLASS_LEFT: FEATUREVERBOSE2(2, "mono" << std::endl); break; - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_RIGHT: + case MosesTraining::PhraseOrientation::REO_CLASS_RIGHT: FEATUREVERBOSE2(2, "swap" << std::endl); break; - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DLEFT: + case MosesTraining::PhraseOrientation::REO_CLASS_DLEFT: FEATUREVERBOSE2(2, "dleft" << std::endl); break; - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DRIGHT: + case MosesTraining::PhraseOrientation::REO_CLASS_DRIGHT: FEATUREVERBOSE2(2, "dright" << std::endl); break; - case MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_UNKNOWN: - // modelType == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_MSLR + case MosesTraining::PhraseOrientation::REO_CLASS_UNKNOWN: + // modelType == MosesTraining::PhraseOrientation::REO_MSLR FEATUREVERBOSE2(2, "unknown->dleft" << std::endl); break; default: @@ -508,23 +551,23 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied( } else { - if ( r2lOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_LEFT ) { + if ( r2lOrientation == MosesTraining::PhraseOrientation::REO_CLASS_LEFT ) { newScores[m_offsetR2LScores+0] += TransformScore(orientationPhraseProperty->GetRightToLeftProbabilityMono()); // if sub-derivation has right-boundary non-terminal: // add recursive actual score of boundary non-terminal from subderivation RightBoundaryR2LScoreRecursive(featureID, prevState, 0x1, newScores, accumulator); - } else if ( r2lOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_RIGHT ) { + } else if ( r2lOrientation == MosesTraining::PhraseOrientation::REO_CLASS_RIGHT ) { newScores[m_offsetR2LScores+1] += TransformScore(orientationPhraseProperty->GetRightToLeftProbabilitySwap()); // if sub-derivation has right-boundary non-terminal: // add recursive actual score of boundary non-terminal from subderivation RightBoundaryR2LScoreRecursive(featureID, prevState, 0x2, newScores, accumulator); - } else if ( ( r2lOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DLEFT ) || - ( r2lOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DRIGHT ) || - ( r2lOrientation == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_UNKNOWN ) ) { + } else if ( ( r2lOrientation == MosesTraining::PhraseOrientation::REO_CLASS_DLEFT ) || + ( r2lOrientation == MosesTraining::PhraseOrientation::REO_CLASS_DRIGHT ) || + ( r2lOrientation == MosesTraining::PhraseOrientation::REO_CLASS_UNKNOWN ) ) { newScores[m_offsetR2LScores+2] += TransformScore(orientationPhraseProperty->GetRightToLeftProbabilityDiscontinuous()); // if sub-derivation has right-boundary non-terminal: @@ -872,17 +915,17 @@ void PhraseOrientationFeature::SparseNonTerminalR2LScore(const Factor* nonTermin } -const std::string* PhraseOrientationFeature::ToString(const MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS o) const +const std::string* PhraseOrientationFeature::ToString(const MosesTraining::PhraseOrientation::REO_CLASS o) const { - if ( o == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_LEFT ) { + if ( o == MosesTraining::PhraseOrientation::REO_CLASS_LEFT ) { return &MORIENT; - } else if ( o == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_RIGHT ) { + } else if ( o == MosesTraining::PhraseOrientation::REO_CLASS_RIGHT ) { return &SORIENT; - } else if ( ( o == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DLEFT ) || - ( o == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_DRIGHT ) || - ( o == MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS_UNKNOWN ) ) { + } else if ( ( o == MosesTraining::PhraseOrientation::REO_CLASS_DLEFT ) || + ( o == MosesTraining::PhraseOrientation::REO_CLASS_DRIGHT ) || + ( o == MosesTraining::PhraseOrientation::REO_CLASS_UNKNOWN ) ) { return &DORIENT; } else { diff --git a/moses/FF/PhraseOrientationFeature.h b/moses/FF/PhraseOrientationFeature.h index e24a37810..c8bb8ceba 100644 --- a/moses/FF/PhraseOrientationFeature.h +++ b/moses/FF/PhraseOrientationFeature.h @@ -16,7 +16,7 @@ #include "StatefulFeatureFunction.h" #include "FFState.h" #include "moses/Factor.h" -#include "phrase-extract/extract-ghkm/PhraseOrientation.h" +#include "phrase-extract/PhraseOrientation.h" #include "moses/PP/OrientationPhraseProperty.h" #include <boost/unordered_set.hpp> @@ -226,6 +226,54 @@ protected: return CompareRightBoundaryRecursive(*prevState, *otherPrevState, useSparseNT); }; + + static void HashCombineLeftBoundaryRecursive(size_t &hash, const PhraseOrientationFeatureState& state, bool useSparseNT) { + if (useSparseNT) { + boost::hash_combine(hash, state.m_leftBoundaryNonTerminalSymbol); + } + // boost::hash_combine(hash, state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex); + // boost::hash_combine(hash, state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations); + + for (size_t i=0; i<state.m_leftBoundaryNonTerminalL2RScores.size(); ++i) { + if (state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations[i]) { + boost::hash_combine(hash, state.m_leftBoundaryNonTerminalL2RScores[i]); + } else { + boost::hash_combine(hash, 0); + } + } + + if (!state.m_leftBoundaryRecursionGuard) { + const PhraseOrientationFeatureState *prevState = state.m_leftBoundaryPrevState; + if (prevState->m_leftBoundaryIsSet) { + HashCombineLeftBoundaryRecursive(hash, *prevState, useSparseNT); + } + } + }; + + static void HashCombineRightBoundaryRecursive(size_t &hash, const PhraseOrientationFeatureState& state, bool useSparseNT) { + if (useSparseNT) { + boost::hash_combine(hash, state.m_rightBoundaryNonTerminalSymbol); + } + // boost::hash_combine(hash, state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex); + // boost::hash_combine(hash, state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations); + + for (size_t i=0; i<state.m_rightBoundaryNonTerminalR2LScores.size(); ++i) { + if (state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations[i]) { + boost::hash_combine(hash, state.m_rightBoundaryNonTerminalR2LScores[i]); + } else { + boost::hash_combine(hash, 0); + } + } + + if (!state.m_rightBoundaryRecursionGuard) { + const PhraseOrientationFeatureState *prevState = state.m_rightBoundaryPrevState; + if (prevState->m_rightBoundaryIsSet) { + HashCombineRightBoundaryRecursive(hash, *prevState, useSparseNT); + } + } + }; + + template<std::size_t N> static bool Smaller(const std::bitset<N>& x, const std::bitset<N>& y) { for (size_t i=0; i<N; ++i) { if (x[i] ^ y[i]) @@ -264,8 +312,8 @@ public: struct ReoClassData { public: - std::vector<MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS> nonTerminalReoClassL2R; - std::vector<MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS> nonTerminalReoClassR2L; + std::vector<MosesTraining::PhraseOrientation::REO_CLASS> nonTerminalReoClassL2R; + std::vector<MosesTraining::PhraseOrientation::REO_CLASS> nonTerminalReoClassR2L; bool firstNonTerminalIsBoundary; bool firstNonTerminalPreviousSourceSpanIsAligned; bool firstNonTerminalFollowingSourceSpanIsAligned; @@ -351,7 +399,7 @@ protected: ScoreComponentCollection* scoreBreakdown, const std::string* o) const; - const std::string* ToString(const MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS o) const; + const std::string* ToString(const MosesTraining::PhraseOrientation::REO_CLASS o) const; static const std::string MORIENT; static const std::string SORIENT; diff --git a/moses/FF/SoftSourceSyntacticConstraintsFeature.cpp b/moses/FF/SoftSourceSyntacticConstraintsFeature.cpp index dc58350b3..7e6121879 100644 --- a/moses/FF/SoftSourceSyntacticConstraintsFeature.cpp +++ b/moses/FF/SoftSourceSyntacticConstraintsFeature.cpp @@ -99,7 +99,6 @@ void SoftSourceSyntacticConstraintsFeature::Load(AllOptions::ptr const& opts) if (!m_targetSourceLHSJointCountFile.empty()) { LoadTargetSourceLeftHandSideJointCountFile(); } - // m_output_default_nonterminal = opts->syntax.output_default_non_terminal; } void SoftSourceSyntacticConstraintsFeature::LoadSourceLabelSet() diff --git a/moses/FF/SoftSourceSyntacticConstraintsFeature.h b/moses/FF/SoftSourceSyntacticConstraintsFeature.h index 459e38521..e62189734 100644 --- a/moses/FF/SoftSourceSyntacticConstraintsFeature.h +++ b/moses/FF/SoftSourceSyntacticConstraintsFeature.h @@ -101,7 +101,6 @@ protected: std::pair<float,float> GetLabelPairProbabilities(const Factor* target, const size_t source) const; - // Word m_output_default_nonterminal; }; diff --git a/moses/FF/SourceGHKMTreeInputMatchFeature.h b/moses/FF/SourceGHKMTreeInputMatchFeature.h index 6c1cc8c59..403dca716 100644 --- a/moses/FF/SourceGHKMTreeInputMatchFeature.h +++ b/moses/FF/SourceGHKMTreeInputMatchFeature.h @@ -9,7 +9,6 @@ namespace Moses // assumes that source-side syntax labels are stored in the target non-terminal field of the rules class SourceGHKMTreeInputMatchFeature : public StatelessFeatureFunction { - // Word m_output_default_nonterminal; public: SourceGHKMTreeInputMatchFeature(const std::string &line); diff --git a/moses/ForestInput.cpp b/moses/ForestInput.cpp index 108012518..68fec17ed 100644 --- a/moses/ForestInput.cpp +++ b/moses/ForestInput.cpp @@ -68,7 +68,7 @@ Read(std::istream &in) assert(topVertices.size() >= 1); } - + const std::vector<FactorType>& factorOrder = m_options->input.factor_order; // Add <s> vertex. @@ -175,7 +175,7 @@ void ForestInput::FindTopVertices(Forest &forest, std::back_inserter(topVertices)); } -void +void ForestInput:: ParseHyperedgeLine(const std::string &line) { diff --git a/moses/HypothesisStackNormal.cpp b/moses/HypothesisStackNormal.cpp index 76ddd3e84..d2ccdb52f 100644 --- a/moses/HypothesisStackNormal.cpp +++ b/moses/HypothesisStackNormal.cpp @@ -74,13 +74,12 @@ pair<HypothesisStackNormal::iterator, bool> HypothesisStackNormal::Add(Hypothesi // prune only if stack is twice as big as needed (lazy pruning) size_t toleratedSize = 2*m_maxHypoStackSize-1; // add in room for stack diversity - if (m_minHypoStackDiversity) - { - // so what happens if maxdistortion is negative? - toleratedSize += m_minHypoStackDiversity - << m_manager.options()->reordering.max_distortion; - } - + if (m_minHypoStackDiversity) { + // so what happens if maxdistortion is negative? + toleratedSize += m_minHypoStackDiversity + << m_manager.options()->reordering.max_distortion; + } + if (m_hypos.size() > toleratedSize) { PruneToSize(m_maxHypoStackSize); } else { diff --git a/moses/InputType.h b/moses/InputType.h index 1e40c158e..80080a3f2 100644 --- a/moses/InputType.h +++ b/moses/InputType.h @@ -190,10 +190,10 @@ public: //! populate this InputType with data from in stream virtual int - Read(std::istream& in) = 0; - // , - // std::vector<FactorType> const& factorOrder, - // AllOptions const& opts) =0; + Read(std::istream& in) = 0; + // , + // std::vector<FactorType> const& factorOrder, + // AllOptions const& opts) =0; //! Output debugging info to stream out virtual void Print(std::ostream&) const =0; diff --git a/moses/Jamfile b/moses/Jamfile index 9349d98f9..f7e05b85b 100644 --- a/moses/Jamfile +++ b/moses/Jamfile @@ -105,7 +105,7 @@ lib moses : TranslationModel/RuleTable/*.cpp TranslationModel/Scope3Parser/*.cpp TranslationModel/CYKPlusParser/*.cpp - ../phrase-extract/extract-ghkm/PhraseOrientation.cpp + ../phrase-extract/PhraseOrientation.cpp FF/*.cpp FF/bilingual-lm/*.cpp FF/OSM-Feature/*.cpp diff --git a/moses/MockHypothesis.cpp b/moses/MockHypothesis.cpp index de74dae5e..438484083 100644 --- a/moses/MockHypothesis.cpp +++ b/moses/MockHypothesis.cpp @@ -64,8 +64,8 @@ MockHypothesisGuard m_targetPhrases.back().CreateFromString(Input, factors, *ti, NULL); m_toptions.push_back(new TranslationOption (range,m_targetPhrases.back())); - m_hypothesis = new Hypothesis(*prevHypo, *m_toptions.back(), newBitmap, - m_manager->GetNextHypoId()); + m_hypothesis = new Hypothesis(*prevHypo, *m_toptions.back(), newBitmap, + m_manager->GetNextHypoId()); } diff --git a/moses/Sentence.cpp b/moses/Sentence.cpp index 4faeb3889..c9c69b648 100644 --- a/moses/Sentence.cpp +++ b/moses/Sentence.cpp @@ -151,11 +151,11 @@ aux_interpret_xml(std::string& line, std::vector<size_t> & xmlWalls, // parse XML markup in translation line using namespace std; if (m_options->input.xml_policy != XmlPassThrough) { - bool OK = ProcessAndStripXMLTags(*m_options, line, - m_xmlOptions, + bool OK = ProcessAndStripXMLTags(*m_options, line, + m_xmlOptions, m_reorderingConstraint, xmlWalls, placeholders); - UTIL_THROW_IF2(!OK, "Unable to parse XML in line: " << line); + UTIL_THROW_IF2(!OK, "Unable to parse XML in line: " << line); } } @@ -170,7 +170,7 @@ init(string line) if (m_options->input.continue_partial_translation) aux_init_partial_translation(line); - + line = Trim(line); aux_interpret_sgml_markup(line); // for "<seg id=..." markup aux_interpret_dlt(line); // some poorly documented cache-based stuff @@ -365,7 +365,7 @@ CreateFromString(vector<FactorType> const& FOrder, string const& phraseString) } Sentence:: -Sentence(AllOptions::ptr const& opts, size_t const transId, string stext) +Sentence(AllOptions::ptr const& opts, size_t const transId, string stext) : InputType(opts, transId) { init(stext); diff --git a/moses/Sentence.h b/moses/Sentence.h index 219307218..c2a846762 100644 --- a/moses/Sentence.h +++ b/moses/Sentence.h @@ -115,7 +115,7 @@ public: } - void init(std::string line); + void init(std::string line); std::vector<std::map<std::string,std::string> > const& GetDltMeta() const { diff --git a/moses/Syntax/F2S/GlueRuleSynthesizer.h b/moses/Syntax/F2S/GlueRuleSynthesizer.h index b4128c5d7..a07c111b0 100644 --- a/moses/Syntax/F2S/GlueRuleSynthesizer.h +++ b/moses/Syntax/F2S/GlueRuleSynthesizer.h @@ -9,7 +9,7 @@ namespace Moses { - class AllOptions; +class AllOptions; namespace Syntax { namespace F2S @@ -20,8 +20,8 @@ class GlueRuleSynthesizer : public HyperTreeCreator Word m_input_default_nonterminal; Word m_output_default_nonterminal; public: - GlueRuleSynthesizer(Moses::AllOptions const& opts, HyperTree &); - + GlueRuleSynthesizer(Moses::AllOptions const& opts, HyperTree &); + // Synthesize the minimal, monotone rule that can be applied to the given // hyperedge and add it to the rule trie. void SynthesizeRule(const Forest::Hyperedge &); diff --git a/moses/Syntax/T2S/GlueRuleSynthesizer.cpp b/moses/Syntax/T2S/GlueRuleSynthesizer.cpp index 04b5da4e7..6f3082654 100644 --- a/moses/Syntax/T2S/GlueRuleSynthesizer.cpp +++ b/moses/Syntax/T2S/GlueRuleSynthesizer.cpp @@ -12,7 +12,7 @@ namespace Syntax namespace T2S { -void +void GlueRuleSynthesizer:: SynthesizeRule(const InputTree::Node &node) { diff --git a/moses/Syntax/T2S/GlueRuleSynthesizer.h b/moses/Syntax/T2S/GlueRuleSynthesizer.h index 3d9e4f8f9..3930db8c4 100644 --- a/moses/Syntax/T2S/GlueRuleSynthesizer.h +++ b/moses/Syntax/T2S/GlueRuleSynthesizer.h @@ -18,11 +18,11 @@ class GlueRuleSynthesizer : public RuleTrieCreator { Word m_output_default_nonterminal; public: - GlueRuleSynthesizer(RuleTrie &trie, Word dflt_nonterm) + GlueRuleSynthesizer(RuleTrie &trie, Word dflt_nonterm) : m_ruleTrie(trie) , m_output_default_nonterminal(dflt_nonterm) {} - + // Synthesize the minimal, montone rule that can be applied to the given node // and add it to the rule trie. void SynthesizeRule(const InputTree::Node &); diff --git a/moses/TranslationTask.cpp b/moses/TranslationTask.cpp index 470d99f60..75df7443b 100644 --- a/moses/TranslationTask.cpp +++ b/moses/TranslationTask.cpp @@ -77,7 +77,7 @@ TranslationTask boost::shared_ptr<IOWrapper> const& ioWrapper) : m_source(source) , m_ioWrapper(ioWrapper) { - m_options = source->options(); + m_options = source->options(); } TranslationTask::~TranslationTask() @@ -104,7 +104,7 @@ TranslationTask else if (algo == SyntaxS2T) { // new-style string-to-tree decoding (ask Phil Williams) - S2TParsingAlgorithm algorithm = m_options->syntax.s2t_parsing_algo; + S2TParsingAlgorithm algorithm = m_options->syntax.s2t_parsing_algo; if (algorithm == RecursiveCYKPlus) { typedef Syntax::S2T::EagerParserCallback Callback; typedef Syntax::S2T::RecursiveCYKPlusParser<Callback> Parser; diff --git a/moses/TreeInput.cpp b/moses/TreeInput.cpp index 1c898ea51..a6a7f80d6 100644 --- a/moses/TreeInput.cpp +++ b/moses/TreeInput.cpp @@ -245,7 +245,6 @@ Read(std::istream& in) string line; if (getline(in, line, '\n').eof()) return 0; - m_labelledSpans.clear(); ProcessAndStripXMLTags(*m_options, line, m_labelledSpans, m_xmlOptions); @@ -265,7 +264,7 @@ Read(std::istream& in) // do source labels vector<XMLParseOutput>::const_iterator iterLabel; - for (iterLabel = m_labelledSpans.begin(); + for (iterLabel = m_labelledSpans.begin(); iterLabel != m_labelledSpans.end(); ++iterLabel) { const XMLParseOutput &labelItem = *iterLabel; const Range &range = labelItem.m_range; @@ -279,7 +278,7 @@ Read(std::istream& in) for (size_t endPos = startPos; endPos < sourceSize; ++endPos) { NonTerminalSet &list = GetLabelSet(startPos, endPos); if (list.size() == 0 || ! only4empty ) { - AddChartLabel(startPos, endPos, m_options->syntax.input_default_non_terminal); + AddChartLabel(startPos, endPos, m_options->syntax.input_default_non_terminal); } } } @@ -300,7 +299,7 @@ TranslationOptionCollection* TreeInput::CreateTranslationOptionCollection() cons return NULL; } -void +void TreeInput:: AddChartLabel(size_t startPos, size_t endPos, const Word &label) { @@ -324,14 +323,14 @@ AddChartLabel(size_t startPos, size_t endPos, const Word &label) } } -void +void TreeInput:: AddChartLabel(size_t startPos, size_t endPos, const string &label) { const std::vector<FactorType>& fOrder = m_options->input.factor_order; Word word(true); - const Factor *factor - = FactorCollection::Instance().AddFactor(Input, fOrder[0], label, true); + const Factor *factor + = FactorCollection::Instance().AddFactor(Input, fOrder[0], label, true); // TODO - no factors word.SetFactor(0, factor); AddChartLabel(startPos, endPos, word); diff --git a/moses/XmlOption.cpp b/moses/XmlOption.cpp index da1339b95..8a517386f 100644 --- a/moses/XmlOption.cpp +++ b/moses/XmlOption.cpp @@ -159,7 +159,7 @@ vector<string> TokenizeXml(const string& str, const std::string& lbrackStr, cons * \param rbrackStr xml tag's right bracket string, typically ">" */ bool -ProcessAndStripXMLTags(AllOptions const& opts, string &line, +ProcessAndStripXMLTags(AllOptions const& opts, string &line, vector<XmlOption const*> &res, ReorderingConstraint &reorderingConstraint, vector< size_t > &walls, diff --git a/moses/XmlOption.h b/moses/XmlOption.h index d5f399763..00bebc2e6 100644 --- a/moses/XmlOption.h +++ b/moses/XmlOption.h @@ -32,8 +32,8 @@ std::vector<std::string> TokenizeXml(const std::string& str, const std::string& bool ProcessAndStripXMLTags(AllOptions const& opts, std::string &line, std::vector<XmlOption const*> &res, - ReorderingConstraint &reorderingConstraint, - std::vector< size_t > &walls, + ReorderingConstraint &reorderingConstraint, + std::vector< size_t > &walls, std::vector< std::pair<size_t, std::string> > &placeholders); diff --git a/phrase-extract/extract-ghkm/Alignment.cpp b/phrase-extract/Alignment.cpp index d12f9398b..cef7a4638 100644 --- a/phrase-extract/extract-ghkm/Alignment.cpp +++ b/phrase-extract/Alignment.cpp @@ -27,10 +27,6 @@ namespace MosesTraining { -namespace Syntax -{ -namespace GHKM -{ void ReadAlignment(const std::string &s, Alignment &a) { @@ -46,7 +42,7 @@ void ReadAlignment(const std::string &s, Alignment &a) } int src = std::atoi(s.substr(begin, end-begin).c_str()); if (end+1 == s.size()) { - throw Exception("Target index missing"); + throw Syntax::Exception("Target index missing"); } begin = end+1; @@ -71,6 +67,4 @@ void FlipAlignment(Alignment &a) } } -} // namespace GHKM -} // namespace Syntax } // namespace MosesTraining diff --git a/phrase-extract/extract-ghkm/Alignment.h b/phrase-extract/Alignment.h index da1279f8f..c25896e6d 100644 --- a/phrase-extract/extract-ghkm/Alignment.h +++ b/phrase-extract/Alignment.h @@ -25,10 +25,6 @@ namespace MosesTraining { -namespace Syntax -{ -namespace GHKM -{ typedef std::vector<std::pair<int, int> > Alignment; @@ -36,6 +32,4 @@ void ReadAlignment(const std::string &, Alignment &); void FlipAlignment(Alignment &); -} // namespace GHKM -} // namespace Syntax } // namespace MosesTraining diff --git a/phrase-extract/ExtractedRule.h b/phrase-extract/ExtractedRule.h index cb2f2261d..adbde43e8 100644 --- a/phrase-extract/ExtractedRule.h +++ b/phrase-extract/ExtractedRule.h @@ -26,6 +26,8 @@ #include <sstream> #include <map> +#include "PhraseOrientation.h" + namespace MosesTraining { @@ -37,8 +39,6 @@ public: std::string target; std::string alignment; std::string alignmentInv; - std::string orientation; - std::string orientationForward; std::string sourceContextLeft; std::string sourceContextRight; std::string targetContextLeft; @@ -51,14 +51,14 @@ public: int endS; float count; double pcfgScore; + PhraseOrientation::REO_CLASS l2rOrientation; + PhraseOrientation::REO_CLASS r2lOrientation; ExtractedRule(int sT, int eT, int sS, int eS) : source() , target() , alignment() , alignmentInv() - , orientation() - , orientationForward() , sourceContextLeft() , sourceContextRight() , targetContextLeft() @@ -70,8 +70,10 @@ public: , startS(sS) , endS(eS) , count(0) - , pcfgScore(0.0) { - } + , pcfgScore(0.0) + , l2rOrientation(PhraseOrientation::REO_CLASS_UNKNOWN) + , r2lOrientation(PhraseOrientation::REO_CLASS_UNKNOWN) + { } }; } diff --git a/phrase-extract/extract-ghkm/PhraseOrientation.cpp b/phrase-extract/PhraseOrientation.cpp index f07e19a46..20297b4de 100644 --- a/phrase-extract/extract-ghkm/PhraseOrientation.cpp +++ b/phrase-extract/PhraseOrientation.cpp @@ -28,10 +28,6 @@ namespace MosesTraining { -namespace Syntax -{ -namespace GHKM -{ std::vector<float> PhraseOrientation::m_l2rOrientationPriorCounts = boost::assign::list_of(0)(0)(0)(0)(0); std::vector<float> PhraseOrientation::m_r2lOrientationPriorCounts = boost::assign::list_of(0)(0)(0)(0)(0); @@ -100,6 +96,18 @@ PhraseOrientation::PhraseOrientation(int sourceSize, Init(sourceSize, targetSize, m_alignedToT, alignedToS, alignedCountS); } +PhraseOrientation::PhraseOrientation(int sourceSize, + int targetSize, + const std::vector<std::vector<int> > &alignedToT, + const std::vector<std::vector<int> > &alignedToS, + const std::vector<int> &alignedCountS) + : m_countF(sourceSize) + , m_countE(targetSize) + , m_alignedToT(alignedToT) +{ + Init(sourceSize, targetSize, m_alignedToT, alignedToS, alignedCountS); +} + void PhraseOrientation::Init(int sourceSize, int targetSize, @@ -470,6 +478,4 @@ void PhraseOrientation::WritePriorCounts(std::ostream& out, const REO_MODEL_TYPE } } -} // namespace GHKM -} // namespace Syntax } // namespace MosesTraining diff --git a/phrase-extract/extract-ghkm/PhraseOrientation.h b/phrase-extract/PhraseOrientation.h index d956e2bc8..1cdfb65be 100644 --- a/phrase-extract/extract-ghkm/PhraseOrientation.h +++ b/phrase-extract/PhraseOrientation.h @@ -32,10 +32,6 @@ namespace MosesTraining { -namespace Syntax -{ -namespace GHKM -{ // The key of the map is the English index and the value is a set of the source ones typedef std::map <int, std::set<int> > HSentenceVertices; @@ -49,6 +45,7 @@ public: enum REO_CLASS {REO_CLASS_LEFT, REO_CLASS_RIGHT, REO_CLASS_DLEFT, REO_CLASS_DRIGHT, REO_CLASS_UNKNOWN}; enum REO_DIR {REO_DIR_L2R, REO_DIR_R2L, REO_DIR_BIDIR}; + PhraseOrientation() {}; PhraseOrientation(int sourceSize, int targetSize, @@ -59,6 +56,12 @@ public: const Moses::AlignmentInfo &alignTerm, const Moses::AlignmentInfo &alignNonTerm); + PhraseOrientation(int sourceSize, + int targetSize, + const std::vector<std::vector<int> > &alignedToT, + const std::vector<std::vector<int> > &alignedToS, + const std::vector<int> &alignedCountS); + REO_CLASS GetOrientationInfo(int startF, int endF, REO_DIR direction) const; REO_CLASS GetOrientationInfo(int startF, int startE, int endF, int endE, REO_DIR direction) const; const std::string GetOrientationInfoString(int startF, int endF, REO_DIR direction=REO_DIR_BIDIR) const; @@ -104,8 +107,8 @@ private: return first < second; }; - const int m_countF; - const int m_countE; + int m_countF; + int m_countE; std::vector<std::vector<int> > m_alignedToT; @@ -121,6 +124,4 @@ private: static std::vector<float> m_r2lOrientationPriorCounts; }; -} // namespace GHKM -} // namespace Syntax } // namespace MosesTraining diff --git a/phrase-extract/RuleExtractionOptions.h b/phrase-extract/RuleExtractionOptions.h index b38258470..aab059cf9 100644 --- a/phrase-extract/RuleExtractionOptions.h +++ b/phrase-extract/RuleExtractionOptions.h @@ -54,6 +54,7 @@ public: bool conditionOnTargetLhs; bool boundaryRules; bool flexScoreFlag; + bool phraseOrientation; RuleExtractionOptions() : maxSpan(10) @@ -86,7 +87,8 @@ public: , unpairedExtractFormat(false) , conditionOnTargetLhs(false) , boundaryRules(false) - , flexScoreFlag(false) {} + , flexScoreFlag(false) + , phraseOrientation(false) {} }; } diff --git a/phrase-extract/extract-rules-main.cpp b/phrase-extract/extract-rules-main.cpp index e6fff965d..62dbbbf0e 100644 --- a/phrase-extract/extract-rules-main.cpp +++ b/phrase-extract/extract-rules-main.cpp @@ -46,6 +46,7 @@ #include "XmlTree.h" #include "InputFileStream.h" #include "OutputFileStream.h" +#include "PhraseOrientation.h" using namespace std; using namespace MosesTraining; @@ -62,6 +63,7 @@ private: Moses::OutputFileStream& m_extractFileInv; Moses::OutputFileStream& m_extractFileContext; Moses::OutputFileStream& m_extractFileContextInv; + PhraseOrientation m_phraseOrientation; vector< ExtractedRule > m_extractedRules; @@ -109,6 +111,7 @@ public: void collectWordLabelCounts(SentenceAlignmentWithSyntax &sentence ); void writeGlueGrammar(const string &, RuleExtractionOptions &options, set< string > &targetLabelCollection, map< string, int > &targetTopLabelCollection); void writeUnknownWordLabel(const string &); +void writePhraseOrientationPriors(const string &); double getPcfgScore(const SyntaxNode &); @@ -142,7 +145,8 @@ int main(int argc, char* argv[]) << " | --UnpairedExtractFormat" << " | --ConditionOnTargetLHS ]" << " | --BoundaryRules[" << options.boundaryRules << "]" - << " | --FlexibilityScore\n"; + << " | --FlexibilityScore" + << " | --PhraseOrientation\n"; exit(1); } @@ -267,6 +271,8 @@ int main(int argc, char* argv[]) options.conditionOnTargetLhs = true; } else if (strcmp(argv[i],"--FlexibilityScore") == 0) { options.flexScoreFlag = true; + } else if (strcmp(argv[i],"--PhraseOrientation") == 0) { + options.phraseOrientation = true; } else if (strcmp(argv[i],"-threads") == 0 || strcmp(argv[i],"--threads") == 0 || strcmp(argv[i],"--Threads") == 0) { @@ -377,6 +383,11 @@ int main(int argc, char* argv[]) if (options.unknownWordLabelFlag) writeUnknownWordLabel(fileNameUnknownWordLabel); + + if (options.phraseOrientation) { + std::string fileNamePhraseOrientationPriors = fileNameExtract + string(".phraseOrientationPriors"); + writePhraseOrientationPriors(fileNamePhraseOrientationPriors); + } } void ExtractTask::Run() @@ -392,6 +403,12 @@ void ExtractTask::extractRules() int countT = m_sentence.target.size(); int countS = m_sentence.source.size(); + // initialize phrase orientation scoring object (for lexicalized reordering model) + if (m_options.phraseOrientation) { + m_sentence.invertAlignment(); // fill m_sentence.alignedToS + m_phraseOrientation = PhraseOrientation(countS, countT, m_sentence.alignedToT, m_sentence.alignedToS, m_sentence.alignedCountS); + } + // phrase repository for creating hiero phrases RuleExist ruleExist(countT); @@ -990,6 +1007,10 @@ void ExtractTask::addRule( int startT, int endT, int startS, int endS, int count } } + rule.alignment.erase(rule.alignment.size()-1); + if (!m_options.onlyDirectFlag) + rule.alignmentInv.erase(rule.alignmentInv.size()-1); + // context (words to left and right) if (m_options.flexScoreFlag) { rule.sourceContextLeft = startS == 0 ? "<s>" : m_sentence.source[startS-1]; @@ -998,9 +1019,14 @@ void ExtractTask::addRule( int startT, int endT, int startS, int endS, int count rule.targetContextRight = endT+1 == m_sentence.target.size() ? "<s>" : m_sentence.target[endT+1]; } - rule.alignment.erase(rule.alignment.size()-1); - if (!m_options.onlyDirectFlag) - rule.alignmentInv.erase(rule.alignmentInv.size()-1); + // phrase orientation (lexicalized reordering model) + if (m_options.phraseOrientation) { + rule.l2rOrientation = m_phraseOrientation.GetOrientationInfo(startS,endS,PhraseOrientation::REO_DIR_L2R); + rule.r2lOrientation = m_phraseOrientation.GetOrientationInfo(startS,endS,PhraseOrientation::REO_DIR_R2L); + // std::cerr << "span " << startS << " " << endS << std::endl; + // std::cerr << "phraseOrientationL2R " << m_phraseOrientation.GetOrientationInfo(startS,endS,PhraseOrientation::REO_DIR_L2R) << std::endl; + // std::cerr << "phraseOrientationR2L " << m_phraseOrientation.GetOrientationInfo(startS,endS,PhraseOrientation::REO_DIR_R2L) << std::endl; + } addRuleToCollection( rule ); } @@ -1070,6 +1096,15 @@ void ExtractTask::writeRulesToFile() if (m_options.pcfgScore) { out << " ||| " << rule->pcfgScore; } + if (m_options.phraseOrientation) { + out << " {{Orientation "; + m_phraseOrientation.WriteOrientation(out,rule->l2rOrientation); + out << " "; + m_phraseOrientation.WriteOrientation(out,rule->r2lOrientation); + m_phraseOrientation.IncrementPriorCount(PhraseOrientation::REO_DIR_L2R,rule->l2rOrientation,1); + m_phraseOrientation.IncrementPriorCount(PhraseOrientation::REO_DIR_R2L,rule->r2lOrientation,1); + out << "}}"; + } out << "\n"; if (!m_options.onlyDirectFlag) { @@ -1119,10 +1154,14 @@ void writeGlueGrammar( const string & fileName, RuleExtractionOptions &options, { ofstream grammarFile; grammarFile.open(fileName.c_str()); + std::string glueRulesPhraseProperty = ""; + if (options.phraseOrientation) { + glueRulesPhraseProperty.append(" ||| ||| {{Orientation 0.25 0.25 0.25 0.25 0.25 0.25 0.25 0.25}}"); + } if (!options.targetSyntax) { - grammarFile << "<s> [X] ||| <s> [S] ||| 1 ||| 0-0 ||| 0" << endl - << "[X][S] </s> [X] ||| [X][S] </s> [S] ||| 1 ||| 0-0 1-1 ||| 0" << endl - << "[X][S] [X][X] [X] ||| [X][S] [X][X] [S] ||| 2.718 ||| 0-0 1-1 ||| 0" << endl; + grammarFile << "<s> [X] ||| <s> [S] ||| 1 ||| 0-0 ||| 0" << glueRulesPhraseProperty << endl + << "[X][S] </s> [X] ||| [X][S] </s> [S] ||| 1 ||| 0-0 1-1 ||| 0" << glueRulesPhraseProperty << endl + << "[X][S] [X][X] [X] ||| [X][S] [X][X] [S] ||| 2.718 ||| 0-0 1-1 ||| 0" << glueRulesPhraseProperty << endl; } else { // chose a top label that is not already a label string topLabel = "QQQQQQ"; @@ -1196,6 +1235,14 @@ void writeUnknownWordLabel(const string & fileName) outFile.close(); } +void writePhraseOrientationPriors(const string &fileName) +{ + ofstream outFile; + outFile.open(fileName.c_str()); + PhraseOrientation::WritePriorCounts(outFile); + outFile.close(); +} + double getPcfgScore(const SyntaxNode &node) { double score = 0.0f; diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl index fe8a3c9ab..4b9b52e11 100755 --- a/scripts/ems/experiment.perl +++ b/scripts/ems/experiment.perl @@ -2364,6 +2364,12 @@ sub define_training_extract_phrases { $cmd .= "-unknown-word-soft-matches $unknown_word_soft_matches "; } + if (&get("TRAINING:phrase-orientation")) { + $cmd .= "-phrase-orientation "; + my $phrase_orientation_priors_file = &versionize(&long_file_name("phrase-orientation-priors","model","")); + $cmd .= "-phrase-orientation-priors-file $phrase_orientation_priors_file "; + } + if (&get("TRAINING:use-ghkm")) { $cmd .= "-ghkm "; } @@ -2372,12 +2378,6 @@ sub define_training_extract_phrases { $cmd .= "-ghkm-tree-fragments "; } - if (&get("TRAINING:ghkm-phrase-orientation")) { - $cmd .= "-ghkm-phrase-orientation "; - my $phrase_orientation_priors_file = &versionize(&long_file_name("phrase-orientation-priors","model","")); - $cmd .= "-phrase-orientation-priors-file $phrase_orientation_priors_file "; - } - if (&get("TRAINING:ghkm-source-labels")) { $cmd .= "-ghkm-source-labels "; my $source_labels_file = &versionize(&long_file_name("source-labels","model","")); @@ -2427,16 +2427,16 @@ sub define_training_build_ttable { if (&get("TRAINING:hierarchical-rule-set")) { - if (&get("TRAINING:ghkm-tree-fragments")) { - $cmd .= "-ghkm-tree-fragments "; - } - - if (&get("TRAINING:ghkm-phrase-orientation")) { - $cmd .= "-ghkm-phrase-orientation "; + if (&get("TRAINING:phrase-orientation")) { + $cmd .= "-phrase-orientation "; my $phrase_orientation_priors_file = &versionize(&long_file_name("phrase-orientation-priors","model","")); $cmd .= "-phrase-orientation-priors-file $phrase_orientation_priors_file "; } + if (&get("TRAINING:ghkm-tree-fragments")) { + $cmd .= "-ghkm-tree-fragments "; + } + if (&get("TRAINING:ghkm-source-labels")) { $cmd .= "-ghkm-source-labels "; my $source_labels_file = &versionize(&long_file_name("source-labels","model","")); @@ -2640,6 +2640,10 @@ sub define_training_create_config { } } + if (&get("TRAINING:phrase-orientation")) { + $cmd .= "-phrase-orientation "; + } + if (&get("TRAINING:ghkm-source-labels")) { $cmd .= "-ghkm-source-labels "; my $source_labels_file = &versionize(&long_file_name("source-labels","model","")); diff --git a/scripts/training/train-model.perl b/scripts/training/train-model.perl index 76f661712..b8a2c13b9 100755 --- a/scripts/training/train-model.perl +++ b/scripts/training/train-model.perl @@ -93,10 +93,10 @@ my($_EXTERNAL_BINDIR, $_GLUE_GRAMMAR_FILE, $_DONT_TUNE_GLUE_GRAMMAR, $_UNKNOWN_WORD_LABEL_FILE, + $_PHRASE_ORIENTATION, + $_PHRASE_ORIENTATION_PRIORS_FILE, $_GHKM, $_GHKM_TREE_FRAGMENTS, - $_GHKM_PHRASE_ORIENTATION, - $_PHRASE_ORIENTATION_PRIORS_FILE, $_GHKM_SOURCE_LABELS, $_GHKM_SOURCE_LABELS_FILE, $_GHKM_PARTS_OF_SPEECH, @@ -210,10 +210,10 @@ $_HELP = 1 'dont-tune-glue-grammar' => \$_DONT_TUNE_GLUE_GRAMMAR, 'unknown-word-label-file=s' => \$_UNKNOWN_WORD_LABEL_FILE, 'unknown-word-soft-matches-file=s' => \$_UNKNOWN_WORD_SOFT_MATCHES_FILE, # give dummy label to unknown word, and allow soft matches to all other labels (with cost determined by sparse features) + 'phrase-orientation' => \$_PHRASE_ORIENTATION, + 'phrase-orientation-priors-file=s' => \$_PHRASE_ORIENTATION_PRIORS_FILE, # currently relevant for Hiero and GHKM extraction only; phrase orientation for PBT has different implementation 'ghkm' => \$_GHKM, 'ghkm-tree-fragments' => \$_GHKM_TREE_FRAGMENTS, - 'ghkm-phrase-orientation' => \$_GHKM_PHRASE_ORIENTATION, - 'phrase-orientation-priors-file=s' => \$_PHRASE_ORIENTATION_PRIORS_FILE, # currently relevant for GHKM extraction only; phrase orientation for PBT has different implementation 'ghkm-source-labels' => \$_GHKM_SOURCE_LABELS, 'ghkm-source-labels-file=s' => \$_GHKM_SOURCE_LABELS_FILE, 'ghkm-parts-of-speech' => \$_GHKM_PARTS_OF_SPEECH, @@ -1561,11 +1561,11 @@ sub extract_phrase { $cmd .= " --PCFG" if $_PCFG; $cmd .= " --UnpairedExtractFormat" if $_ALT_DIRECT_RULE_SCORE_1 || $_ALT_DIRECT_RULE_SCORE_2; $cmd .= " --ConditionOnTargetLHS" if $_ALT_DIRECT_RULE_SCORE_1; + $cmd .= " --PhraseOrientation" if $_PHRASE_ORIENTATION; + $cmd .= " --PhraseOrientationPriors $_PHRASE_ORIENTATION_PRIORS_FILE" if defined($_PHRASE_ORIENTATION_PRIORS_FILE); if (defined($_GHKM)) { $cmd .= " --TreeFragments" if $_GHKM_TREE_FRAGMENTS; - $cmd .= " --PhraseOrientation" if $_GHKM_PHRASE_ORIENTATION; - $cmd .= " --PhraseOrientationPriors $_PHRASE_ORIENTATION_PRIORS_FILE" if defined($_PHRASE_ORIENTATION_PRIORS_FILE); $cmd .= " --SourceLabels" if $_GHKM_SOURCE_LABELS; $cmd .= " --PartsOfSpeech" if $_GHKM_PARTS_OF_SPEECH; $cmd .= " --PartsOfSpeechFactor" if $_GHKM_PARTS_OF_SPEECH_FACTOR; @@ -1701,6 +1701,7 @@ sub score_phrase_phrase_extract { my $LOG_PROB = (defined($_SCORE_OPTIONS) && $_SCORE_OPTIONS =~ /LogProb/); my $NEG_LOG_PROB = (defined($_SCORE_OPTIONS) && $_SCORE_OPTIONS =~ /NegLogProb/); my $NO_LEX = (defined($_SCORE_OPTIONS) && $_SCORE_OPTIONS =~ /NoLex/); + my $MIN_COUNT = (defined($_SCORE_OPTIONS) && $_SCORE_OPTIONS =~ /MinCount ([\d\.]+)/) ? $1 : undef; my $MIN_COUNT_HIERARCHICAL = (defined($_SCORE_OPTIONS) && $_SCORE_OPTIONS =~ /MinCountHierarchical ([\d\.]+)/) ? $1 : undef; my $SOURCE_LABELS = (defined($_SCORE_OPTIONS) && $_SCORE_OPTIONS =~ /SourceLabels/); my $SOURCE_LABEL_COUNTS_LHS = (defined($_SCORE_OPTIONS) && $_SCORE_OPTIONS =~ /SourceLabelCountsLHS/); @@ -1748,13 +1749,14 @@ sub score_phrase_phrase_extract { $cmd .= " --SpanLength" if $SPAN_LENGTH && $inverse eq ""; $cmd .= " --UnalignedPenalty" if $UNALIGNED_COUNT; $cmd .= " --UnalignedFunctionWordPenalty ".($inverse ? $UNALIGNED_FW_F : $UNALIGNED_FW_E) if $UNALIGNED_FW_COUNT; + $cmd .= " --MinCount $MIN_COUNT" if $MIN_COUNT; $cmd .= " --MinCountHierarchical $MIN_COUNT_HIERARCHICAL" if $MIN_COUNT_HIERARCHICAL; $cmd .= " --PCFG" if $_PCFG; $cmd .= " --UnpairedExtractFormat" if $_ALT_DIRECT_RULE_SCORE_1 || $_ALT_DIRECT_RULE_SCORE_2; $cmd .= " --ConditionOnTargetLHS" if $_ALT_DIRECT_RULE_SCORE_1; $cmd .= " --TreeFragments" if $_GHKM_TREE_FRAGMENTS; - $cmd .= " --PhraseOrientation" if $_GHKM_PHRASE_ORIENTATION; - $cmd .= " --PhraseOrientationPriors $_PHRASE_ORIENTATION_PRIORS_FILE" if $_GHKM_PHRASE_ORIENTATION && defined($_PHRASE_ORIENTATION_PRIORS_FILE); + $cmd .= " --PhraseOrientation" if $_PHRASE_ORIENTATION; + $cmd .= " --PhraseOrientationPriors $_PHRASE_ORIENTATION_PRIORS_FILE" if $_PHRASE_ORIENTATION && defined($_PHRASE_ORIENTATION_PRIORS_FILE); $cmd .= " --SourceLabels $_GHKM_SOURCE_LABELS_FILE" if $_GHKM_SOURCE_LABELS && defined($_GHKM_SOURCE_LABELS_FILE); $cmd .= " --PartsOfSpeech $_GHKM_PARTS_OF_SPEECH_FILE" if $_GHKM_PARTS_OF_SPEECH && defined($_GHKM_PARTS_OF_SPEECH_FILE); $cmd .= " $DOMAIN" if $DOMAIN; @@ -2365,6 +2367,7 @@ sub create_ini { print INI "PhrasePenalty\n"; print INI "SoftMatchingFeature name=SM0 path=$_UNKNOWN_WORD_SOFT_MATCHES_FILE\n" if $_TARGET_SYNTAX && defined($_UNKNOWN_WORD_SOFT_MATCHES_FILE); print INI "SoftSourceSyntacticConstraintsFeature sourceLabelSetFile=$_GHKM_SOURCE_LABELS_FILE\n" if $_GHKM_SOURCE_LABELS && defined($_GHKM_SOURCE_LABELS_FILE); + print INI "PhraseOrientationFeature\n" if $_PHRASE_ORIENTATION; print INI $feature_spec; print INI "\n# dense weights for feature functions\n"; @@ -2375,6 +2378,7 @@ sub create_ini { print INI "WordPenalty0= -1\n"; print INI "PhrasePenalty0= 0.2\n"; print INI "SoftSourceSyntacticConstraintsFeature0= -0.2 -0.2 -0.2 0.1 0.1 0.1\n" if $_GHKM_SOURCE_LABELS && defined($_GHKM_SOURCE_LABELS_FILE); + print INI "PhraseOrientationFeature0= 0.05 0.05 0.05 0.05 0.05 0.05\n" if $_PHRASE_ORIENTATION; print INI $weight_spec; close(INI); } diff --git a/vw/Classifier.h b/vw/Classifier.h index 31bd6ff82..39b3461ad 100644 --- a/vw/Classifier.h +++ b/vw/Classifier.h @@ -83,8 +83,8 @@ protected: // some of VW settings are hard-coded because they are always needed in our scenario // (e.g. quadratic source X target features) -const std::string VW_DEFAULT_OPTIONS = " --hash all --noconstant -q st -t --ldf_override s "; -const std::string VW_DEFAULT_PARSER_OPTIONS = " --quiet --hash all --noconstant -q st -t --csoaa_ldf s "; +const std::string VW_DEFAULT_OPTIONS = " --hash all --noconstant -q st -t --ldf_override sc "; +const std::string VW_DEFAULT_PARSER_OPTIONS = " --quiet --hash all --noconstant -q st -t --csoaa_ldf sc "; /** * Produce VW training file (does not use the VW library!) |