diff options
author | Rico Sennrich <rico.sennrich@gmx.ch> | 2013-09-08 01:04:01 +0400 |
---|---|---|
committer | Rico Sennrich <rico.sennrich@gmx.ch> | 2013-09-08 01:04:01 +0400 |
commit | 7138056b8fd60b4572f016f038360dd5fb19a39c (patch) | |
tree | bb14de73775788233ff8aeb800da016365358826 /phrase-extract | |
parent | b27d140619ff7340d7cac26b9ab8fb368c460af7 (diff) |
flexibility scores
Diffstat (limited to 'phrase-extract')
-rw-r--r-- | phrase-extract/ExtractedRule.h | 12 | ||||
-rw-r--r-- | phrase-extract/PhraseExtractionOptions.h | 10 | ||||
-rw-r--r-- | phrase-extract/RuleExtractionOptions.h | 5 | ||||
-rw-r--r-- | phrase-extract/extract-main.cpp | 96 | ||||
-rw-r--r-- | phrase-extract/extract-rules-main.cpp | 112 |
5 files changed, 225 insertions, 10 deletions
diff --git a/phrase-extract/ExtractedRule.h b/phrase-extract/ExtractedRule.h index 8e3513582..a6cd5074d 100644 --- a/phrase-extract/ExtractedRule.h +++ b/phrase-extract/ExtractedRule.h @@ -41,6 +41,12 @@ public: std::string alignmentInv; std::string orientation; std::string orientationForward; + std::string sourceContextLeft; + std::string sourceContextRight; + std::string targetContextLeft; + std::string targetContextRight; + std::string sourceHoleString; + std::string targetHoleString; int startT; int endT; int startS; @@ -57,6 +63,12 @@ public: , alignmentInv() , orientation() , orientationForward() + , sourceContextLeft() + , sourceContextRight() + , targetContextLeft() + , targetContextRight() + , sourceHoleString() + , targetHoleString() , startT(sT) , endT(eT) , startS(sS) diff --git a/phrase-extract/PhraseExtractionOptions.h b/phrase-extract/PhraseExtractionOptions.h index 38063108b..a410473f4 100644 --- a/phrase-extract/PhraseExtractionOptions.h +++ b/phrase-extract/PhraseExtractionOptions.h @@ -48,6 +48,7 @@ private: bool onlyOutputSpanInfo; bool gzOutput; std::string instanceWeightsFile; //weights for each sentence + bool flexScoreFlag; public: std::vector<std::string> placeholders; @@ -65,7 +66,8 @@ public: translationFlag(true), includeSentenceIdFlag(false), onlyOutputSpanInfo(false), - gzOutput(false) {} + gzOutput(false), + flexScoreFlag(false) {} //functions for initialization of options void initAllModelsOutputFlag(const bool initallModelsOutputFlag) { @@ -107,6 +109,9 @@ public: void initInstanceWeightsFile(const char* initInstanceWeightsFile) { instanceWeightsFile = std::string(initInstanceWeightsFile); } + void initFlexScoreFlag(const bool initflexScoreFlag){ + flexScoreFlag=initflexScoreFlag; + } // functions for getting values bool isAllModelsOutputFlag() const { @@ -148,6 +153,9 @@ public: std::string getInstanceWeightsFile() const { return instanceWeightsFile; } + bool isFlexScoreFlag() const { + return flexScoreFlag; + } }; } diff --git a/phrase-extract/RuleExtractionOptions.h b/phrase-extract/RuleExtractionOptions.h index d437c679c..a9b0ce9e6 100644 --- a/phrase-extract/RuleExtractionOptions.h +++ b/phrase-extract/RuleExtractionOptions.h @@ -54,6 +54,7 @@ public: bool unpairedExtractFormat; bool conditionOnTargetLhs; bool boundaryRules; + bool flexScoreFlag; RuleExtractionOptions() : maxSpan(10) @@ -86,8 +87,8 @@ public: , gzOutput(false) , unpairedExtractFormat(false) , conditionOnTargetLhs(false) - , boundaryRules(false) { - } + , boundaryRules(false) + , flexScoreFlag(false) {} }; } diff --git a/phrase-extract/extract-main.cpp b/phrase-extract/extract-main.cpp index 4804f83b8..5bc018173 100644 --- a/phrase-extract/extract-main.cpp +++ b/phrase-extract/extract-main.cpp @@ -80,6 +80,8 @@ int sentenceOffset = 0; std::vector<std::string> Tokenize(const std::string& str, const std::string& delimiters = " \t"); +bool flexScoreFlag = false; + } namespace MosesTraining @@ -88,18 +90,22 @@ namespace MosesTraining class ExtractTask { public: - ExtractTask(size_t id, SentenceAlignment &sentence,PhraseExtractionOptions &initoptions, Moses::OutputFileStream &extractFile, Moses::OutputFileStream &extractFileInv,Moses::OutputFileStream &extractFileOrientation): + ExtractTask(size_t id, SentenceAlignment &sentence,PhraseExtractionOptions &initoptions, Moses::OutputFileStream &extractFile, Moses::OutputFileStream &extractFileInv,Moses::OutputFileStream &extractFileOrientation, Moses::OutputFileStream &extractFileContext, Moses::OutputFileStream &extractFileContextInv): m_sentence(sentence), m_options(initoptions), m_extractFile(extractFile), m_extractFileInv(extractFileInv), - m_extractFileOrientation(extractFileOrientation) {} + m_extractFileOrientation(extractFileOrientation), + m_extractFileContext(extractFileContext), + m_extractFileContextInv(extractFileContextInv) {} void Run(); private: vector< string > m_extractedPhrases; vector< string > m_extractedPhrasesInv; vector< string > m_extractedPhrasesOri; vector< string > m_extractedPhrasesSid; + vector< string > m_extractedPhrasesContext; + vector< string > m_extractedPhrasesContextInv; void extractBase(SentenceAlignment &); void extract(SentenceAlignment &); void addPhrase(SentenceAlignment &, int, int, int, int, string &); @@ -112,6 +118,8 @@ private: Moses::OutputFileStream &m_extractFile; Moses::OutputFileStream &m_extractFileInv; Moses::OutputFileStream &m_extractFileOrientation; + Moses::OutputFileStream &m_extractFileContext; + Moses::OutputFileStream &m_extractFileContextInv; }; } @@ -129,6 +137,8 @@ int main(int argc, char* argv[]) Moses::OutputFileStream extractFile; Moses::OutputFileStream extractFileInv; Moses::OutputFileStream extractFileOrientation; + Moses::OutputFileStream extractFileContext; + Moses::OutputFileStream extractFileContextInv; const char* const &fileNameE = argv[1]; const char* const &fileNameF = argv[2]; const char* const &fileNameA = argv[3]; @@ -140,6 +150,8 @@ int main(int argc, char* argv[]) options.initOnlyOutputSpanInfo(true); } else if (strcmp(argv[i],"orientation") == 0 || strcmp(argv[i],"--Orientation") == 0) { options.initOrientationFlag(true); + } else if (strcmp(argv[i],"--FlexibilityScore") == 0) { + options.initFlexScoreFlag(true); } else if (strcmp(argv[i],"--NoTTable") == 0) { options.initTranslationFlag(false); } else if (strcmp(argv[i], "--IncludeSentenceId") == 0) { @@ -254,8 +266,15 @@ int main(int argc, char* argv[]) string fileNameExtractOrientation = fileNameExtract + ".o" + (options.isGzOutput()?".gz":""); extractFileOrientation.Open(fileNameExtractOrientation.c_str()); } + if (options.isFlexScoreFlag()) { + string fileNameExtractContext = fileNameExtract + ".context" + (options.isGzOutput()?".gz":""); + string fileNameExtractContextInv = fileNameExtract + ".context.inv" + (options.isGzOutput()?".gz":""); + extractFileContext.Open(fileNameExtractContext.c_str()); + extractFileContextInv.Open(fileNameExtractContextInv.c_str()); + } int i = sentenceOffset; + while(true) { i++; if (i%10000 == 0) cerr << "." << flush; @@ -280,7 +299,7 @@ int main(int argc, char* argv[]) cout << "LOG: PHRASES_BEGIN:" << endl; } if (sentence.create( englishString, foreignString, alignmentString, weightString, i, false)) { - ExtractTask *task = new ExtractTask(i-1, sentence, options, extractFile , extractFileInv, extractFileOrientation); + ExtractTask *task = new ExtractTask(i-1, sentence, options, extractFile , extractFileInv, extractFileOrientation, extractFileContext, extractFileContextInv); task->Run(); delete task; @@ -302,6 +321,11 @@ int main(int argc, char* argv[]) if (options.isOrientationFlag()) { extractFileOrientation.Close(); } + + if (options.isFlexScoreFlag()) { + extractFileContext.Close(); + extractFileContextInv.Close(); + } } } @@ -315,6 +339,8 @@ void ExtractTask::Run() m_extractedPhrasesInv.clear(); m_extractedPhrasesOri.clear(); m_extractedPhrasesSid.clear(); + m_extractedPhrasesContext.clear(); + m_extractedPhrasesContextInv.clear(); } @@ -680,6 +706,8 @@ void ExtractTask::addPhrase( SentenceAlignment &sentence, int startE, int endE, ostringstream outextractstr; ostringstream outextractstrInv; ostringstream outextractstrOrientation; + ostringstream outextractstrContext; + ostringstream outextractstrContextInv; if (m_options.isOnlyOutputSpanInfo()) { cout << startF << " " << endF << " " << startE << " " << endE << endl; @@ -693,19 +721,25 @@ void ExtractTask::addPhrase( SentenceAlignment &sentence, int startE, int endE, for(int fi=startF; fi<=endF; fi++) { if (m_options.isTranslationFlag()) outextractstr << sentence.source[fi] << " "; if (m_options.isOrientationFlag()) outextractstrOrientation << sentence.source[fi] << " "; + if (m_options.isFlexScoreFlag()) outextractstrContext << sentence.source[fi] << " "; } if (m_options.isTranslationFlag()) outextractstr << "||| "; if (m_options.isOrientationFlag()) outextractstrOrientation << "||| "; + if (m_options.isFlexScoreFlag()) outextractstrContext << "||| "; // target for(int ei=startE; ei<=endE; ei++) { if (m_options.isTranslationFlag()) outextractstr << sentence.target[ei] << " "; if (m_options.isTranslationFlag()) outextractstrInv << sentence.target[ei] << " "; if (m_options.isOrientationFlag()) outextractstrOrientation << sentence.target[ei] << " "; + if (m_options.isFlexScoreFlag()) outextractstrContext << sentence.target[ei] << " "; + if (m_options.isFlexScoreFlag()) outextractstrContextInv << sentence.target[ei] << " "; } if (m_options.isTranslationFlag()) outextractstr << "|||"; if (m_options.isTranslationFlag()) outextractstrInv << "||| "; if (m_options.isOrientationFlag()) outextractstrOrientation << "||| "; + if (m_options.isFlexScoreFlag()) outextractstrContext << "||| "; + if (m_options.isFlexScoreFlag()) outextractstrContextInv << "||| "; // source (for inverse) @@ -714,6 +748,12 @@ void ExtractTask::addPhrase( SentenceAlignment &sentence, int startE, int endE, outextractstrInv << sentence.source[fi] << " "; outextractstrInv << "|||"; } + if (m_options.isFlexScoreFlag()) { + for(int fi=startF; fi<=endF; fi++) + outextractstrContextInv << sentence.source[fi] << " "; + outextractstrContextInv << "|||"; + } + // alignment if (m_options.isTranslationFlag()) { for(int ei=startE; ei<=endE; ei++) { @@ -743,6 +783,46 @@ void ExtractTask::addPhrase( SentenceAlignment &sentence, int startE, int endE, } + + // generate two lines for every extracted phrase: + // once with left, once with right context + if (m_options.isFlexScoreFlag()) { + + string strContext = outextractstrContext.str(); + string strContextInv = outextractstrContextInv.str(); + + ostringstream outextractstrContextRight(strContext, ostringstream::app); + ostringstream outextractstrContextRightInv(strContextInv, ostringstream::app); + + // write context to left + outextractstrContext << "< "; + if (startF == 0) outextractstrContext << "<s>"; + else outextractstrContext << sentence.source[startF-1]; + + outextractstrContextInv << " < "; + if (startE == 0) outextractstrContextInv << "<s>"; + else outextractstrContextInv << sentence.target[startE-1]; + + // write context to right + outextractstrContextRight << "> "; + if (endF+1 == sentence.source.size()) outextractstrContextRight << "<s>"; + else outextractstrContextRight << sentence.source[endF+1]; + + outextractstrContextRightInv << " > "; + if (endE+1 == sentence.target.size()) outextractstrContextRightInv << "<s>"; + else outextractstrContextRightInv << sentence.target[endE+1]; + + outextractstrContext << "\n"; + outextractstrContextInv << "\n"; + outextractstrContextRight << "\n"; + outextractstrContextRightInv << "\n"; + + m_extractedPhrasesContext.push_back(outextractstrContext.str()); + m_extractedPhrasesContextInv.push_back(outextractstrContextInv.str()); + m_extractedPhrasesContext.push_back(outextractstrContextRight.str()); + m_extractedPhrasesContextInv.push_back(outextractstrContextRightInv.str()); + } + if (m_options.isTranslationFlag()) outextractstr << "\n"; if (m_options.isTranslationFlag()) outextractstrInv << "\n"; if (m_options.isOrientationFlag()) outextractstrOrientation << "\n"; @@ -760,6 +840,8 @@ void ExtractTask::writePhrasesToFile() ostringstream outextractFile; ostringstream outextractFileInv; ostringstream outextractFileOrientation; + ostringstream outextractFileContext; + ostringstream outextractFileContextInv; for(vector<string>::const_iterator phrase=m_extractedPhrases.begin(); phrase!=m_extractedPhrases.end(); phrase++) { outextractFile<<phrase->data(); @@ -770,10 +852,18 @@ void ExtractTask::writePhrasesToFile() for(vector<string>::const_iterator phrase=m_extractedPhrasesOri.begin(); phrase!=m_extractedPhrasesOri.end(); phrase++) { outextractFileOrientation<<phrase->data(); } + for(vector<string>::const_iterator phrase=m_extractedPhrasesContext.begin();phrase!=m_extractedPhrasesContext.end();phrase++){ + outextractFileContext<<phrase->data(); + } + for(vector<string>::const_iterator phrase=m_extractedPhrasesContextInv.begin();phrase!=m_extractedPhrasesContextInv.end();phrase++){ + outextractFileContextInv<<phrase->data(); + } m_extractFile << outextractFile.str(); m_extractFileInv << outextractFileInv.str(); m_extractFileOrientation << outextractFileOrientation.str(); + m_extractFileContext << outextractFileContext.str(); + m_extractFileContextInv << outextractFileContextInv.str(); } // if proper conditioning, we need the number of times a source phrase occured diff --git a/phrase-extract/extract-rules-main.cpp b/phrase-extract/extract-rules-main.cpp index f8e315e2c..97a593085 100644 --- a/phrase-extract/extract-rules-main.cpp +++ b/phrase-extract/extract-rules-main.cpp @@ -62,6 +62,8 @@ private: const RuleExtractionOptions &m_options; Moses::OutputFileStream& m_extractFile; Moses::OutputFileStream& m_extractFileInv; + Moses::OutputFileStream& m_extractFileContext; + Moses::OutputFileStream& m_extractFileContextInv; vector< ExtractedRule > m_extractedRules; @@ -94,11 +96,13 @@ private: } public: - ExtractTask(SentenceAlignmentWithSyntax &sentence, const RuleExtractionOptions &options, Moses::OutputFileStream &extractFile, Moses::OutputFileStream &extractFileInv): + ExtractTask(SentenceAlignmentWithSyntax &sentence, const RuleExtractionOptions &options, Moses::OutputFileStream &extractFile, Moses::OutputFileStream &extractFileInv, Moses::OutputFileStream &extractFileContext, Moses::OutputFileStream &extractFileContextInv): m_sentence(sentence), m_options(options), m_extractFile(extractFile), - m_extractFileInv(extractFileInv) {} + m_extractFileInv(extractFileInv), + m_extractFileContext(extractFileContext), + m_extractFileContextInv(extractFileContextInv) {} void Run(); }; @@ -138,7 +142,8 @@ int main(int argc, char* argv[]) << " | --AllowOnlyUnalignedWords | --DisallowNonTermConsecTarget |--NonTermConsecSource | --NoNonTermFirstWord | --NoFractionalCounting" << " | --UnpairedExtractFormat" << " | --ConditionOnTargetLHS ]" - << " | --BoundaryRules[" << options.boundaryRules << "]"; + << " | --BoundaryRules[" << options.boundaryRules << "]" + << " | --FlexibilityScore\n"; exit(1); } @@ -263,6 +268,8 @@ int main(int argc, char* argv[]) options.unpairedExtractFormat = true; } else if (strcmp(argv[i],"--ConditionOnTargetLHS") == 0) { options.conditionOnTargetLhs = true; + } else if (strcmp(argv[i],"--FlexibilityScore") == 0) { + options.flexScoreFlag = true; } else if (strcmp(argv[i],"-threads") == 0 || strcmp(argv[i],"--threads") == 0 || strcmp(argv[i],"--Threads") == 0) { @@ -301,10 +308,20 @@ int main(int argc, char* argv[]) string fileNameExtractInv = fileNameExtract + ".inv" + (options.gzOutput?".gz":""); Moses::OutputFileStream extractFile; Moses::OutputFileStream extractFileInv; + Moses::OutputFileStream extractFileContext; + Moses::OutputFileStream extractFileContextInv; extractFile.Open((fileNameExtract + (options.gzOutput?".gz":"")).c_str()); if (!options.onlyDirectFlag) extractFileInv.Open(fileNameExtractInv.c_str()); + if (options.flexScoreFlag) { + string fileNameExtractContext = fileNameExtract + ".context" + (options.gzOutput?".gz":""); + extractFileContext.Open(fileNameExtractContext.c_str()); + if (!options.onlyDirectFlag) { + string fileNameExtractContextInv = fileNameExtract + ".context.inv" + (options.gzOutput?".gz":""); + extractFileContextInv.Open(fileNameExtractContextInv.c_str()); + } + } // stats on labels for glue grammar and unknown word label probabilities set< string > targetLabelCollection, sourceLabelCollection; @@ -339,7 +356,7 @@ int main(int argc, char* argv[]) if (options.unknownWordLabelFlag) { collectWordLabelCounts(sentence); } - ExtractTask *task = new ExtractTask(sentence, options, extractFile, extractFileInv); + ExtractTask *task = new ExtractTask(sentence, options, extractFile, extractFileInv, extractFileContext, extractFileContextInv); task->Run(); delete task; } @@ -355,6 +372,11 @@ int main(int argc, char* argv[]) if (!options.onlyDirectFlag) extractFileInv.Close(); } + if (options.flexScoreFlag) { + extractFileContext.Close(); + if (!options.onlyDirectFlag) extractFileContextInv.Close(); + } + if (options.glueGrammarFlag) writeGlueGrammar(fileNameGlueGrammar, options, targetLabelCollection, targetTopLabelCollection); @@ -698,6 +720,46 @@ void ExtractTask::saveHieroPhrase( int startT, int endT, int startS, int endS // alignment saveHieroAlignment(startT, endT, startS, endS, indexS, indexT, holeColl, rule); + // context (words to left and right) + if (m_options.flexScoreFlag) { + rule.sourceContextLeft = startS == 0 ? "<s>" : m_sentence.source[startS-1]; + rule.sourceContextRight = endS+1 == m_sentence.source.size() ? "<s>" : m_sentence.source[endS+1]; + rule.targetContextLeft = startT == 0 ? "<s>" : m_sentence.target[startT-1]; + rule.targetContextRight = endT+1 == m_sentence.target.size() ? "<s>" : m_sentence.target[endT+1]; + rule.sourceHoleString = ""; + rule.targetHoleString = ""; + + HoleList::const_iterator iterHole; + for (iterHole = holeColl.GetHoles().begin(); iterHole != holeColl.GetHoles().end(); ++iterHole) { + const Hole &hole = *iterHole; + rule.sourceHoleString += hole.GetLabel(0) + ": "; + + // rule starts with nonterminal; end of NT is considered left context + if (hole.GetStart(0) == startS) { + rule.sourceContextLeft = m_sentence.source[hole.GetEnd(0)]; + } + // rule ends with nonterminal; start of NT is considered right context + else if (hole.GetEnd(0) == endS) { + rule.sourceContextRight = m_sentence.source[hole.GetStart(0)]; + } + + if (hole.GetStart(1) == startT) { + rule.targetContextLeft = m_sentence.target[hole.GetEnd(1)]; + } + else if (hole.GetEnd(1) == endT) { + rule.targetContextRight = m_sentence.target[hole.GetStart(1)]; + } + + for (int i = hole.GetStart(0); i <= hole.GetEnd(0); ++i) { + rule.sourceHoleString += m_sentence.source[i] + " "; + } + rule.targetHoleString += hole.GetLabel(1) + ": "; + for (int i = hole.GetStart(1); i <= hole.GetEnd(1); ++i) { + rule.targetHoleString += m_sentence.target[i] + " "; + } + } + } + addRuleToCollection( rule ); } @@ -938,6 +1000,14 @@ void ExtractTask::addRule( int startT, int endT, int startS, int endS, int count } } + // context (words to left and right) + if (m_options.flexScoreFlag) { + rule.sourceContextLeft = startS == 0 ? "<s>" : m_sentence.source[startS-1]; + rule.sourceContextRight = endS+1 == m_sentence.source.size() ? "<s>" : m_sentence.source[endS+1]; + rule.targetContextLeft = startT == 0 ? "<s>" : m_sentence.target[startT-1]; + rule.targetContextRight = endT+1 == m_sentence.target.size() ? "<s>" : m_sentence.target[endT+1]; + } + rule.alignment.erase(rule.alignment.size()-1); if (!m_options.onlyDirectFlag) rule.alignmentInv.erase(rule.alignmentInv.size()-1); @@ -997,6 +1067,8 @@ void ExtractTask::writeRulesToFile() vector<ExtractedRule>::const_iterator rule; ostringstream out; ostringstream outInv; + ostringstream outContext; + ostringstream outContextInv; for(rule = m_extractedRules.begin(); rule != m_extractedRules.end(); rule++ ) { if (rule->count == 0) continue; @@ -1019,9 +1091,41 @@ void ExtractTask::writeRulesToFile() << rule->alignmentInv << " ||| " << rule->count << "\n"; } + + if (m_options.flexScoreFlag) { + for(int iContext=0;iContext<2;iContext++){ + outContext << rule->source << " ||| " + << rule->target << " ||| " + << rule->alignment << " ||| "; + iContext ? outContext << "< " << rule->sourceContextLeft << "\n" : outContext << "> " << rule->sourceContextRight << "\n"; + + if (!m_options.onlyDirectFlag) { + outContextInv << rule->target << " ||| " + << rule->source << " ||| " + << rule->alignmentInv << " ||| "; + iContext ? outContextInv << "< " << rule->targetContextLeft << "\n" : outContextInv << "> " << rule->targetContextRight << "\n"; + } + } + + if (rule->sourceHoleString != "") { + outContext << rule->source << " ||| " + << rule->target << " ||| " + << rule->alignment << " ||| v " + << rule->sourceHoleString << "\n"; + } + + if (!m_options.onlyDirectFlag and rule->targetHoleString != "") { + outContextInv << rule->target << " ||| " + << rule->source << " ||| " + << rule->alignmentInv << " ||| v " + << rule->targetHoleString << "\n"; + } + } } m_extractFile << out.str(); m_extractFileInv << outInv.str(); + m_extractFileContext << outContext.str(); + m_extractFileContextInv << outContextInv.str(); } void writeGlueGrammar( const string & fileName, RuleExtractionOptions &options, set< string > &targetLabelCollection, map< string, int > &targetTopLabelCollection ) |